From 911d477b4852cb4cb21ed363b8aef5eee3333479 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 19 Aug 2026 08:44:05 +0200 Subject: [PATCH 1/4] feat: Add HTTPX-based HTTP client --- README.md | 11 +- pyproject.toml | 1 + src/apify_client/http_clients/__init__.py | 39 +++- src/apify_client/http_clients/_httpx.py | 244 ++++++++++++++++++++++ tests/integration/conftest.py | 76 +++++-- tests/integration/test_apify_client.py | 4 + tests/integration/test_dataset.py | 2 + tests/integration/test_key_value_store.py | 2 + tests/integration/test_log.py | 4 + tests/unit/conftest.py | 23 +- tests/unit/test_client_headers.py | 73 ++++++- tests/unit/test_client_streaming.py | 4 +- tests/unit/test_client_timeouts.py | 77 +++++-- tests/unit/test_http_clients.py | 110 ++++++++++ tests/unit/test_logging.py | 4 +- tests/unit/test_pluggable_http_client.py | 46 ++++ uv.lock | 6 +- 17 files changed, 668 insertions(+), 58 deletions(-) create mode 100644 src/apify_client/http_clients/_httpx.py diff --git a/README.md b/README.md index ba5b9084..7627e5fe 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,15 @@ uv add "apify-client[brotli]" ``` + [Impit](https://github.com/apify/impit) is the default HTTP client and is installed automatically. To use the + built-in [HTTPX](https://www.python-httpx.org/) client instead, install its optional extra: + + ```bash + pip install "apify-client[httpx]" + # or + uv add "apify-client[httpx]" + ``` + - From [conda-forge](https://anaconda.org/conda-forge/apify-client), it can be installed with [conda](https://docs.conda.io/en/latest/): ```bash @@ -124,7 +133,7 @@ For a guided walkthrough — authenticating, running an Actor, and reading its r - **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call ([Timeouts](https://docs.apify.com/api/client/python/docs/concepts/timeouts)). - **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)). - **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks ([Convenience methods](https://docs.apify.com/api/client/python/docs/concepts/convenience-methods)). -- **Pluggable HTTP layer** — swap the default [Impit](https://github.com/apify/impit)-based HTTP client for `httpx`, `requests`, `aiohttp`, or any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). +- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX](https://www.python-httpx.org/) client, or plug in any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). - **Structured errors** — every API error surfaces as an [`ApifyApiError`](https://docs.apify.com/api/client/python/reference/class/ApifyApiError) with HTTP-specific subclasses for precise handling ([Error handling](https://docs.apify.com/api/client/python/docs/concepts/error-handling)). - **Debug logging** — opt-in structured logging on the `apify_client` logger captures request URLs, status codes, retry attempts, and more ([Logging](https://docs.apify.com/api/client/python/docs/concepts/logging)). diff --git a/pyproject.toml b/pyproject.toml index 01e15bc8..b8922597 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ [project.optional-dependencies] brotli = ["brotli>=1.0.9"] +httpx = ["httpx>=0.27.0,<1.0.0"] [project.urls] "Apify Homepage" = "https://apify.com" diff --git a/src/apify_client/http_clients/__init__.py b/src/apify_client/http_clients/__init__.py index 417a0705..d1e06c90 100644 --- a/src/apify_client/http_clients/__init__.py +++ b/src/apify_client/http_clients/__init__.py @@ -1,10 +1,35 @@ +from apify_client._utils.try_import import install_import_hook as _install_import_hook +from apify_client._utils.try_import import try_import as _try_import from apify_client.http_clients._base import HttpClient, HttpClientAsync, HttpResponse from apify_client.http_clients._impit import ImpitHttpClient, ImpitHttpClientAsync -__all__ = [ - 'HttpClient', - 'HttpClientAsync', - 'HttpResponse', - 'ImpitHttpClient', - 'ImpitHttpClientAsync', -] +_install_import_hook(__name__) + +# `httpx` is an optional extra, so it's wrapped in try_import. Accessing the HTTPX clients +# without the extra installed raises a clear ImportError instead of failing at package import time. +with _try_import( + __name__, + 'HttpxHttpClient', + 'HttpxHttpClientAsync', + dependency_name='httpx', +) as _httpx_import: + from apify_client.http_clients._httpx import HttpxHttpClient, HttpxHttpClientAsync + +if _httpx_import.available: + __all__ = [ + 'HttpClient', + 'HttpClientAsync', + 'HttpResponse', + 'HttpxHttpClient', + 'HttpxHttpClientAsync', + 'ImpitHttpClient', + 'ImpitHttpClientAsync', + ] +else: + __all__ = [ + 'HttpClient', + 'HttpClientAsync', + 'HttpResponse', + 'ImpitHttpClient', + 'ImpitHttpClientAsync', + ] diff --git a/src/apify_client/http_clients/_httpx.py b/src/apify_client/http_clients/_httpx.py new file mode 100644 index 00000000..8e256492 --- /dev/null +++ b/src/apify_client/http_clients/_httpx.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import httpx +from typing_extensions import override + +from apify_client._consts import ( + DEFAULT_MAX_RETRIES, + DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + DEFAULT_TIMEOUT_LONG, + DEFAULT_TIMEOUT_MAX, + DEFAULT_TIMEOUT_MEDIUM, + DEFAULT_TIMEOUT_SHORT, +) +from apify_client._docs import docs_group +from apify_client.http_clients._base import HttpClient, HttpClientAsync + +if TYPE_CHECKING: + from datetime import timedelta + + from apify_client._statistics import ClientStatistics + from apify_client.http_compressors._base import HttpCompressor + + +_PERMANENT_ERRORS = ( + # A request HTTPX rejects before sending it, e.g. one carrying an invalid header value. + httpx.LocalProtocolError, + # A URL scheme HTTPX refuses to speak, which repeating the request cannot change. + httpx.UnsupportedProtocol, + # An over-long redirect chain is a routing loop, which repeating the request cannot break. + httpx.TooManyRedirects, + # Only `Response.raise_for_status()` raises this, and the client never calls it - the shared pipeline decides on + # status codes from the response itself. + httpx.HTTPStatusError, +) +"""HTTPX errors that a retry cannot fix. Everything else in the `httpx.HTTPError` tree counts as transient.""" + + +@docs_group('HTTP clients') +class HttpxHttpClient(HttpClient): + """Synchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/). + + This client wraps `httpx.Client` and adds automatic retries with exponential backoff for rate-limited + (HTTP 429) and server error (HTTP 5xx) responses. + + Requires the `httpx` extra: `pip install "apify-client[httpx]"`. + """ + + def __init__( + self, + *, + token: str | None = None, + timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT, + timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM, + timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, + timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, + max_retries: int = DEFAULT_MAX_RETRIES, + min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + statistics: ClientStatistics | None = None, + headers: dict[str, str] | None = None, + http_compressor: HttpCompressor | None = None, + ) -> None: + """Initialize the HTTPX-based synchronous HTTP client. + + Args: + token: Apify API token for authentication. + timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). + timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). + timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). + timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. + max_retries: Maximum number of retry attempts for failed requests. + min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). + statistics: Statistics tracker for API calls. Created automatically if not provided. + headers: Additional HTTP headers to include in all requests. + http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. + """ + super().__init__( + token=token, + timeout_short=timeout_short, + timeout_medium=timeout_medium, + timeout_long=timeout_long, + timeout_max=timeout_max, + max_retries=max_retries, + min_delay_between_retries=min_delay_between_retries, + statistics=statistics, + headers=headers, + http_compressor=http_compressor, + ) + + self._httpx_client = httpx.Client( + follow_redirects=True, + event_hooks={'response': [self._clear_response_cookies]}, + ) + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures + # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the + # response status code, not here. + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + + @override + def close(self) -> None: + """Close the underlying HTTPX connection pool.""" + self._httpx_client.close() + + def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_client.cookies.clear() + + @override + def send_request( + self, + *, + method: str, + url: str, + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> httpx.Response: + request = self._httpx_client.build_request( + method=method, + url=url, + headers=headers, + content=content, + timeout=timeout, + ) + _restore_explicit_cookie_header(request, headers) + return self._httpx_client.send(request, stream=stream) + + +@docs_group('HTTP clients') +class HttpxHttpClientAsync(HttpClientAsync): + """Asynchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/). + + This client wraps `httpx.AsyncClient` and adds automatic retries with exponential backoff for rate-limited + (HTTP 429) and server error (HTTP 5xx) responses. + + Requires the `httpx` extra: `pip install "apify-client[httpx]"`. + """ + + def __init__( + self, + *, + token: str | None = None, + timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT, + timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM, + timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, + timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, + max_retries: int = DEFAULT_MAX_RETRIES, + min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + statistics: ClientStatistics | None = None, + headers: dict[str, str] | None = None, + http_compressor: HttpCompressor | None = None, + ) -> None: + """Initialize the HTTPX-based asynchronous HTTP client. + + Args: + token: Apify API token for authentication. + timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). + timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). + timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). + timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. + max_retries: Maximum number of retry attempts for failed requests. + min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). + statistics: Statistics tracker for API calls. Created automatically if not provided. + headers: Additional HTTP headers to include in all requests. + http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. + """ + super().__init__( + token=token, + timeout_short=timeout_short, + timeout_medium=timeout_medium, + timeout_long=timeout_long, + timeout_max=timeout_max, + max_retries=max_retries, + min_delay_between_retries=min_delay_between_retries, + statistics=statistics, + headers=headers, + http_compressor=http_compressor, + ) + + self._httpx_async_client = httpx.AsyncClient( + follow_redirects=True, + event_hooks={'response': [self._clear_response_cookies]}, + ) + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures + # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the + # response status code, not here. + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + + @override + async def aclose(self) -> None: + """Close the underlying asynchronous HTTPX connection pool.""" + await self._httpx_async_client.aclose() + + async def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_async_client.cookies.clear() + + @override + async def send_request( + self, + *, + method: str, + url: str, + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> httpx.Response: + request = self._httpx_async_client.build_request( + method=method, + url=url, + headers=headers, + content=content, + timeout=timeout, + ) + _restore_explicit_cookie_header(request, headers) + return await self._httpx_async_client.send(request, stream=stream) + + +def _restore_explicit_cookie_header(request: httpx.Request, headers: dict[str, str]) -> None: + """Keep only cookies explicitly supplied for this request, never cookies from HTTPX's shared jar.""" + explicit_cookie = next((value for key, value in headers.items() if key.lower() == 'cookie'), None) + if explicit_cookie is None: + request.headers.pop('cookie', None) + else: + request.headers['cookie'] = explicit_cookie diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1db53b71..e0c9bed6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -2,6 +2,7 @@ import json import os +from dataclasses import dataclass from typing import TYPE_CHECKING import pytest @@ -17,9 +18,35 @@ from apify_client import ApifyClient, ApifyClientAsync from apify_client._consts import DEFAULT_API_URL from apify_client._utils.crypto import create_hmac_signature, create_storage_content_signature +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import AsyncGenerator, Generator + + +@dataclass(frozen=True) +class HttpClientClasses: + """Synchronous and asynchronous variants of a built-in HTTP client.""" + + sync: type[HttpClient] + async_: type[HttpClientAsync] + + +DEFAULT_HTTP_CLIENT_CLASSES = HttpClientClasses(sync=ImpitHttpClient, async_=ImpitHttpClientAsync) +"""HTTP clients the live-API suite runs with unless a test asks for another transport.""" + +ALL_HTTP_CLIENT_CLASSES = [ + pytest.param(DEFAULT_HTTP_CLIENT_CLASSES, id='impit'), + pytest.param(HttpClientClasses(sync=HttpxHttpClient, async_=HttpxHttpClientAsync), id='httpx'), +] +"""Every built-in HTTP client, for tests that exercise transport behavior rather than an API resource.""" # ============================================================================ @@ -110,17 +137,17 @@ def test_kvs_of_another_user(api_token_2: str) -> Generator[KvsFixture]: @pytest.fixture -def apify_client(api_token: str) -> ApifyClient: - """Sync Apify client instance.""" - api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL - return ApifyClient(api_token, api_url=api_url) +def http_client_classes(request: pytest.FixtureRequest) -> HttpClientClasses: + """Return the sync and async classes of the HTTP client the test runs with. + Defaults to Impit so the live-API suite isn't multiplied by every transport. A transport-level test opts into + the full matrix with `@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True)`. + """ + if not hasattr(request, 'param'): + return DEFAULT_HTTP_CLIENT_CLASSES -@pytest.fixture -def apify_client_async(api_token: str) -> ApifyClientAsync: - """Async Apify client instance.""" - api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL - return ApifyClientAsync(api_token, api_url=api_url) + assert isinstance(request.param, HttpClientClasses) + return request.param @pytest.fixture(params=['sync', 'async']) @@ -130,13 +157,30 @@ def client_type(request: pytest.FixtureRequest) -> str: @pytest.fixture -def client( +async def client( client_type: str, - apify_client: ApifyClient, - apify_client_async: ApifyClientAsync, -) -> ApifyClient | ApifyClientAsync: - """Return sync or async client based on parametrization.""" - return apify_client if client_type == 'sync' else apify_client_async + api_token: str, + http_client_classes: HttpClientClasses, +) -> AsyncGenerator[ApifyClient | ApifyClientAsync]: + """Return each sync/async and HTTP client implementation combination.""" + api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL + if client_type == 'sync': + http_client = http_client_classes.sync() + yield ApifyClient.with_custom_http_client( + api_token, + api_url=api_url, + http_client=http_client, + ) + http_client.close() + return + + http_client_async = http_client_classes.async_() + yield ApifyClientAsync.with_custom_http_client( + api_token, + api_url=api_url, + http_client=http_client_async, + ) + await http_client_async.aclose() @pytest.fixture diff --git a/tests/integration/test_apify_client.py b/tests/integration/test_apify_client.py index 126f40b3..4c15eab8 100644 --- a/tests/integration/test_apify_client.py +++ b/tests/integration/test_apify_client.py @@ -4,13 +4,17 @@ from typing import TYPE_CHECKING +import pytest + from .._utils import maybe_await +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import UserPrivateInfo, UserPublicInfo if TYPE_CHECKING: from apify_client import ApifyClient, ApifyClientAsync +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_apify_client(client: ApifyClient | ApifyClientAsync) -> None: """Test basic apify client functionality.""" user_client = client.user('me') diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index 333c7229..b7acab4c 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -18,6 +18,7 @@ maybe_await, poll_until_condition, ) +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import Dataset, DatasetListItem, DatasetStatistics, ListOfDatasets from apify_client._resource_clients.dataset import DatasetItemsPage from apify_client.errors import ApifyApiError @@ -698,6 +699,7 @@ async def get_items() -> DatasetItemsPage: await maybe_await(dataset_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_dataset_stream_items(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming dataset items.""" dataset_name = get_random_resource_name('dataset') diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index 5d1d9238..fee82954 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -19,6 +19,7 @@ maybe_sleep, poll_until_condition, ) +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import KeyValueStore, KeyValueStoreKey, ListOfKeys, ListOfKeyValueStores from apify_client.errors import ApifyApiError from apify_client.http_clients import HttpResponse @@ -706,6 +707,7 @@ async def get_keys() -> ListOfKeys: await maybe_await(store_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_key_value_store_stream_record_own(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming a record from one's own key-value store (no signature).""" store_name = get_random_resource_name('kvs') diff --git a/tests/integration/test_log.py b/tests/integration/test_log.py index df687e2f..13db91f8 100644 --- a/tests/integration/test_log.py +++ b/tests/integration/test_log.py @@ -5,7 +5,10 @@ from contextlib import AbstractAsyncContextManager, AbstractContextManager from typing import TYPE_CHECKING +import pytest + from .._utils import maybe_await +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import ListOfBuilds, Run from apify_client.http_clients import HttpResponse @@ -72,6 +75,7 @@ async def test_log_get_as_bytes(client: ApifyClient | ApifyClientAsync) -> None: await maybe_await(run_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_log_stream_from_run(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming a run's log via the stream() context manager.""" actor = client.actor(HELLO_WORLD_ACTOR) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 124c12cd..98d48d70 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -7,7 +7,14 @@ from pytest_httpserver import HTTPServer from apify_client import ApifyClient, ApifyClientAsync -from apify_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) if TYPE_CHECKING: from collections.abc import Iterable @@ -43,13 +50,23 @@ def async_client(httpserver: HTTPServer) -> ApifyClientAsync: return ApifyClientAsync(token='test', api_url=httpserver.url_for('/').removesuffix('/')) -@pytest.fixture(params=[pytest.param(ImpitHttpClient, id='impit')]) +@pytest.fixture( + params=[ + pytest.param(ImpitHttpClient, id='impit'), + pytest.param(HttpxHttpClient, id='httpx'), + ] +) def http_client_class(request: pytest.FixtureRequest) -> type[HttpClient]: """Return each built-in synchronous HTTP client class.""" return request.param -@pytest.fixture(params=[pytest.param(ImpitHttpClientAsync, id='impit')]) +@pytest.fixture( + params=[ + pytest.param(ImpitHttpClientAsync, id='impit'), + pytest.param(HttpxHttpClientAsync, id='httpx'), + ] +) def http_client_async_class(request: pytest.FixtureRequest) -> type[HttpClientAsync]: """Return each built-in asynchronous HTTP client class.""" return request.param diff --git a/tests/unit/test_client_headers.py b/tests/unit/test_client_headers.py index 981d6857..b8e0b259 100644 --- a/tests/unit/test_client_headers.py +++ b/tests/unit/test_client_headers.py @@ -6,8 +6,11 @@ from importlib import metadata from typing import TYPE_CHECKING +import httpx from werkzeug import Request, Response +from apify_client.http_clients import HttpxHttpClient, HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync + if TYPE_CHECKING: from pytest_httpserver import HTTPServer @@ -19,6 +22,18 @@ def _parse_accept_encoding(header: str) -> set[str]: return {enc.strip() for enc in header.split(',')} +def _transport_wire_headers( + client_class: type[HttpClient | HttpClientAsync], +) -> tuple[dict[str, str], set[str]]: + """Return the headers the transport adds on its own and the content encodings it advertises.""" + if issubclass(client_class, (ImpitHttpClient, ImpitHttpClientAsync)): + return {}, {'zstd', 'gzip', 'deflate', 'br'} + # HTTPX advertises whichever decoders happen to be installed alongside it, so read the set off the client + # itself rather than hard-coding it and breaking whenever the environment gains or loses a codec. + with httpx.Client() as probe: + return {'Connection': 'keep-alive'}, _parse_accept_encoding(probe.headers['accept-encoding']) + + def _header_handler(request: Request) -> Response: return Response( status=200, @@ -43,15 +58,17 @@ async def test_default_headers_async(httpserver: HTTPServer, http_client_async_c response = await client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_async_class) expected_headers = { 'User-Agent': _get_user_agent(), 'Accept': 'application/json, */*', 'Authorization': 'Bearer placeholder_token', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings def test_default_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient]) -> None: @@ -63,15 +80,17 @@ def test_default_headers_sync(httpserver: HTTPServer, http_client_class: type[Ht response = client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_class) expected_headers = { 'User-Agent': _get_user_agent(), 'Accept': 'application/json, */*', 'Authorization': 'Bearer placeholder_token', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings async def test_headers_async(httpserver: HTTPServer, http_client_async_class: type[HttpClientAsync]) -> None: @@ -86,6 +105,7 @@ async def test_headers_async(httpserver: HTTPServer, http_client_async_class: ty response = await client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_async_class) expected_headers = { 'Test-Header': 'blah', @@ -93,9 +113,10 @@ async def test_headers_async(httpserver: HTTPServer, http_client_async_class: ty 'Accept': 'application/json, */*', 'Authorization': 'strange_value', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient]) -> None: @@ -114,6 +135,7 @@ def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient response = client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_class) expected_headers = { 'Test-Header': 'blah', @@ -121,9 +143,10 @@ def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient 'Accept': 'application/json, */*', 'Authorization': 'strange_value', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings async def test_per_request_headers_override_defaults_async( @@ -158,3 +181,45 @@ def test_per_request_headers_override_defaults_sync( # WSGI joins duplicate headers into one comma-separated value, so exact equality # also proves the authorization header was sent only once. assert request_headers['Authorization'] == 'Bearer per-request' + + +def _echo_cookie_handler(request: Request) -> Response: + return Response(json.dumps({'cookie': request.headers.get('Cookie')}), content_type='application/json') + + +def test_httpx_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """A Set-Cookie response must not silently leak into a later API request through HTTPX's shared cookie jar.""" + httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + client.call(method='GET', url=httpserver.url_for('/set-cookie')) + response = client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +async def test_httpx_async_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """The asynchronous HTTPX pool also remains stateless between API calls.""" + httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + async with HttpxHttpClientAsync() as client: + await client.call(method='GET', url=httpserver.url_for('/set-cookie')) + response = await client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +def test_httpx_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: + """Disabling the shared cookie jar must not remove a Cookie header explicitly supplied by the caller.""" + httpserver.expect_request('/echo-explicit-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + response = client.call( + method='GET', + url=httpserver.url_for('/echo-explicit-cookie'), + headers={'Cookie': 'explicit=value'}, + ) + + assert response.json() == {'cookie': 'explicit=value'} diff --git a/tests/unit/test_client_streaming.py b/tests/unit/test_client_streaming.py index d369455e..9e5782e7 100644 --- a/tests/unit/test_client_streaming.py +++ b/tests/unit/test_client_streaming.py @@ -105,7 +105,7 @@ def test_protocol_check_leaves_stream_unread_sync( with client.dataset(DATASET_ID).stream_items(item_format='json') as response: assert isinstance(response, HttpResponse) - # `is_stream_consumed` is transport state, not part of the protocol, but the built-in client exposes it. + # `is_stream_consumed` is transport state, not part of the protocol, but both built-in clients expose it. raw: Any = response assert raw.is_stream_consumed is False @@ -124,6 +124,6 @@ async def test_protocol_check_leaves_stream_unread_async( async with client.dataset(DATASET_ID).stream_items(item_format='json') as response: assert isinstance(response, HttpResponse) - # `is_stream_consumed` is transport state, not part of the protocol, but the built-in client exposes it. + # `is_stream_consumed` is transport state, not part of the protocol, but both built-in clients expose it. raw: Any = response assert raw.is_stream_consumed is False diff --git a/tests/unit/test_client_timeouts.py b/tests/unit/test_client_timeouts.py index b4410acd..58fd03b2 100644 --- a/tests/unit/test_client_timeouts.py +++ b/tests/unit/test_client_timeouts.py @@ -5,16 +5,27 @@ from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock +import httpx import impit import pytest from apify_client._logging import LoggerOnce, logger_name -from apify_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) from apify_client.http_clients import _base as http_client_base if TYPE_CHECKING: from _pytest.logging import LogCaptureFixture +UNSET_HTTPX_TIMEOUT = {'connect': None, 'read': None, 'write': None, 'pool': None} +"""What HTTPX stores on a request built with `timeout=None`: every sub-timeout unset, not the client default.""" + @pytest.fixture def fresh_logger_once(monkeypatch: pytest.MonkeyPatch) -> None: @@ -26,6 +37,12 @@ def successful_response() -> Mock: return Mock(status_code=200) +def retryable_error(client: HttpClient | HttpClientAsync) -> Exception: + if isinstance(client, (ImpitHttpClient, ImpitHttpClientAsync)): + return impit.TimeoutException('timeout') + return httpx.ReadTimeout('timeout', request=httpx.Request('GET', 'https://example.com')) + + @pytest.mark.parametrize( ('timeout', 'expected'), [ @@ -93,7 +110,7 @@ async def test_timeout_resolves_for_async_clients( def test_compute_timeout_with_timedelta(http_client_class: type[HttpClient]) -> None: - """Concrete timedeltas double per attempt, are capped at the maximum, and `no_timeout` stays unbounded.""" + """Concrete timedeltas double per attempt and are capped at the configured maximum.""" client = http_client_class(timeout_max=timedelta(seconds=20)) assert client._compute_timeout(timedelta(seconds=5), attempt=1) == 5.0 @@ -160,7 +177,7 @@ def test_dynamic_timeout_sync_client(http_client_class: type[HttpClient], monkey def send_request(*_args: Any, **kwargs: Any) -> Mock: timeouts.append(kwargs['timeout']) if len(timeouts) < 4: - raise impit.TimeoutException('timeout') + raise retryable_error(client) return successful_response() monkeypatch.setattr(client, 'send_request', send_request) @@ -185,7 +202,7 @@ async def test_dynamic_timeout_async_client( async def send_request(*_args: Any, **kwargs: Any) -> Mock: timeouts.append(kwargs['timeout']) if len(timeouts) < 4: - raise impit.TimeoutException('timeout') + raise retryable_error(client) return successful_response() monkeypatch.setattr(client, 'send_request', send_request) @@ -196,23 +213,39 @@ async def send_request(*_args: Any, **kwargs: Any) -> Mock: assert response.status_code == 200 -def test_no_timeout_mapping_for_sync_adapter() -> None: - """The synchronous adapter maps no-timeout to Impit's effectively unbounded value.""" - client = ImpitHttpClient() - client._impit_client = Mock(request=Mock(return_value=successful_response())) - - client.send_request(method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False) - - assert client._impit_client.request.call_args.kwargs['timeout'] == 86_400 - - -async def test_no_timeout_mapping_for_async_adapter() -> None: - """The asynchronous adapter maps no-timeout to Impit's effectively unbounded value.""" - client = ImpitHttpClientAsync() - client._impit_async_client = Mock(request=AsyncMock(return_value=successful_response())) - - await client.send_request( +def test_no_timeout_mapping_for_sync_adapters(monkeypatch: pytest.MonkeyPatch) -> None: + """Each synchronous adapter maps no-timeout to its underlying library semantics.""" + impit_client = ImpitHttpClient() + impit_client._impit_client = Mock(request=Mock(return_value=successful_response())) + impit_client.send_request( method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False ) - - assert client._impit_async_client.request.call_args.kwargs['timeout'] == 86_400 + assert impit_client._impit_client.request.call_args.kwargs['timeout'] == 86_400 + + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. + with HttpxHttpClient() as httpx_client: + send = Mock(return_value=successful_response()) + monkeypatch.setattr(httpx_client._httpx_client, 'send', send) + httpx_client.send_request( + method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False + ) + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT + + +async def test_no_timeout_mapping_for_async_adapters(monkeypatch: pytest.MonkeyPatch) -> None: + """Each asynchronous adapter maps no-timeout to its underlying library semantics.""" + impit_client = ImpitHttpClientAsync() + impit_client._impit_async_client = Mock(request=AsyncMock(return_value=successful_response())) + await impit_client.send_request( + method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False + ) + assert impit_client._impit_async_client.request.call_args.kwargs['timeout'] == 86_400 + + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. + async with HttpxHttpClientAsync() as httpx_client: + send = AsyncMock(return_value=successful_response()) + monkeypatch.setattr(httpx_client._httpx_async_client, 'send', send) + await httpx_client.send_request( + method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False + ) + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index adbf9ce7..c7f426df 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, Mock import brotli +import httpx import impit import pytest @@ -21,6 +22,8 @@ HttpClient, HttpClientAsync, HttpResponse, + HttpxHttpClient, + HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -264,6 +267,24 @@ async def test_http_client_async_creates_async_impit_client() -> None: await client.aclose() +def test_http_client_creates_sync_httpx_client() -> None: + """The synchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" + client = HttpxHttpClient(token='test_token_123') + + assert isinstance(client._httpx_client, httpx.Client) + client.close() + assert client._httpx_client.is_closed + + +async def test_http_client_async_creates_async_httpx_client() -> None: + """The asynchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" + client = HttpxHttpClientAsync(token='test_token_123') + + assert isinstance(client._httpx_async_client, httpx.AsyncClient) + await client.aclose() + assert client._httpx_async_client.is_closed + + def test_parse_params_none() -> None: """Test _parse_params with None input.""" assert HttpClient._parse_params(None) is None @@ -392,6 +413,70 @@ async def test_async_http_client_classifies_timeout_errors() -> None: assert not client.is_timeout_error(ValueError('test')) +@pytest.mark.parametrize( + 'exc', + [ + # Even the generic base class is transient: HTTPX subclasses it for every failure mode, so an + # unclassified failure is safer to retry. + pytest.param(httpx.HTTPError('unclassified failure'), id='bare HTTPError'), + pytest.param(httpx.TimeoutException('timeout'), id='TimeoutException'), + pytest.param(httpx.NetworkError('network error'), id='NetworkError'), + pytest.param(httpx.RemoteProtocolError('remote protocol error'), id='RemoteProtocolError'), + pytest.param(httpx.DecodingError('decoding error'), id='DecodingError'), + # One `ProxyError` covers both a proxy rejecting the CONNECT tunnel and a 407, so a transient case cannot + # be told from a permanent one - retrying is the safer default. + pytest.param(httpx.ProxyError('proxy error'), id='ProxyError'), + ], +) +def test_httpx_is_retryable_transport_error(exc: Exception) -> None: + """A transient HTTPX transport failure is classified as retryable.""" + with HttpxHttpClient() as client: + assert client.is_retryable_transport_error(exc) + + +@pytest.mark.parametrize( + 'exc', + [ + pytest.param(httpx.LocalProtocolError('invalid header value'), id='LocalProtocolError'), + pytest.param(httpx.UnsupportedProtocol('unsupported scheme'), id='UnsupportedProtocol'), + pytest.param(httpx.TooManyRedirects('too many redirects'), id='TooManyRedirects'), + pytest.param( + httpx.HTTPStatusError( + 'status error', + request=httpx.Request('GET', 'https://example.com'), + response=httpx.Response(500), + ), + id='HTTPStatusError', + ), + # HTTPX reports a bad URL outside the `httpx.HTTPError` tree entirely. + pytest.param(httpx.InvalidURL('unsupported scheme'), id='InvalidURL'), + pytest.param(ValueError('value error'), id='ValueError'), + pytest.param(RuntimeError('runtime error'), id='RuntimeError'), + pytest.param(Exception('generic exception'), id='Exception'), + ], +) +def test_httpx_is_not_retryable_transport_error(exc: Exception) -> None: + """A transport failure a retry cannot fix, and anything outside HTTPX's hierarchy, is not retried.""" + with HttpxHttpClient() as client: + assert not client.is_retryable_transport_error(exc) + + +def test_sync_httpx_client_classifies_timeout_errors() -> None: + """The built-in synchronous HTTPX client exposes transport-neutral timeout classification.""" + with HttpxHttpClient() as client: + assert client.is_timeout_error(TimeoutError('test')) + assert client.is_timeout_error(httpx.TimeoutException('test')) + assert not client.is_timeout_error(ValueError('test')) + + +async def test_async_httpx_client_classifies_timeout_errors() -> None: + """The built-in asynchronous HTTPX client exposes transport-neutral timeout classification.""" + async with HttpxHttpClientAsync() as client: + assert client.is_timeout_error(TimeoutError('test')) + assert client.is_timeout_error(httpx.TimeoutException('test')) + assert not client.is_timeout_error(ValueError('test')) + + def test_permanent_transport_error_is_not_retried() -> None: """A transport error a retry cannot fix fails on the first attempt instead of burning the whole backoff.""" client = ImpitHttpClient(token='test_token', min_delay_between_retries=timedelta(0)) @@ -429,6 +514,31 @@ def test_transient_transport_error_is_retried() -> None: assert request.call_count == 3 +def test_httpx_permanent_transport_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX adapter feeds the same fail-fast classification into the shared pipeline.""" + with HttpxHttpClient(token='test_token', min_delay_between_retries=timedelta(0)) as client: + send_request = Mock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) + monkeypatch.setattr(client, 'send_request', send_request) + + with pytest.raises(httpx.UnsupportedProtocol): + client.call(method='GET', url='https://api.test.com/endpoint') + + send_request.assert_called_once() + + +def test_httpx_transient_transport_error_is_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX adapter keeps a transient transport failure inside the shared retry loop.""" + with HttpxHttpClient(token='test_token', max_retries=2, min_delay_between_retries=timedelta(0)) as client: + send_request = Mock(side_effect=httpx.TimeoutException('timeout')) + monkeypatch.setattr(client, 'send_request', send_request) + + with pytest.raises(httpx.TimeoutException): + client.call(method='GET', url='https://api.test.com/endpoint') + + # `max_retries` attempts inside the backoff loop, plus the final one it makes after the last delay. + assert send_request.call_count == 3 + + def test_error_response_read_failure_is_retried_and_closed() -> None: """A failure while buffering a streamed error body is retried like a failed send, and the response is closed.""" client = ImpitHttpClient(token='test_token', max_retries=1, min_delay_between_retries=timedelta(0)) diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index 35b10e2e..4c2f2271 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -877,7 +877,7 @@ def test_streamed_log_sync_does_not_leak_exception_on_stream_timeout( http_client_class: type[HttpClient], monkeypatch: pytest.MonkeyPatch, ) -> None: - """The streaming thread ends quietly when the transport times out while reading the log stream.""" + """The streaming thread ends quietly when either transport times out while reading the log stream.""" monkeypatch.setattr(StreamedLog, '_stream_timeout', timedelta(seconds=1)) release_server = threading.Event() @@ -970,7 +970,7 @@ async def test_streamed_log_async_does_not_error_on_stream_timeout( http_client_async_class: type[HttpClientAsync], monkeypatch: pytest.MonkeyPatch, ) -> None: - """The async streaming task treats a transport stream timeout as an expected terminal condition.""" + """The async streaming task treats either transport's stream timeout as an expected terminal condition.""" monkeypatch.setattr(StreamedLogAsync, '_stream_timeout', timedelta(seconds=1)) release_server = threading.Event() diff --git a/tests/unit/test_pluggable_http_client.py b/tests/unit/test_pluggable_http_client.py index 2fd20899..779125b8 100644 --- a/tests/unit/test_pluggable_http_client.py +++ b/tests/unit/test_pluggable_http_client.py @@ -2,9 +2,12 @@ import asyncio import json as jsonlib +import subprocess +import sys from dataclasses import dataclass, field from datetime import timedelta from http.client import HTTPConnection +from textwrap import dedent from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock from urllib.parse import urlsplit @@ -353,6 +356,8 @@ def test_public_exports() -> None: 'HttpClient', 'HttpClientAsync', 'HttpResponse', + 'HttpxHttpClient', + 'HttpxHttpClientAsync', 'ImpitHttpClient', 'ImpitHttpClientAsync', ): @@ -362,6 +367,47 @@ def test_public_exports() -> None: assert not hasattr(http_clients_module, 'HttpClientBase') +def test_httpx_clients_raise_clear_error_when_extra_missing() -> None: + """Missing HTTPX keeps normal and star imports usable while explicit HTTPX access raises a clear error.""" + script = dedent( + """ + import sys + + class BlockHttpx: + def find_spec(self, name, *_args): + if name == 'httpx' or name.startswith('httpx.'): + raise ModuleNotFoundError(f"No module named '{name}'", name='httpx') + return None + + sys.meta_path.insert(0, BlockHttpx()) + + import apify_client.http_clients as module + assert module.HttpClient is not None + assert module.ImpitHttpClient is not None + + namespace = {} + exec('from apify_client.http_clients import *', namespace) + assert namespace['HttpClient'] is module.HttpClient + assert 'HttpxHttpClient' not in namespace + + for name in ('HttpxHttpClient', 'HttpxHttpClientAsync'): + try: + getattr(module, name) + except ImportError as exc: + assert "No module named 'httpx'" in str(exc) + else: + raise AssertionError(f'{name} did not raise ImportError') + """ + ) + result = subprocess.run( # noqa: S603 + [sys.executable, '-c', script], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + def test_apify_client_http_client_property_returns_correct_type() -> None: """Test that http_client property returns the correct type.""" # With default diff --git a/uv.lock b/uv.lock index 0f531050..0e2383f7 100644 --- a/uv.lock +++ b/uv.lock @@ -54,6 +54,9 @@ dependencies = [ brotli = [ { name = "brotli" }, ] +httpx = [ + { name = "httpx" }, +] [package.dev-dependencies] dev = [ @@ -80,12 +83,13 @@ dev = [ requires-dist = [ { name = "brotli", marker = "extra == 'brotli'", specifier = ">=1.0.9" }, { name = "colorama", specifier = ">=0.4.0" }, + { name = "httpx", marker = "extra == 'httpx'", specifier = ">=0.27.0,<1.0.0" }, { name = "impit", specifier = "~=0.13.0" }, { name = "more-itertools", specifier = ">=10.0.0" }, { name = "pydantic", extras = ["email"], specifier = ">=2.11.0" }, { name = "typing-extensions", specifier = ">=4.6.0" }, ] -provides-extras = ["brotli"] +provides-extras = ["brotli", "httpx"] [package.metadata.requires-dev] dev = [ From 5535734054ac19b3b771b021db8a4ef557cf7d4b Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 14:43:28 +0200 Subject: [PATCH 2/4] test: Cover HTTPX cookie isolation and async transport parity --- .rules.md | 1 + README.md | 3 +- src/apify_client/http_clients/_httpx.py | 38 +++++++++----- tests/unit/test_client_headers.py | 70 ++++++++++++++++++++++++- tests/unit/test_client_timeouts.py | 56 ++++++++++++-------- tests/unit/test_http_clients.py | 39 +++++++++++--- tests/unit/test_logging.py | 8 +-- 7 files changed, 168 insertions(+), 47 deletions(-) diff --git a/.rules.md b/.rules.md index cced5614..0efd660a 100644 --- a/.rules.md +++ b/.rules.md @@ -63,6 +63,7 @@ Docstrings are written on sync clients and **automatically copied** to async cli - `HttpClient`/`HttpClientAsync` — base classes in `http_clients/_base.py` holding the shared request pipeline (retries, timeouts, API errors); transports implement the `send_request`, error-classification, and lifecycle hooks - `ImpitHttpClient`/`ImpitHttpClientAsync` — default implementation (Rust-based Impit) +- `HttpxHttpClient`/`HttpxHttpClientAsync` — built-in alternative behind the `httpx` optional extra - `HttpResponse` — Protocol (not a concrete class) for response objects - Users can plug in custom HTTP clients via `ApifyClient.with_custom_http_client()` diff --git a/README.md b/README.md index 7627e5fe..d4a6f2ad 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,8 @@ ``` [Impit](https://github.com/apify/impit) is the default HTTP client and is installed automatically. To use the - built-in [HTTPX](https://www.python-httpx.org/) client instead, install its optional extra: + built-in [HTTPX](https://www.python-httpx.org/) client instead, install its optional extra and pass + `http_client=HttpxHttpClient()` to `ApifyClient.with_custom_http_client()`: ```bash pip install "apify-client[httpx]" diff --git a/src/apify_client/http_clients/_httpx.py b/src/apify_client/http_clients/_httpx.py index 8e256492..ccf931dd 100644 --- a/src/apify_client/http_clients/_httpx.py +++ b/src/apify_client/http_clients/_httpx.py @@ -44,6 +44,10 @@ class HttpxHttpClient(HttpClient): This client wraps `httpx.Client` and adds automatic retries with exponential backoff for rate-limited (HTTP 429) and server error (HTTP 5xx) responses. + HTTPX applies a request timeout to each socket operation rather than to the request as a whole, so a response + whose body arrives slowly keeps resetting it and can outlast both the requested timeout and `timeout_max`. The + default Impit client enforces the same value as a deadline for the whole request, body included. + Requires the `httpx` extra: `pip install "apify-client[httpx]"`. """ @@ -100,8 +104,8 @@ def is_timeout_error(self, exc: Exception) -> bool: @override def is_retryable_transport_error(self, exc: Exception) -> bool: # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in - # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures - # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the + # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX adds later is retried rather than + # silently treated as fatal. HTTP status code errors are handled by the shared pipeline based on the # response status code, not here. return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) @@ -110,10 +114,6 @@ def close(self) -> None: """Close the underlying HTTPX connection pool.""" self._httpx_client.close() - def _clear_response_cookies(self, _response: httpx.Response) -> None: - """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" - self._httpx_client.cookies.clear() - @override def send_request( self, @@ -135,6 +135,10 @@ def send_request( _restore_explicit_cookie_header(request, headers) return self._httpx_client.send(request, stream=stream) + def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_client.cookies.clear() + @docs_group('HTTP clients') class HttpxHttpClientAsync(HttpClientAsync): @@ -143,6 +147,10 @@ class HttpxHttpClientAsync(HttpClientAsync): This client wraps `httpx.AsyncClient` and adds automatic retries with exponential backoff for rate-limited (HTTP 429) and server error (HTTP 5xx) responses. + HTTPX applies a request timeout to each socket operation rather than to the request as a whole, so a response + whose body arrives slowly keeps resetting it and can outlast both the requested timeout and `timeout_max`. The + default Impit client enforces the same value as a deadline for the whole request, body included. + Requires the `httpx` extra: `pip install "apify-client[httpx]"`. """ @@ -199,8 +207,8 @@ def is_timeout_error(self, exc: Exception) -> bool: @override def is_retryable_transport_error(self, exc: Exception) -> bool: # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in - # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures - # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the + # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX adds later is retried rather than + # silently treated as fatal. HTTP status code errors are handled by the shared pipeline based on the # response status code, not here. return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) @@ -209,10 +217,6 @@ async def aclose(self) -> None: """Close the underlying asynchronous HTTPX connection pool.""" await self._httpx_async_client.aclose() - async def _clear_response_cookies(self, _response: httpx.Response) -> None: - """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" - self._httpx_async_client.cookies.clear() - @override async def send_request( self, @@ -234,9 +238,17 @@ async def send_request( _restore_explicit_cookie_header(request, headers) return await self._httpx_async_client.send(request, stream=stream) + async def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_async_client.cookies.clear() + def _restore_explicit_cookie_header(request: httpx.Request, headers: dict[str, str]) -> None: - """Keep only cookies explicitly supplied for this request, never cookies from HTTPX's shared jar.""" + """Keep only cookies explicitly supplied for this request, never cookies from HTTPX's shared jar. + + HTTPX drops the `Cookie` header when it builds a redirect request and rebuilds it from the jar, so an explicit + cookie only reaches the first hop of a redirected request. + """ explicit_cookie = next((value for key, value in headers.items() if key.lower() == 'cookie'), None) if explicit_cookie is None: request.headers.pop('cookie', None) diff --git a/tests/unit/test_client_headers.py b/tests/unit/test_client_headers.py index b8e0b259..846df89e 100644 --- a/tests/unit/test_client_headers.py +++ b/tests/unit/test_client_headers.py @@ -188,12 +188,13 @@ def _echo_cookie_handler(request: Request) -> Response: def test_httpx_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: - """A Set-Cookie response must not silently leak into a later API request through HTTPX's shared cookie jar.""" + """A Set-Cookie response must not enter HTTPX's shared cookie jar, nor leak into a later API request.""" httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) with HttpxHttpClient() as client: client.call(method='GET', url=httpserver.url_for('/set-cookie')) + assert len(client._httpx_client.cookies) == 0 response = client.call(method='GET', url=httpserver.url_for('/echo-cookie')) assert response.json() == {'cookie': None} @@ -206,11 +207,64 @@ async def test_httpx_async_does_not_reuse_server_cookies(httpserver: HTTPServer) async with HttpxHttpClientAsync() as client: await client.call(method='GET', url=httpserver.url_for('/set-cookie')) + assert len(client._httpx_async_client.cookies) == 0 response = await client.call(method='GET', url=httpserver.url_for('/echo-cookie')) assert response.json() == {'cookie': None} +def test_httpx_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: + """A cookie another in-flight request left in the shared jar must not ride along on the next request.""" + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + client._httpx_client.cookies.set('session', 'secret', domain=httpserver.host) + response = client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +async def test_httpx_async_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: + """The asynchronous pool, where concurrent requests really do share one jar, drops leftover cookies too.""" + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + async with HttpxHttpClientAsync() as client: + client._httpx_async_client.cookies.set('session', 'secret', domain=httpserver.host) + response = await client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +def test_httpx_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: + """A cookie set by a redirecting response must not ride along on the next hop, which HTTPX builds from its jar.""" + httpserver.expect_request('/redirect').respond_with_data( + '', + status=302, + headers={'Set-Cookie': 'session=secret', 'Location': '/echo-cookie'}, + ) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + response = client.call(method='GET', url=httpserver.url_for('/redirect')) + + assert response.json() == {'cookie': None} + + +async def test_httpx_async_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: + """The asynchronous pool keeps a redirecting response's cookie off the next hop too.""" + httpserver.expect_request('/redirect').respond_with_data( + '', + status=302, + headers={'Set-Cookie': 'session=secret', 'Location': '/echo-cookie'}, + ) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + async with HttpxHttpClientAsync() as client: + response = await client.call(method='GET', url=httpserver.url_for('/redirect')) + + assert response.json() == {'cookie': None} + + def test_httpx_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: """Disabling the shared cookie jar must not remove a Cookie header explicitly supplied by the caller.""" httpserver.expect_request('/echo-explicit-cookie').respond_with_handler(_echo_cookie_handler) @@ -223,3 +277,17 @@ def test_httpx_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: ) assert response.json() == {'cookie': 'explicit=value'} + + +async def test_httpx_async_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: + """The asynchronous pool forwards an explicitly supplied Cookie header as well.""" + httpserver.expect_request('/echo-explicit-cookie').respond_with_handler(_echo_cookie_handler) + + async with HttpxHttpClientAsync() as client: + response = await client.call( + method='GET', + url=httpserver.url_for('/echo-explicit-cookie'), + headers={'Cookie': 'explicit=value'}, + ) + + assert response.json() == {'cookie': 'explicit=value'} diff --git a/tests/unit/test_client_timeouts.py b/tests/unit/test_client_timeouts.py index 58fd03b2..54f747c1 100644 --- a/tests/unit/test_client_timeouts.py +++ b/tests/unit/test_client_timeouts.py @@ -110,7 +110,7 @@ async def test_timeout_resolves_for_async_clients( def test_compute_timeout_with_timedelta(http_client_class: type[HttpClient]) -> None: - """Concrete timedeltas double per attempt and are capped at the configured maximum.""" + """Concrete timedeltas double per attempt, are capped at the maximum, and `no_timeout` stays unbounded.""" client = http_client_class(timeout_max=timedelta(seconds=20)) assert client._compute_timeout(timedelta(seconds=5), attempt=1) == 5.0 @@ -213,39 +213,51 @@ async def send_request(*_args: Any, **kwargs: Any) -> Mock: assert response.status_code == 200 -def test_no_timeout_mapping_for_sync_adapters(monkeypatch: pytest.MonkeyPatch) -> None: - """Each synchronous adapter maps no-timeout to its underlying library semantics.""" - impit_client = ImpitHttpClient() - impit_client._impit_client = Mock(request=Mock(return_value=successful_response())) - impit_client.send_request( +def test_no_timeout_mapping_for_sync_impit_adapter() -> None: + """The synchronous Impit adapter maps no-timeout to Impit's effectively unbounded value.""" + client = ImpitHttpClient() + client._impit_client = Mock(request=Mock(return_value=successful_response())) + + client.send_request(method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False) + + assert client._impit_client.request.call_args.kwargs['timeout'] == 86_400 + + +async def test_no_timeout_mapping_for_async_impit_adapter() -> None: + """The asynchronous Impit adapter maps no-timeout to Impit's effectively unbounded value.""" + client = ImpitHttpClientAsync() + client._impit_async_client = Mock(request=AsyncMock(return_value=successful_response())) + + await client.send_request( method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False ) - assert impit_client._impit_client.request.call_args.kwargs['timeout'] == 86_400 + assert client._impit_async_client.request.call_args.kwargs['timeout'] == 86_400 + + +def test_no_timeout_mapping_for_sync_httpx_adapter(monkeypatch: pytest.MonkeyPatch) -> None: + """The synchronous HTTPX adapter maps no-timeout to every HTTPX sub-timeout being unset.""" # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. - with HttpxHttpClient() as httpx_client: + with HttpxHttpClient() as client: send = Mock(return_value=successful_response()) - monkeypatch.setattr(httpx_client._httpx_client, 'send', send) - httpx_client.send_request( + monkeypatch.setattr(client._httpx_client, 'send', send) + + client.send_request( method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False ) + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT -async def test_no_timeout_mapping_for_async_adapters(monkeypatch: pytest.MonkeyPatch) -> None: - """Each asynchronous adapter maps no-timeout to its underlying library semantics.""" - impit_client = ImpitHttpClientAsync() - impit_client._impit_async_client = Mock(request=AsyncMock(return_value=successful_response())) - await impit_client.send_request( - method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False - ) - assert impit_client._impit_async_client.request.call_args.kwargs['timeout'] == 86_400 - +async def test_no_timeout_mapping_for_async_httpx_adapter(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX adapter maps no-timeout to every HTTPX sub-timeout being unset.""" # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. - async with HttpxHttpClientAsync() as httpx_client: + async with HttpxHttpClientAsync() as client: send = AsyncMock(return_value=successful_response()) - monkeypatch.setattr(httpx_client._httpx_async_client, 'send', send) - await httpx_client.send_request( + monkeypatch.setattr(client._httpx_async_client, 'send', send) + + await client.send_request( method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False ) + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index c7f426df..b2b0e369 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -517,26 +517,53 @@ def test_transient_transport_error_is_retried() -> None: def test_httpx_permanent_transport_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None: """The HTTPX adapter feeds the same fail-fast classification into the shared pipeline.""" with HttpxHttpClient(token='test_token', min_delay_between_retries=timedelta(0)) as client: - send_request = Mock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) - monkeypatch.setattr(client, 'send_request', send_request) + send = Mock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) + monkeypatch.setattr(client._httpx_client, 'send', send) with pytest.raises(httpx.UnsupportedProtocol): client.call(method='GET', url='https://api.test.com/endpoint') - send_request.assert_called_once() + send.assert_called_once() + + +async def test_httpx_permanent_transport_error_is_not_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX adapter applies the same policy, failing on the first attempt.""" + async with HttpxHttpClientAsync(token='test_token', min_delay_between_retries=timedelta(0)) as client: + send = AsyncMock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) + monkeypatch.setattr(client._httpx_async_client, 'send', send) + + with pytest.raises(httpx.UnsupportedProtocol): + await client.call(method='GET', url='https://api.test.com/endpoint') + + send.assert_awaited_once() def test_httpx_transient_transport_error_is_retried(monkeypatch: pytest.MonkeyPatch) -> None: """The HTTPX adapter keeps a transient transport failure inside the shared retry loop.""" with HttpxHttpClient(token='test_token', max_retries=2, min_delay_between_retries=timedelta(0)) as client: - send_request = Mock(side_effect=httpx.TimeoutException('timeout')) - monkeypatch.setattr(client, 'send_request', send_request) + send = Mock(side_effect=httpx.TimeoutException('timeout')) + monkeypatch.setattr(client._httpx_client, 'send', send) with pytest.raises(httpx.TimeoutException): client.call(method='GET', url='https://api.test.com/endpoint') # `max_retries` attempts inside the backoff loop, plus the final one it makes after the last delay. - assert send_request.call_count == 3 + assert send.call_count == 3 + + +async def test_httpx_transient_transport_error_is_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX adapter keeps a transient transport failure inside the shared retry loop too.""" + async with HttpxHttpClientAsync( + token='test_token', max_retries=2, min_delay_between_retries=timedelta(0) + ) as client: + send = AsyncMock(side_effect=httpx.TimeoutException('timeout')) + monkeypatch.setattr(client._httpx_async_client, 'send', send) + + with pytest.raises(httpx.TimeoutException): + await client.call(method='GET', url='https://api.test.com/endpoint') + + # `max_retries` attempts inside the backoff loop, plus the final one it makes after the last delay. + assert send.await_count == 3 def test_error_response_read_failure_is_retried_and_closed() -> None: diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index 4c2f2271..341fb664 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -885,7 +885,7 @@ def test_streamed_log_sync_does_not_leak_exception_on_stream_timeout( def _slow_handler(_request: Request) -> Response: def generate_logs() -> Iterator[bytes]: # Emit one complete line, then keep the connection open (as a running Actor would) past the - # client-side total timeout without sending anything more. + # client-side stream timeout without sending anything more. yield b'2025-05-13T07:24:12.588Z ACTOR: still running\n' release_server.wait(timeout=30) @@ -910,7 +910,7 @@ def generate_logs() -> Iterator[bytes]: try: with caplog.at_level(logging.DEBUG, logger=logger_name): thread = streamed_log.start() - # Wait past the 1s total timeout so the streaming request fails inside the thread. + # Wait past the 1s stream timeout so the streaming request fails inside the thread. thread.join(timeout=5) assert not thread.is_alive(), 'streaming thread did not end after the stream timed out' finally: @@ -977,7 +977,7 @@ async def test_streamed_log_async_does_not_error_on_stream_timeout( def _slow_handler(_request: Request) -> Response: def generate_logs() -> Iterator[bytes]: - # Emit one complete line, then keep the connection open past the client-side total timeout. + # Emit one complete line, then keep the connection open past the client-side stream timeout. yield b'2025-05-13T07:24:12.588Z ACTOR: still running\n' release_server.wait(timeout=30) @@ -999,7 +999,7 @@ def generate_logs() -> Iterator[bytes]: try: with caplog.at_level(logging.DEBUG, logger=logger_name): task = streamed_log.start() - # The 1s total timeout fails the request inside the task; it must end on its own without our help. + # The 1s stream timeout fails the request inside the task; it must end on its own without our help. done, _pending = await asyncio.wait({task}, timeout=5) assert task in done, 'async streaming task did not end after the stream timed out' finally: From 4062ae8528cba669e41c5f99dda5147fa3ead321 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 18:32:40 +0200 Subject: [PATCH 3/4] feat: Use HTTPX2 instead of HTTPX for the built-in HTTP client --- .rules.md | 2 +- README.md | 10 +- pyproject.toml | 2 +- src/apify_client/http_clients/__init__.py | 18 +-- .../http_clients/{_httpx.py => _httpx2.py} | 98 ++++++------- tests/integration/conftest.py | 6 +- tests/unit/conftest.py | 8 +- tests/unit/test_client_headers.py | 54 +++---- tests/unit/test_client_timeouts.py | 36 ++--- tests/unit/test_http_clients.py | 132 +++++++++--------- tests/unit/test_pluggable_http_client.py | 22 +-- uv.lock | 59 +++++++- 12 files changed, 248 insertions(+), 199 deletions(-) rename src/apify_client/http_clients/{_httpx.py => _httpx2.py} (71%) diff --git a/.rules.md b/.rules.md index 0efd660a..a334de8a 100644 --- a/.rules.md +++ b/.rules.md @@ -63,7 +63,7 @@ Docstrings are written on sync clients and **automatically copied** to async cli - `HttpClient`/`HttpClientAsync` — base classes in `http_clients/_base.py` holding the shared request pipeline (retries, timeouts, API errors); transports implement the `send_request`, error-classification, and lifecycle hooks - `ImpitHttpClient`/`ImpitHttpClientAsync` — default implementation (Rust-based Impit) -- `HttpxHttpClient`/`HttpxHttpClientAsync` — built-in alternative behind the `httpx` optional extra +- `Httpx2HttpClient`/`Httpx2HttpClientAsync` — built-in alternative behind the `httpx2` optional extra - `HttpResponse` — Protocol (not a concrete class) for response objects - Users can plug in custom HTTP clients via `ApifyClient.with_custom_http_client()` diff --git a/README.md b/README.md index d4a6f2ad..f2cdfbaf 100644 --- a/README.md +++ b/README.md @@ -58,13 +58,13 @@ ``` [Impit](https://github.com/apify/impit) is the default HTTP client and is installed automatically. To use the - built-in [HTTPX](https://www.python-httpx.org/) client instead, install its optional extra and pass - `http_client=HttpxHttpClient()` to `ApifyClient.with_custom_http_client()`: + built-in [HTTPX2](https://github.com/pydantic/httpx2) client instead, install its optional extra and pass + `http_client=Httpx2HttpClient()` to `ApifyClient.with_custom_http_client()`: ```bash - pip install "apify-client[httpx]" + pip install "apify-client[httpx2]" # or - uv add "apify-client[httpx]" + uv add "apify-client[httpx2]" ``` - From [conda-forge](https://anaconda.org/conda-forge/apify-client), it can be installed with [conda](https://docs.conda.io/en/latest/): @@ -134,7 +134,7 @@ For a guided walkthrough — authenticating, running an Actor, and reading its r - **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call ([Timeouts](https://docs.apify.com/api/client/python/docs/concepts/timeouts)). - **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)). - **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks ([Convenience methods](https://docs.apify.com/api/client/python/docs/concepts/convenience-methods)). -- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX](https://www.python-httpx.org/) client, or plug in any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). +- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX2](https://github.com/pydantic/httpx2) client, or plug in any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). - **Structured errors** — every API error surfaces as an [`ApifyApiError`](https://docs.apify.com/api/client/python/reference/class/ApifyApiError) with HTTP-specific subclasses for precise handling ([Error handling](https://docs.apify.com/api/client/python/docs/concepts/error-handling)). - **Debug logging** — opt-in structured logging on the `apify_client` logger captures request URLs, status codes, retry attempts, and more ([Logging](https://docs.apify.com/api/client/python/docs/concepts/logging)). diff --git a/pyproject.toml b/pyproject.toml index b8922597..f0182ebb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ [project.optional-dependencies] brotli = ["brotli>=1.0.9"] -httpx = ["httpx>=0.27.0,<1.0.0"] +httpx2 = ["httpx2>=2.0.0,<3.0.0"] [project.urls] "Apify Homepage" = "https://apify.com" diff --git a/src/apify_client/http_clients/__init__.py b/src/apify_client/http_clients/__init__.py index d1e06c90..73b56b24 100644 --- a/src/apify_client/http_clients/__init__.py +++ b/src/apify_client/http_clients/__init__.py @@ -5,23 +5,23 @@ _install_import_hook(__name__) -# `httpx` is an optional extra, so it's wrapped in try_import. Accessing the HTTPX clients +# `httpx2` is an optional extra, so it's wrapped in try_import. Accessing the HTTPX2 clients # without the extra installed raises a clear ImportError instead of failing at package import time. with _try_import( __name__, - 'HttpxHttpClient', - 'HttpxHttpClientAsync', - dependency_name='httpx', -) as _httpx_import: - from apify_client.http_clients._httpx import HttpxHttpClient, HttpxHttpClientAsync + 'Httpx2HttpClient', + 'Httpx2HttpClientAsync', + dependency_name='httpx2', +) as _httpx2_import: + from apify_client.http_clients._httpx2 import Httpx2HttpClient, Httpx2HttpClientAsync -if _httpx_import.available: +if _httpx2_import.available: __all__ = [ 'HttpClient', 'HttpClientAsync', 'HttpResponse', - 'HttpxHttpClient', - 'HttpxHttpClientAsync', + 'Httpx2HttpClient', + 'Httpx2HttpClientAsync', 'ImpitHttpClient', 'ImpitHttpClientAsync', ] diff --git a/src/apify_client/http_clients/_httpx.py b/src/apify_client/http_clients/_httpx2.py similarity index 71% rename from src/apify_client/http_clients/_httpx.py rename to src/apify_client/http_clients/_httpx2.py index ccf931dd..6ae27229 100644 --- a/src/apify_client/http_clients/_httpx.py +++ b/src/apify_client/http_clients/_httpx2.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -import httpx +import httpx2 from typing_extensions import override from apify_client._consts import ( @@ -24,31 +24,31 @@ _PERMANENT_ERRORS = ( - # A request HTTPX rejects before sending it, e.g. one carrying an invalid header value. - httpx.LocalProtocolError, - # A URL scheme HTTPX refuses to speak, which repeating the request cannot change. - httpx.UnsupportedProtocol, + # A request HTTPX2 rejects before sending it, e.g. one carrying an invalid header value. + httpx2.LocalProtocolError, + # A URL scheme HTTPX2 refuses to speak, which repeating the request cannot change. + httpx2.UnsupportedProtocol, # An over-long redirect chain is a routing loop, which repeating the request cannot break. - httpx.TooManyRedirects, + httpx2.TooManyRedirects, # Only `Response.raise_for_status()` raises this, and the client never calls it - the shared pipeline decides on # status codes from the response itself. - httpx.HTTPStatusError, + httpx2.HTTPStatusError, ) -"""HTTPX errors that a retry cannot fix. Everything else in the `httpx.HTTPError` tree counts as transient.""" +"""HTTPX2 errors that a retry cannot fix. Everything else in the `httpx2.HTTPError` tree counts as transient.""" @docs_group('HTTP clients') -class HttpxHttpClient(HttpClient): - """Synchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/). +class Httpx2HttpClient(HttpClient): + """Synchronous HTTP client for the Apify API built on top of [HTTPX2](https://github.com/pydantic/httpx2). - This client wraps `httpx.Client` and adds automatic retries with exponential backoff for rate-limited + This client wraps `httpx2.Client` and adds automatic retries with exponential backoff for rate-limited (HTTP 429) and server error (HTTP 5xx) responses. - HTTPX applies a request timeout to each socket operation rather than to the request as a whole, so a response + HTTPX2 applies a request timeout to each socket operation rather than to the request as a whole, so a response whose body arrives slowly keeps resetting it and can outlast both the requested timeout and `timeout_max`. The default Impit client enforces the same value as a deadline for the whole request, body included. - Requires the `httpx` extra: `pip install "apify-client[httpx]"`. + Requires the `httpx2` extra: `pip install "apify-client[httpx2]"`. """ def __init__( @@ -65,7 +65,7 @@ def __init__( headers: dict[str, str] | None = None, http_compressor: HttpCompressor | None = None, ) -> None: - """Initialize the HTTPX-based synchronous HTTP client. + """Initialize the HTTPX2-based synchronous HTTP client. Args: token: Apify API token for authentication. @@ -92,27 +92,27 @@ def __init__( http_compressor=http_compressor, ) - self._httpx_client = httpx.Client( + self._httpx2_client = httpx2.Client( follow_redirects=True, event_hooks={'response': [self._clear_response_cookies]}, ) @override def is_timeout_error(self, exc: Exception) -> bool: - return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) + return super().is_timeout_error(exc) or isinstance(exc, httpx2.TimeoutException) @override def is_retryable_transport_error(self, exc: Exception) -> bool: - # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in - # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX adds later is retried rather than + # Every error from HTTPX2's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX2 adds later is retried rather than # silently treated as fatal. HTTP status code errors are handled by the shared pipeline based on the # response status code, not here. - return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + return isinstance(exc, httpx2.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) @override def close(self) -> None: - """Close the underlying HTTPX connection pool.""" - self._httpx_client.close() + """Close the underlying HTTPX2 connection pool.""" + self._httpx2_client.close() @override def send_request( @@ -124,8 +124,8 @@ def send_request( content: bytes | None, timeout: float | None, stream: bool, - ) -> httpx.Response: - request = self._httpx_client.build_request( + ) -> httpx2.Response: + request = self._httpx2_client.build_request( method=method, url=url, headers=headers, @@ -133,25 +133,25 @@ def send_request( timeout=timeout, ) _restore_explicit_cookie_header(request, headers) - return self._httpx_client.send(request, stream=stream) + return self._httpx2_client.send(request, stream=stream) - def _clear_response_cookies(self, _response: httpx.Response) -> None: - """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" - self._httpx_client.cookies.clear() + def _clear_response_cookies(self, _response: httpx2.Response) -> None: + """Prevent HTTPX2's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx2_client.cookies.clear() @docs_group('HTTP clients') -class HttpxHttpClientAsync(HttpClientAsync): - """Asynchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/). +class Httpx2HttpClientAsync(HttpClientAsync): + """Asynchronous HTTP client for the Apify API built on top of [HTTPX2](https://github.com/pydantic/httpx2). - This client wraps `httpx.AsyncClient` and adds automatic retries with exponential backoff for rate-limited + This client wraps `httpx2.AsyncClient` and adds automatic retries with exponential backoff for rate-limited (HTTP 429) and server error (HTTP 5xx) responses. - HTTPX applies a request timeout to each socket operation rather than to the request as a whole, so a response + HTTPX2 applies a request timeout to each socket operation rather than to the request as a whole, so a response whose body arrives slowly keeps resetting it and can outlast both the requested timeout and `timeout_max`. The default Impit client enforces the same value as a deadline for the whole request, body included. - Requires the `httpx` extra: `pip install "apify-client[httpx]"`. + Requires the `httpx2` extra: `pip install "apify-client[httpx2]"`. """ def __init__( @@ -168,7 +168,7 @@ def __init__( headers: dict[str, str] | None = None, http_compressor: HttpCompressor | None = None, ) -> None: - """Initialize the HTTPX-based asynchronous HTTP client. + """Initialize the HTTPX2-based asynchronous HTTP client. Args: token: Apify API token for authentication. @@ -195,27 +195,27 @@ def __init__( http_compressor=http_compressor, ) - self._httpx_async_client = httpx.AsyncClient( + self._httpx2_async_client = httpx2.AsyncClient( follow_redirects=True, event_hooks={'response': [self._clear_response_cookies]}, ) @override def is_timeout_error(self, exc: Exception) -> bool: - return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) + return super().is_timeout_error(exc) or isinstance(exc, httpx2.TimeoutException) @override def is_retryable_transport_error(self, exc: Exception) -> bool: - # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in - # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX adds later is retried rather than + # Every error from HTTPX2's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX2 adds later is retried rather than # silently treated as fatal. HTTP status code errors are handled by the shared pipeline based on the # response status code, not here. - return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + return isinstance(exc, httpx2.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) @override async def aclose(self) -> None: - """Close the underlying asynchronous HTTPX connection pool.""" - await self._httpx_async_client.aclose() + """Close the underlying asynchronous HTTPX2 connection pool.""" + await self._httpx2_async_client.aclose() @override async def send_request( @@ -227,8 +227,8 @@ async def send_request( content: bytes | None, timeout: float | None, stream: bool, - ) -> httpx.Response: - request = self._httpx_async_client.build_request( + ) -> httpx2.Response: + request = self._httpx2_async_client.build_request( method=method, url=url, headers=headers, @@ -236,17 +236,17 @@ async def send_request( timeout=timeout, ) _restore_explicit_cookie_header(request, headers) - return await self._httpx_async_client.send(request, stream=stream) + return await self._httpx2_async_client.send(request, stream=stream) - async def _clear_response_cookies(self, _response: httpx.Response) -> None: - """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" - self._httpx_async_client.cookies.clear() + async def _clear_response_cookies(self, _response: httpx2.Response) -> None: + """Prevent HTTPX2's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx2_async_client.cookies.clear() -def _restore_explicit_cookie_header(request: httpx.Request, headers: dict[str, str]) -> None: - """Keep only cookies explicitly supplied for this request, never cookies from HTTPX's shared jar. +def _restore_explicit_cookie_header(request: httpx2.Request, headers: dict[str, str]) -> None: + """Keep only cookies explicitly supplied for this request, never cookies from HTTPX2's shared jar. - HTTPX drops the `Cookie` header when it builds a redirect request and rebuilds it from the jar, so an explicit + HTTPX2 drops the `Cookie` header when it builds a redirect request and rebuilds it from the jar, so an explicit cookie only reaches the first hop of a redirected request. """ explicit_cookie = next((value for key, value in headers.items() if key.lower() == 'cookie'), None) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e0c9bed6..93e22bf3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -21,8 +21,8 @@ from apify_client.http_clients import ( HttpClient, HttpClientAsync, - HttpxHttpClient, - HttpxHttpClientAsync, + Httpx2HttpClient, + Httpx2HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -44,7 +44,7 @@ class HttpClientClasses: ALL_HTTP_CLIENT_CLASSES = [ pytest.param(DEFAULT_HTTP_CLIENT_CLASSES, id='impit'), - pytest.param(HttpClientClasses(sync=HttpxHttpClient, async_=HttpxHttpClientAsync), id='httpx'), + pytest.param(HttpClientClasses(sync=Httpx2HttpClient, async_=Httpx2HttpClientAsync), id='httpx2'), ] """Every built-in HTTP client, for tests that exercise transport behavior rather than an API resource.""" diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 98d48d70..5369e730 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -10,8 +10,8 @@ from apify_client.http_clients import ( HttpClient, HttpClientAsync, - HttpxHttpClient, - HttpxHttpClientAsync, + Httpx2HttpClient, + Httpx2HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -53,7 +53,7 @@ def async_client(httpserver: HTTPServer) -> ApifyClientAsync: @pytest.fixture( params=[ pytest.param(ImpitHttpClient, id='impit'), - pytest.param(HttpxHttpClient, id='httpx'), + pytest.param(Httpx2HttpClient, id='httpx2'), ] ) def http_client_class(request: pytest.FixtureRequest) -> type[HttpClient]: @@ -64,7 +64,7 @@ def http_client_class(request: pytest.FixtureRequest) -> type[HttpClient]: @pytest.fixture( params=[ pytest.param(ImpitHttpClientAsync, id='impit'), - pytest.param(HttpxHttpClientAsync, id='httpx'), + pytest.param(Httpx2HttpClientAsync, id='httpx2'), ] ) def http_client_async_class(request: pytest.FixtureRequest) -> type[HttpClientAsync]: diff --git a/tests/unit/test_client_headers.py b/tests/unit/test_client_headers.py index 846df89e..8706c31b 100644 --- a/tests/unit/test_client_headers.py +++ b/tests/unit/test_client_headers.py @@ -6,10 +6,10 @@ from importlib import metadata from typing import TYPE_CHECKING -import httpx +import httpx2 from werkzeug import Request, Response -from apify_client.http_clients import HttpxHttpClient, HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync +from apify_client.http_clients import Httpx2HttpClient, Httpx2HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync if TYPE_CHECKING: from pytest_httpserver import HTTPServer @@ -28,9 +28,9 @@ def _transport_wire_headers( """Return the headers the transport adds on its own and the content encodings it advertises.""" if issubclass(client_class, (ImpitHttpClient, ImpitHttpClientAsync)): return {}, {'zstd', 'gzip', 'deflate', 'br'} - # HTTPX advertises whichever decoders happen to be installed alongside it, so read the set off the client + # HTTPX2 advertises whichever decoders happen to be installed alongside it, so read the set off the client # itself rather than hard-coding it and breaking whenever the environment gains or loses a codec. - with httpx.Client() as probe: + with httpx2.Client() as probe: return {'Connection': 'keep-alive'}, _parse_accept_encoding(probe.headers['accept-encoding']) @@ -187,56 +187,56 @@ def _echo_cookie_handler(request: Request) -> Response: return Response(json.dumps({'cookie': request.headers.get('Cookie')}), content_type='application/json') -def test_httpx_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: - """A Set-Cookie response must not enter HTTPX's shared cookie jar, nor leak into a later API request.""" +def test_httpx2_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """A Set-Cookie response must not enter HTTPX2's shared cookie jar, nor leak into a later API request.""" httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - with HttpxHttpClient() as client: + with Httpx2HttpClient() as client: client.call(method='GET', url=httpserver.url_for('/set-cookie')) - assert len(client._httpx_client.cookies) == 0 + assert len(client._httpx2_client.cookies) == 0 response = client.call(method='GET', url=httpserver.url_for('/echo-cookie')) assert response.json() == {'cookie': None} -async def test_httpx_async_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: - """The asynchronous HTTPX pool also remains stateless between API calls.""" +async def test_httpx2_async_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """The asynchronous HTTPX2 pool also remains stateless between API calls.""" httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - async with HttpxHttpClientAsync() as client: + async with Httpx2HttpClientAsync() as client: await client.call(method='GET', url=httpserver.url_for('/set-cookie')) - assert len(client._httpx_async_client.cookies) == 0 + assert len(client._httpx2_async_client.cookies) == 0 response = await client.call(method='GET', url=httpserver.url_for('/echo-cookie')) assert response.json() == {'cookie': None} -def test_httpx_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: +def test_httpx2_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: """A cookie another in-flight request left in the shared jar must not ride along on the next request.""" httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - with HttpxHttpClient() as client: - client._httpx_client.cookies.set('session', 'secret', domain=httpserver.host) + with Httpx2HttpClient() as client: + client._httpx2_client.cookies.set('session', 'secret', domain=httpserver.host) response = client.call(method='GET', url=httpserver.url_for('/echo-cookie')) assert response.json() == {'cookie': None} -async def test_httpx_async_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: +async def test_httpx2_async_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: """The asynchronous pool, where concurrent requests really do share one jar, drops leftover cookies too.""" httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - async with HttpxHttpClientAsync() as client: - client._httpx_async_client.cookies.set('session', 'secret', domain=httpserver.host) + async with Httpx2HttpClientAsync() as client: + client._httpx2_async_client.cookies.set('session', 'secret', domain=httpserver.host) response = await client.call(method='GET', url=httpserver.url_for('/echo-cookie')) assert response.json() == {'cookie': None} -def test_httpx_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: - """A cookie set by a redirecting response must not ride along on the next hop, which HTTPX builds from its jar.""" +def test_httpx2_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: + """A cookie set by a redirecting response must not ride along on the next hop, which HTTPX2 builds from its jar.""" httpserver.expect_request('/redirect').respond_with_data( '', status=302, @@ -244,13 +244,13 @@ def test_httpx_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPS ) httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - with HttpxHttpClient() as client: + with Httpx2HttpClient() as client: response = client.call(method='GET', url=httpserver.url_for('/redirect')) assert response.json() == {'cookie': None} -async def test_httpx_async_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: +async def test_httpx2_async_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: """The asynchronous pool keeps a redirecting response's cookie off the next hop too.""" httpserver.expect_request('/redirect').respond_with_data( '', @@ -259,17 +259,17 @@ async def test_httpx_async_does_not_carry_server_cookies_across_a_redirect(https ) httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - async with HttpxHttpClientAsync() as client: + async with Httpx2HttpClientAsync() as client: response = await client.call(method='GET', url=httpserver.url_for('/redirect')) assert response.json() == {'cookie': None} -def test_httpx_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: +def test_httpx2_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: """Disabling the shared cookie jar must not remove a Cookie header explicitly supplied by the caller.""" httpserver.expect_request('/echo-explicit-cookie').respond_with_handler(_echo_cookie_handler) - with HttpxHttpClient() as client: + with Httpx2HttpClient() as client: response = client.call( method='GET', url=httpserver.url_for('/echo-explicit-cookie'), @@ -279,11 +279,11 @@ def test_httpx_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: assert response.json() == {'cookie': 'explicit=value'} -async def test_httpx_async_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: +async def test_httpx2_async_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: """The asynchronous pool forwards an explicitly supplied Cookie header as well.""" httpserver.expect_request('/echo-explicit-cookie').respond_with_handler(_echo_cookie_handler) - async with HttpxHttpClientAsync() as client: + async with Httpx2HttpClientAsync() as client: response = await client.call( method='GET', url=httpserver.url_for('/echo-explicit-cookie'), diff --git a/tests/unit/test_client_timeouts.py b/tests/unit/test_client_timeouts.py index 54f747c1..f2ce6bb4 100644 --- a/tests/unit/test_client_timeouts.py +++ b/tests/unit/test_client_timeouts.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock -import httpx +import httpx2 import impit import pytest @@ -13,8 +13,8 @@ from apify_client.http_clients import ( HttpClient, HttpClientAsync, - HttpxHttpClient, - HttpxHttpClientAsync, + Httpx2HttpClient, + Httpx2HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -23,8 +23,8 @@ if TYPE_CHECKING: from _pytest.logging import LogCaptureFixture -UNSET_HTTPX_TIMEOUT = {'connect': None, 'read': None, 'write': None, 'pool': None} -"""What HTTPX stores on a request built with `timeout=None`: every sub-timeout unset, not the client default.""" +UNSET_HTTPX2_TIMEOUT = {'connect': None, 'read': None, 'write': None, 'pool': None} +"""What HTTPX2 stores on a request built with `timeout=None`: every sub-timeout unset, not the client default.""" @pytest.fixture @@ -40,7 +40,7 @@ def successful_response() -> Mock: def retryable_error(client: HttpClient | HttpClientAsync) -> Exception: if isinstance(client, (ImpitHttpClient, ImpitHttpClientAsync)): return impit.TimeoutException('timeout') - return httpx.ReadTimeout('timeout', request=httpx.Request('GET', 'https://example.com')) + return httpx2.ReadTimeout('timeout', request=httpx2.Request('GET', 'https://example.com')) @pytest.mark.parametrize( @@ -235,29 +235,29 @@ async def test_no_timeout_mapping_for_async_impit_adapter() -> None: assert client._impit_async_client.request.call_args.kwargs['timeout'] == 86_400 -def test_no_timeout_mapping_for_sync_httpx_adapter(monkeypatch: pytest.MonkeyPatch) -> None: - """The synchronous HTTPX adapter maps no-timeout to every HTTPX sub-timeout being unset.""" - # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. - with HttpxHttpClient() as client: +def test_no_timeout_mapping_for_sync_httpx2_adapter(monkeypatch: pytest.MonkeyPatch) -> None: + """The synchronous HTTPX2 adapter maps no-timeout to every HTTPX2 sub-timeout being unset.""" + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX2. + with Httpx2HttpClient() as client: send = Mock(return_value=successful_response()) - monkeypatch.setattr(client._httpx_client, 'send', send) + monkeypatch.setattr(client._httpx2_client, 'send', send) client.send_request( method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False ) - assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX2_TIMEOUT -async def test_no_timeout_mapping_for_async_httpx_adapter(monkeypatch: pytest.MonkeyPatch) -> None: - """The asynchronous HTTPX adapter maps no-timeout to every HTTPX sub-timeout being unset.""" - # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. - async with HttpxHttpClientAsync() as client: +async def test_no_timeout_mapping_for_async_httpx2_adapter(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX2 adapter maps no-timeout to every HTTPX2 sub-timeout being unset.""" + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX2. + async with Httpx2HttpClientAsync() as client: send = AsyncMock(return_value=successful_response()) - monkeypatch.setattr(client._httpx_async_client, 'send', send) + monkeypatch.setattr(client._httpx2_async_client, 'send', send) await client.send_request( method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False ) - assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX2_TIMEOUT diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index b2b0e369..3e441189 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -11,7 +11,7 @@ from unittest.mock import AsyncMock, Mock import brotli -import httpx +import httpx2 import impit import pytest @@ -22,8 +22,8 @@ HttpClient, HttpClientAsync, HttpResponse, - HttpxHttpClient, - HttpxHttpClientAsync, + Httpx2HttpClient, + Httpx2HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -267,22 +267,22 @@ async def test_http_client_async_creates_async_impit_client() -> None: await client.aclose() -def test_http_client_creates_sync_httpx_client() -> None: - """The synchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" - client = HttpxHttpClient(token='test_token_123') +def test_http_client_creates_sync_httpx2_client() -> None: + """The synchronous HTTPX2 adapter creates the underlying HTTPX2 client, and the close hook closes its pool.""" + client = Httpx2HttpClient(token='test_token_123') - assert isinstance(client._httpx_client, httpx.Client) + assert isinstance(client._httpx2_client, httpx2.Client) client.close() - assert client._httpx_client.is_closed + assert client._httpx2_client.is_closed -async def test_http_client_async_creates_async_httpx_client() -> None: - """The asynchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" - client = HttpxHttpClientAsync(token='test_token_123') +async def test_http_client_async_creates_async_httpx2_client() -> None: + """The asynchronous HTTPX2 adapter creates the underlying HTTPX2 client, and the close hook closes its pool.""" + client = Httpx2HttpClientAsync(token='test_token_123') - assert isinstance(client._httpx_async_client, httpx.AsyncClient) + assert isinstance(client._httpx2_async_client, httpx2.AsyncClient) await client.aclose() - assert client._httpx_async_client.is_closed + assert client._httpx2_async_client.is_closed def test_parse_params_none() -> None: @@ -416,64 +416,64 @@ async def test_async_http_client_classifies_timeout_errors() -> None: @pytest.mark.parametrize( 'exc', [ - # Even the generic base class is transient: HTTPX subclasses it for every failure mode, so an + # Even the generic base class is transient: HTTPX2 subclasses it for every failure mode, so an # unclassified failure is safer to retry. - pytest.param(httpx.HTTPError('unclassified failure'), id='bare HTTPError'), - pytest.param(httpx.TimeoutException('timeout'), id='TimeoutException'), - pytest.param(httpx.NetworkError('network error'), id='NetworkError'), - pytest.param(httpx.RemoteProtocolError('remote protocol error'), id='RemoteProtocolError'), - pytest.param(httpx.DecodingError('decoding error'), id='DecodingError'), + pytest.param(httpx2.HTTPError('unclassified failure'), id='bare HTTPError'), + pytest.param(httpx2.TimeoutException('timeout'), id='TimeoutException'), + pytest.param(httpx2.NetworkError('network error'), id='NetworkError'), + pytest.param(httpx2.RemoteProtocolError('remote protocol error'), id='RemoteProtocolError'), + pytest.param(httpx2.DecodingError('decoding error'), id='DecodingError'), # One `ProxyError` covers both a proxy rejecting the CONNECT tunnel and a 407, so a transient case cannot # be told from a permanent one - retrying is the safer default. - pytest.param(httpx.ProxyError('proxy error'), id='ProxyError'), + pytest.param(httpx2.ProxyError('proxy error'), id='ProxyError'), ], ) -def test_httpx_is_retryable_transport_error(exc: Exception) -> None: - """A transient HTTPX transport failure is classified as retryable.""" - with HttpxHttpClient() as client: +def test_httpx2_is_retryable_transport_error(exc: Exception) -> None: + """A transient HTTPX2 transport failure is classified as retryable.""" + with Httpx2HttpClient() as client: assert client.is_retryable_transport_error(exc) @pytest.mark.parametrize( 'exc', [ - pytest.param(httpx.LocalProtocolError('invalid header value'), id='LocalProtocolError'), - pytest.param(httpx.UnsupportedProtocol('unsupported scheme'), id='UnsupportedProtocol'), - pytest.param(httpx.TooManyRedirects('too many redirects'), id='TooManyRedirects'), + pytest.param(httpx2.LocalProtocolError('invalid header value'), id='LocalProtocolError'), + pytest.param(httpx2.UnsupportedProtocol('unsupported scheme'), id='UnsupportedProtocol'), + pytest.param(httpx2.TooManyRedirects('too many redirects'), id='TooManyRedirects'), pytest.param( - httpx.HTTPStatusError( + httpx2.HTTPStatusError( 'status error', - request=httpx.Request('GET', 'https://example.com'), - response=httpx.Response(500), + request=httpx2.Request('GET', 'https://example.com'), + response=httpx2.Response(500), ), id='HTTPStatusError', ), - # HTTPX reports a bad URL outside the `httpx.HTTPError` tree entirely. - pytest.param(httpx.InvalidURL('unsupported scheme'), id='InvalidURL'), + # HTTPX2 reports a bad URL outside the `httpx2.HTTPError` tree entirely. + pytest.param(httpx2.InvalidURL('unsupported scheme'), id='InvalidURL'), pytest.param(ValueError('value error'), id='ValueError'), pytest.param(RuntimeError('runtime error'), id='RuntimeError'), pytest.param(Exception('generic exception'), id='Exception'), ], ) -def test_httpx_is_not_retryable_transport_error(exc: Exception) -> None: - """A transport failure a retry cannot fix, and anything outside HTTPX's hierarchy, is not retried.""" - with HttpxHttpClient() as client: +def test_httpx2_is_not_retryable_transport_error(exc: Exception) -> None: + """A transport failure a retry cannot fix, and anything outside HTTPX2's hierarchy, is not retried.""" + with Httpx2HttpClient() as client: assert not client.is_retryable_transport_error(exc) -def test_sync_httpx_client_classifies_timeout_errors() -> None: - """The built-in synchronous HTTPX client exposes transport-neutral timeout classification.""" - with HttpxHttpClient() as client: +def test_sync_httpx2_client_classifies_timeout_errors() -> None: + """The built-in synchronous HTTPX2 client exposes transport-neutral timeout classification.""" + with Httpx2HttpClient() as client: assert client.is_timeout_error(TimeoutError('test')) - assert client.is_timeout_error(httpx.TimeoutException('test')) + assert client.is_timeout_error(httpx2.TimeoutException('test')) assert not client.is_timeout_error(ValueError('test')) -async def test_async_httpx_client_classifies_timeout_errors() -> None: - """The built-in asynchronous HTTPX client exposes transport-neutral timeout classification.""" - async with HttpxHttpClientAsync() as client: +async def test_async_httpx2_client_classifies_timeout_errors() -> None: + """The built-in asynchronous HTTPX2 client exposes transport-neutral timeout classification.""" + async with Httpx2HttpClientAsync() as client: assert client.is_timeout_error(TimeoutError('test')) - assert client.is_timeout_error(httpx.TimeoutException('test')) + assert client.is_timeout_error(httpx2.TimeoutException('test')) assert not client.is_timeout_error(ValueError('test')) @@ -514,52 +514,52 @@ def test_transient_transport_error_is_retried() -> None: assert request.call_count == 3 -def test_httpx_permanent_transport_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None: - """The HTTPX adapter feeds the same fail-fast classification into the shared pipeline.""" - with HttpxHttpClient(token='test_token', min_delay_between_retries=timedelta(0)) as client: - send = Mock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) - monkeypatch.setattr(client._httpx_client, 'send', send) +def test_httpx2_permanent_transport_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX2 adapter feeds the same fail-fast classification into the shared pipeline.""" + with Httpx2HttpClient(token='test_token', min_delay_between_retries=timedelta(0)) as client: + send = Mock(side_effect=httpx2.UnsupportedProtocol('unsupported scheme')) + monkeypatch.setattr(client._httpx2_client, 'send', send) - with pytest.raises(httpx.UnsupportedProtocol): + with pytest.raises(httpx2.UnsupportedProtocol): client.call(method='GET', url='https://api.test.com/endpoint') send.assert_called_once() -async def test_httpx_permanent_transport_error_is_not_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: - """The asynchronous HTTPX adapter applies the same policy, failing on the first attempt.""" - async with HttpxHttpClientAsync(token='test_token', min_delay_between_retries=timedelta(0)) as client: - send = AsyncMock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) - monkeypatch.setattr(client._httpx_async_client, 'send', send) +async def test_httpx2_permanent_transport_error_is_not_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX2 adapter applies the same policy, failing on the first attempt.""" + async with Httpx2HttpClientAsync(token='test_token', min_delay_between_retries=timedelta(0)) as client: + send = AsyncMock(side_effect=httpx2.UnsupportedProtocol('unsupported scheme')) + monkeypatch.setattr(client._httpx2_async_client, 'send', send) - with pytest.raises(httpx.UnsupportedProtocol): + with pytest.raises(httpx2.UnsupportedProtocol): await client.call(method='GET', url='https://api.test.com/endpoint') send.assert_awaited_once() -def test_httpx_transient_transport_error_is_retried(monkeypatch: pytest.MonkeyPatch) -> None: - """The HTTPX adapter keeps a transient transport failure inside the shared retry loop.""" - with HttpxHttpClient(token='test_token', max_retries=2, min_delay_between_retries=timedelta(0)) as client: - send = Mock(side_effect=httpx.TimeoutException('timeout')) - monkeypatch.setattr(client._httpx_client, 'send', send) +def test_httpx2_transient_transport_error_is_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX2 adapter keeps a transient transport failure inside the shared retry loop.""" + with Httpx2HttpClient(token='test_token', max_retries=2, min_delay_between_retries=timedelta(0)) as client: + send = Mock(side_effect=httpx2.TimeoutException('timeout')) + monkeypatch.setattr(client._httpx2_client, 'send', send) - with pytest.raises(httpx.TimeoutException): + with pytest.raises(httpx2.TimeoutException): client.call(method='GET', url='https://api.test.com/endpoint') # `max_retries` attempts inside the backoff loop, plus the final one it makes after the last delay. assert send.call_count == 3 -async def test_httpx_transient_transport_error_is_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: - """The asynchronous HTTPX adapter keeps a transient transport failure inside the shared retry loop too.""" - async with HttpxHttpClientAsync( +async def test_httpx2_transient_transport_error_is_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX2 adapter keeps a transient transport failure inside the shared retry loop too.""" + async with Httpx2HttpClientAsync( token='test_token', max_retries=2, min_delay_between_retries=timedelta(0) ) as client: - send = AsyncMock(side_effect=httpx.TimeoutException('timeout')) - monkeypatch.setattr(client._httpx_async_client, 'send', send) + send = AsyncMock(side_effect=httpx2.TimeoutException('timeout')) + monkeypatch.setattr(client._httpx2_async_client, 'send', send) - with pytest.raises(httpx.TimeoutException): + with pytest.raises(httpx2.TimeoutException): await client.call(method='GET', url='https://api.test.com/endpoint') # `max_retries` attempts inside the backoff loop, plus the final one it makes after the last delay. diff --git a/tests/unit/test_pluggable_http_client.py b/tests/unit/test_pluggable_http_client.py index 779125b8..37725f19 100644 --- a/tests/unit/test_pluggable_http_client.py +++ b/tests/unit/test_pluggable_http_client.py @@ -356,8 +356,8 @@ def test_public_exports() -> None: 'HttpClient', 'HttpClientAsync', 'HttpResponse', - 'HttpxHttpClient', - 'HttpxHttpClientAsync', + 'Httpx2HttpClient', + 'Httpx2HttpClientAsync', 'ImpitHttpClient', 'ImpitHttpClientAsync', ): @@ -367,19 +367,19 @@ def test_public_exports() -> None: assert not hasattr(http_clients_module, 'HttpClientBase') -def test_httpx_clients_raise_clear_error_when_extra_missing() -> None: - """Missing HTTPX keeps normal and star imports usable while explicit HTTPX access raises a clear error.""" +def test_httpx2_clients_raise_clear_error_when_extra_missing() -> None: + """Missing HTTPX2 keeps normal and star imports usable while explicit HTTPX2 access raises a clear error.""" script = dedent( """ import sys - class BlockHttpx: + class BlockHttpx2: def find_spec(self, name, *_args): - if name == 'httpx' or name.startswith('httpx.'): - raise ModuleNotFoundError(f"No module named '{name}'", name='httpx') + if name == 'httpx2' or name.startswith('httpx2.'): + raise ModuleNotFoundError(f"No module named '{name}'", name='httpx2') return None - sys.meta_path.insert(0, BlockHttpx()) + sys.meta_path.insert(0, BlockHttpx2()) import apify_client.http_clients as module assert module.HttpClient is not None @@ -388,13 +388,13 @@ def find_spec(self, name, *_args): namespace = {} exec('from apify_client.http_clients import *', namespace) assert namespace['HttpClient'] is module.HttpClient - assert 'HttpxHttpClient' not in namespace + assert 'Httpx2HttpClient' not in namespace - for name in ('HttpxHttpClient', 'HttpxHttpClientAsync'): + for name in ('Httpx2HttpClient', 'Httpx2HttpClientAsync'): try: getattr(module, name) except ImportError as exc: - assert "No module named 'httpx'" in str(exc) + assert "No module named 'httpx2'" in str(exc) else: raise AssertionError(f'{name} did not raise ImportError') """ diff --git a/uv.lock b/uv.lock index 0e2383f7..94d84184 100644 --- a/uv.lock +++ b/uv.lock @@ -3,7 +3,8 @@ revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14'", - "python_full_version < '3.14'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "(python_full_version < '3.14' and sys_platform != 'emscripten') or (python_full_version < '3.12' and sys_platform == 'emscripten')", ] [options] @@ -54,8 +55,8 @@ dependencies = [ brotli = [ { name = "brotli" }, ] -httpx = [ - { name = "httpx" }, +httpx2 = [ + { name = "httpx2" }, ] [package.dev-dependencies] @@ -83,13 +84,13 @@ dev = [ requires-dist = [ { name = "brotli", marker = "extra == 'brotli'", specifier = ">=1.0.9" }, { name = "colorama", specifier = ">=0.4.0" }, - { name = "httpx", marker = "extra == 'httpx'", specifier = ">=0.27.0,<1.0.0" }, + { name = "httpx2", marker = "extra == 'httpx2'", specifier = ">=2.0.0,<3.0.0" }, { name = "impit", specifier = "~=0.13.0" }, { name = "more-itertools", specifier = ">=10.0.0" }, { name = "pydantic", extras = ["email"], specifier = ">=2.11.0" }, { name = "typing-extensions", specifier = ">=4.6.0" }, ] -provides-extras = ["brotli", "httpx"] +provides-extras = ["brotli", "httpx2"] [package.metadata.requires-dev] dev = [ @@ -739,6 +740,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/74/d370e55600d9bcfa0d9794b0166126d49291a3d2b20c268fc98c453a4948/httpcore2-2.12.0-py3-none-any.whl", hash = "sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb", size = 83074, upload-time = "2026-08-18T13:22:05.854Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -754,6 +768,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "python_full_version >= '3.12' and sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/f8/579a8b51e42e38ee32647df9f08aa25643ae788e275cc625b199829c4671/httpx2-2.12.0.tar.gz", hash = "sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf", size = 100040, upload-time = "2026-08-18T13:22:09.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/95/411ba65569158e862368917aaf56597f3e5fa3b91b0502919638465a08f3/httpx2-2.12.0-py3-none-any.whl", hash = "sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36", size = 95427, upload-time = "2026-08-18T13:22:06.834Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + [[package]] name = "identify" version = "2.6.19" @@ -1512,6 +1552,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "ty" version = "0.0.72" From 0e3914a7823ca6dc6cf5d1c08ae68780f7cd29ad Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 18:49:34 +0200 Subject: [PATCH 4/4] refactor: Keep the HTTPX naming for the httpx2-backed HTTP client --- .rules.md | 2 +- README.md | 7 +- pyproject.toml | 2 +- src/apify_client/http_clients/__init__.py | 16 +-- .../http_clients/{_httpx2.py => _httpx.py} | 100 ++++++------- tests/integration/conftest.py | 6 +- tests/unit/conftest.py | 8 +- tests/unit/test_client_headers.py | 54 +++---- tests/unit/test_client_timeouts.py | 36 ++--- tests/unit/test_http_clients.py | 132 +++++++++--------- tests/unit/test_pluggable_http_client.py | 16 +-- uv.lock | 2 +- 12 files changed, 192 insertions(+), 189 deletions(-) rename src/apify_client/http_clients/{_httpx2.py => _httpx.py} (71%) diff --git a/.rules.md b/.rules.md index a334de8a..3563212e 100644 --- a/.rules.md +++ b/.rules.md @@ -63,7 +63,7 @@ Docstrings are written on sync clients and **automatically copied** to async cli - `HttpClient`/`HttpClientAsync` — base classes in `http_clients/_base.py` holding the shared request pipeline (retries, timeouts, API errors); transports implement the `send_request`, error-classification, and lifecycle hooks - `ImpitHttpClient`/`ImpitHttpClientAsync` — default implementation (Rust-based Impit) -- `Httpx2HttpClient`/`Httpx2HttpClientAsync` — built-in alternative behind the `httpx2` optional extra +- `HttpxHttpClient`/`HttpxHttpClientAsync` — built-in alternative behind the `httpx2` optional extra - `HttpResponse` — Protocol (not a concrete class) for response objects - Users can plug in custom HTTP clients via `ApifyClient.with_custom_http_client()` diff --git a/README.md b/README.md index f2cdfbaf..d9224865 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,9 @@ ``` [Impit](https://github.com/apify/impit) is the default HTTP client and is installed automatically. To use the - built-in [HTTPX2](https://github.com/pydantic/httpx2) client instead, install its optional extra and pass - `http_client=Httpx2HttpClient()` to `ApifyClient.with_custom_http_client()`: + built-in [HTTPX](https://github.com/pydantic/httpx2) client instead, install its optional extra and pass + `http_client=HttpxHttpClient()` to `ApifyClient.with_custom_http_client()`. The extra installs `httpx2`, + Pydantic's maintained continuation of HTTPX: ```bash pip install "apify-client[httpx2]" @@ -134,7 +135,7 @@ For a guided walkthrough — authenticating, running an Actor, and reading its r - **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call ([Timeouts](https://docs.apify.com/api/client/python/docs/concepts/timeouts)). - **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)). - **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks ([Convenience methods](https://docs.apify.com/api/client/python/docs/concepts/convenience-methods)). -- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX2](https://github.com/pydantic/httpx2) client, or plug in any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). +- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX](https://github.com/pydantic/httpx2) client, or plug in any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). - **Structured errors** — every API error surfaces as an [`ApifyApiError`](https://docs.apify.com/api/client/python/reference/class/ApifyApiError) with HTTP-specific subclasses for precise handling ([Error handling](https://docs.apify.com/api/client/python/docs/concepts/error-handling)). - **Debug logging** — opt-in structured logging on the `apify_client` logger captures request URLs, status codes, retry attempts, and more ([Logging](https://docs.apify.com/api/client/python/docs/concepts/logging)). diff --git a/pyproject.toml b/pyproject.toml index f0182ebb..dc7409c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ [project.optional-dependencies] brotli = ["brotli>=1.0.9"] -httpx2 = ["httpx2>=2.0.0,<3.0.0"] +httpx2 = ["httpx2>=2.0.0"] [project.urls] "Apify Homepage" = "https://apify.com" diff --git a/src/apify_client/http_clients/__init__.py b/src/apify_client/http_clients/__init__.py index 73b56b24..b5f3a11a 100644 --- a/src/apify_client/http_clients/__init__.py +++ b/src/apify_client/http_clients/__init__.py @@ -5,23 +5,23 @@ _install_import_hook(__name__) -# `httpx2` is an optional extra, so it's wrapped in try_import. Accessing the HTTPX2 clients +# `httpx2` is an optional extra, so it's wrapped in try_import. Accessing the HTTPX clients # without the extra installed raises a clear ImportError instead of failing at package import time. with _try_import( __name__, - 'Httpx2HttpClient', - 'Httpx2HttpClientAsync', + 'HttpxHttpClient', + 'HttpxHttpClientAsync', dependency_name='httpx2', -) as _httpx2_import: - from apify_client.http_clients._httpx2 import Httpx2HttpClient, Httpx2HttpClientAsync +) as _httpx_import: + from apify_client.http_clients._httpx import HttpxHttpClient, HttpxHttpClientAsync -if _httpx2_import.available: +if _httpx_import.available: __all__ = [ 'HttpClient', 'HttpClientAsync', 'HttpResponse', - 'Httpx2HttpClient', - 'Httpx2HttpClientAsync', + 'HttpxHttpClient', + 'HttpxHttpClientAsync', 'ImpitHttpClient', 'ImpitHttpClientAsync', ] diff --git a/src/apify_client/http_clients/_httpx2.py b/src/apify_client/http_clients/_httpx.py similarity index 71% rename from src/apify_client/http_clients/_httpx2.py rename to src/apify_client/http_clients/_httpx.py index 6ae27229..fc39a34f 100644 --- a/src/apify_client/http_clients/_httpx2.py +++ b/src/apify_client/http_clients/_httpx.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -import httpx2 +import httpx2 as httpx from typing_extensions import override from apify_client._consts import ( @@ -24,31 +24,32 @@ _PERMANENT_ERRORS = ( - # A request HTTPX2 rejects before sending it, e.g. one carrying an invalid header value. - httpx2.LocalProtocolError, - # A URL scheme HTTPX2 refuses to speak, which repeating the request cannot change. - httpx2.UnsupportedProtocol, + # A request HTTPX rejects before sending it, e.g. one carrying an invalid header value. + httpx.LocalProtocolError, + # A URL scheme HTTPX refuses to speak, which repeating the request cannot change. + httpx.UnsupportedProtocol, # An over-long redirect chain is a routing loop, which repeating the request cannot break. - httpx2.TooManyRedirects, + httpx.TooManyRedirects, # Only `Response.raise_for_status()` raises this, and the client never calls it - the shared pipeline decides on # status codes from the response itself. - httpx2.HTTPStatusError, + httpx.HTTPStatusError, ) -"""HTTPX2 errors that a retry cannot fix. Everything else in the `httpx2.HTTPError` tree counts as transient.""" +"""HTTPX errors that a retry cannot fix. Everything else in the `httpx.HTTPError` tree counts as transient.""" @docs_group('HTTP clients') -class Httpx2HttpClient(HttpClient): - """Synchronous HTTP client for the Apify API built on top of [HTTPX2](https://github.com/pydantic/httpx2). +class HttpxHttpClient(HttpClient): + """Synchronous HTTP client for the Apify API built on top of [HTTPX](https://github.com/pydantic/httpx2). - This client wraps `httpx2.Client` and adds automatic retries with exponential backoff for rate-limited + This client wraps `httpx.Client` and adds automatic retries with exponential backoff for rate-limited (HTTP 429) and server error (HTTP 5xx) responses. - HTTPX2 applies a request timeout to each socket operation rather than to the request as a whole, so a response + HTTPX applies a request timeout to each socket operation rather than to the request as a whole, so a response whose body arrives slowly keeps resetting it and can outlast both the requested timeout and `timeout_max`. The default Impit client enforces the same value as a deadline for the whole request, body included. - Requires the `httpx2` extra: `pip install "apify-client[httpx2]"`. + Requires the `httpx2` extra: `pip install "apify-client[httpx2]"`. That extra installs `httpx2`, Pydantic's + maintained continuation of HTTPX, which this module imports under the `httpx` name. """ def __init__( @@ -65,7 +66,7 @@ def __init__( headers: dict[str, str] | None = None, http_compressor: HttpCompressor | None = None, ) -> None: - """Initialize the HTTPX2-based synchronous HTTP client. + """Initialize the HTTPX-based synchronous HTTP client. Args: token: Apify API token for authentication. @@ -92,27 +93,27 @@ def __init__( http_compressor=http_compressor, ) - self._httpx2_client = httpx2.Client( + self._httpx_client = httpx.Client( follow_redirects=True, event_hooks={'response': [self._clear_response_cookies]}, ) @override def is_timeout_error(self, exc: Exception) -> bool: - return super().is_timeout_error(exc) or isinstance(exc, httpx2.TimeoutException) + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) @override def is_retryable_transport_error(self, exc: Exception) -> bool: - # Every error from HTTPX2's own hierarchy counts as transient except the permanently-failing types listed in - # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX2 adds later is retried rather than + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX adds later is retried rather than # silently treated as fatal. HTTP status code errors are handled by the shared pipeline based on the # response status code, not here. - return isinstance(exc, httpx2.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) @override def close(self) -> None: - """Close the underlying HTTPX2 connection pool.""" - self._httpx2_client.close() + """Close the underlying HTTPX connection pool.""" + self._httpx_client.close() @override def send_request( @@ -124,8 +125,8 @@ def send_request( content: bytes | None, timeout: float | None, stream: bool, - ) -> httpx2.Response: - request = self._httpx2_client.build_request( + ) -> httpx.Response: + request = self._httpx_client.build_request( method=method, url=url, headers=headers, @@ -133,25 +134,26 @@ def send_request( timeout=timeout, ) _restore_explicit_cookie_header(request, headers) - return self._httpx2_client.send(request, stream=stream) + return self._httpx_client.send(request, stream=stream) - def _clear_response_cookies(self, _response: httpx2.Response) -> None: - """Prevent HTTPX2's shared cookie jar from leaking server cookies into later API requests.""" - self._httpx2_client.cookies.clear() + def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_client.cookies.clear() @docs_group('HTTP clients') -class Httpx2HttpClientAsync(HttpClientAsync): - """Asynchronous HTTP client for the Apify API built on top of [HTTPX2](https://github.com/pydantic/httpx2). +class HttpxHttpClientAsync(HttpClientAsync): + """Asynchronous HTTP client for the Apify API built on top of [HTTPX](https://github.com/pydantic/httpx2). - This client wraps `httpx2.AsyncClient` and adds automatic retries with exponential backoff for rate-limited + This client wraps `httpx.AsyncClient` and adds automatic retries with exponential backoff for rate-limited (HTTP 429) and server error (HTTP 5xx) responses. - HTTPX2 applies a request timeout to each socket operation rather than to the request as a whole, so a response + HTTPX applies a request timeout to each socket operation rather than to the request as a whole, so a response whose body arrives slowly keeps resetting it and can outlast both the requested timeout and `timeout_max`. The default Impit client enforces the same value as a deadline for the whole request, body included. - Requires the `httpx2` extra: `pip install "apify-client[httpx2]"`. + Requires the `httpx2` extra: `pip install "apify-client[httpx2]"`. That extra installs `httpx2`, Pydantic's + maintained continuation of HTTPX, which this module imports under the `httpx` name. """ def __init__( @@ -168,7 +170,7 @@ def __init__( headers: dict[str, str] | None = None, http_compressor: HttpCompressor | None = None, ) -> None: - """Initialize the HTTPX2-based asynchronous HTTP client. + """Initialize the HTTPX-based asynchronous HTTP client. Args: token: Apify API token for authentication. @@ -195,27 +197,27 @@ def __init__( http_compressor=http_compressor, ) - self._httpx2_async_client = httpx2.AsyncClient( + self._httpx_async_client = httpx.AsyncClient( follow_redirects=True, event_hooks={'response': [self._clear_response_cookies]}, ) @override def is_timeout_error(self, exc: Exception) -> bool: - return super().is_timeout_error(exc) or isinstance(exc, httpx2.TimeoutException) + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) @override def is_retryable_transport_error(self, exc: Exception) -> bool: - # Every error from HTTPX2's own hierarchy counts as transient except the permanently-failing types listed in - # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX2 adds later is retried rather than + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX adds later is retried rather than # silently treated as fatal. HTTP status code errors are handled by the shared pipeline based on the # response status code, not here. - return isinstance(exc, httpx2.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) @override async def aclose(self) -> None: - """Close the underlying asynchronous HTTPX2 connection pool.""" - await self._httpx2_async_client.aclose() + """Close the underlying asynchronous HTTPX connection pool.""" + await self._httpx_async_client.aclose() @override async def send_request( @@ -227,8 +229,8 @@ async def send_request( content: bytes | None, timeout: float | None, stream: bool, - ) -> httpx2.Response: - request = self._httpx2_async_client.build_request( + ) -> httpx.Response: + request = self._httpx_async_client.build_request( method=method, url=url, headers=headers, @@ -236,17 +238,17 @@ async def send_request( timeout=timeout, ) _restore_explicit_cookie_header(request, headers) - return await self._httpx2_async_client.send(request, stream=stream) + return await self._httpx_async_client.send(request, stream=stream) - async def _clear_response_cookies(self, _response: httpx2.Response) -> None: - """Prevent HTTPX2's shared cookie jar from leaking server cookies into later API requests.""" - self._httpx2_async_client.cookies.clear() + async def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_async_client.cookies.clear() -def _restore_explicit_cookie_header(request: httpx2.Request, headers: dict[str, str]) -> None: - """Keep only cookies explicitly supplied for this request, never cookies from HTTPX2's shared jar. +def _restore_explicit_cookie_header(request: httpx.Request, headers: dict[str, str]) -> None: + """Keep only cookies explicitly supplied for this request, never cookies from HTTPX's shared jar. - HTTPX2 drops the `Cookie` header when it builds a redirect request and rebuilds it from the jar, so an explicit + HTTPX drops the `Cookie` header when it builds a redirect request and rebuilds it from the jar, so an explicit cookie only reaches the first hop of a redirected request. """ explicit_cookie = next((value for key, value in headers.items() if key.lower() == 'cookie'), None) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 93e22bf3..e0c9bed6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -21,8 +21,8 @@ from apify_client.http_clients import ( HttpClient, HttpClientAsync, - Httpx2HttpClient, - Httpx2HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -44,7 +44,7 @@ class HttpClientClasses: ALL_HTTP_CLIENT_CLASSES = [ pytest.param(DEFAULT_HTTP_CLIENT_CLASSES, id='impit'), - pytest.param(HttpClientClasses(sync=Httpx2HttpClient, async_=Httpx2HttpClientAsync), id='httpx2'), + pytest.param(HttpClientClasses(sync=HttpxHttpClient, async_=HttpxHttpClientAsync), id='httpx'), ] """Every built-in HTTP client, for tests that exercise transport behavior rather than an API resource.""" diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 5369e730..98d48d70 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -10,8 +10,8 @@ from apify_client.http_clients import ( HttpClient, HttpClientAsync, - Httpx2HttpClient, - Httpx2HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -53,7 +53,7 @@ def async_client(httpserver: HTTPServer) -> ApifyClientAsync: @pytest.fixture( params=[ pytest.param(ImpitHttpClient, id='impit'), - pytest.param(Httpx2HttpClient, id='httpx2'), + pytest.param(HttpxHttpClient, id='httpx'), ] ) def http_client_class(request: pytest.FixtureRequest) -> type[HttpClient]: @@ -64,7 +64,7 @@ def http_client_class(request: pytest.FixtureRequest) -> type[HttpClient]: @pytest.fixture( params=[ pytest.param(ImpitHttpClientAsync, id='impit'), - pytest.param(Httpx2HttpClientAsync, id='httpx2'), + pytest.param(HttpxHttpClientAsync, id='httpx'), ] ) def http_client_async_class(request: pytest.FixtureRequest) -> type[HttpClientAsync]: diff --git a/tests/unit/test_client_headers.py b/tests/unit/test_client_headers.py index 8706c31b..3ecda41b 100644 --- a/tests/unit/test_client_headers.py +++ b/tests/unit/test_client_headers.py @@ -6,10 +6,10 @@ from importlib import metadata from typing import TYPE_CHECKING -import httpx2 +import httpx2 as httpx from werkzeug import Request, Response -from apify_client.http_clients import Httpx2HttpClient, Httpx2HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync +from apify_client.http_clients import HttpxHttpClient, HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync if TYPE_CHECKING: from pytest_httpserver import HTTPServer @@ -28,9 +28,9 @@ def _transport_wire_headers( """Return the headers the transport adds on its own and the content encodings it advertises.""" if issubclass(client_class, (ImpitHttpClient, ImpitHttpClientAsync)): return {}, {'zstd', 'gzip', 'deflate', 'br'} - # HTTPX2 advertises whichever decoders happen to be installed alongside it, so read the set off the client + # HTTPX advertises whichever decoders happen to be installed alongside it, so read the set off the client # itself rather than hard-coding it and breaking whenever the environment gains or loses a codec. - with httpx2.Client() as probe: + with httpx.Client() as probe: return {'Connection': 'keep-alive'}, _parse_accept_encoding(probe.headers['accept-encoding']) @@ -187,56 +187,56 @@ def _echo_cookie_handler(request: Request) -> Response: return Response(json.dumps({'cookie': request.headers.get('Cookie')}), content_type='application/json') -def test_httpx2_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: - """A Set-Cookie response must not enter HTTPX2's shared cookie jar, nor leak into a later API request.""" +def test_httpx_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """A Set-Cookie response must not enter HTTPX's shared cookie jar, nor leak into a later API request.""" httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - with Httpx2HttpClient() as client: + with HttpxHttpClient() as client: client.call(method='GET', url=httpserver.url_for('/set-cookie')) - assert len(client._httpx2_client.cookies) == 0 + assert len(client._httpx_client.cookies) == 0 response = client.call(method='GET', url=httpserver.url_for('/echo-cookie')) assert response.json() == {'cookie': None} -async def test_httpx2_async_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: - """The asynchronous HTTPX2 pool also remains stateless between API calls.""" +async def test_httpx_async_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """The asynchronous HTTPX pool also remains stateless between API calls.""" httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - async with Httpx2HttpClientAsync() as client: + async with HttpxHttpClientAsync() as client: await client.call(method='GET', url=httpserver.url_for('/set-cookie')) - assert len(client._httpx2_async_client.cookies) == 0 + assert len(client._httpx_async_client.cookies) == 0 response = await client.call(method='GET', url=httpserver.url_for('/echo-cookie')) assert response.json() == {'cookie': None} -def test_httpx2_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: +def test_httpx_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: """A cookie another in-flight request left in the shared jar must not ride along on the next request.""" httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - with Httpx2HttpClient() as client: - client._httpx2_client.cookies.set('session', 'secret', domain=httpserver.host) + with HttpxHttpClient() as client: + client._httpx_client.cookies.set('session', 'secret', domain=httpserver.host) response = client.call(method='GET', url=httpserver.url_for('/echo-cookie')) assert response.json() == {'cookie': None} -async def test_httpx2_async_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: +async def test_httpx_async_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: """The asynchronous pool, where concurrent requests really do share one jar, drops leftover cookies too.""" httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - async with Httpx2HttpClientAsync() as client: - client._httpx2_async_client.cookies.set('session', 'secret', domain=httpserver.host) + async with HttpxHttpClientAsync() as client: + client._httpx_async_client.cookies.set('session', 'secret', domain=httpserver.host) response = await client.call(method='GET', url=httpserver.url_for('/echo-cookie')) assert response.json() == {'cookie': None} -def test_httpx2_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: - """A cookie set by a redirecting response must not ride along on the next hop, which HTTPX2 builds from its jar.""" +def test_httpx_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: + """A cookie set by a redirecting response must not ride along on the next hop, which HTTPX builds from its jar.""" httpserver.expect_request('/redirect').respond_with_data( '', status=302, @@ -244,13 +244,13 @@ def test_httpx2_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTP ) httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - with Httpx2HttpClient() as client: + with HttpxHttpClient() as client: response = client.call(method='GET', url=httpserver.url_for('/redirect')) assert response.json() == {'cookie': None} -async def test_httpx2_async_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: +async def test_httpx_async_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: """The asynchronous pool keeps a redirecting response's cookie off the next hop too.""" httpserver.expect_request('/redirect').respond_with_data( '', @@ -259,17 +259,17 @@ async def test_httpx2_async_does_not_carry_server_cookies_across_a_redirect(http ) httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) - async with Httpx2HttpClientAsync() as client: + async with HttpxHttpClientAsync() as client: response = await client.call(method='GET', url=httpserver.url_for('/redirect')) assert response.json() == {'cookie': None} -def test_httpx2_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: +def test_httpx_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: """Disabling the shared cookie jar must not remove a Cookie header explicitly supplied by the caller.""" httpserver.expect_request('/echo-explicit-cookie').respond_with_handler(_echo_cookie_handler) - with Httpx2HttpClient() as client: + with HttpxHttpClient() as client: response = client.call( method='GET', url=httpserver.url_for('/echo-explicit-cookie'), @@ -279,11 +279,11 @@ def test_httpx2_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: assert response.json() == {'cookie': 'explicit=value'} -async def test_httpx2_async_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: +async def test_httpx_async_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: """The asynchronous pool forwards an explicitly supplied Cookie header as well.""" httpserver.expect_request('/echo-explicit-cookie').respond_with_handler(_echo_cookie_handler) - async with Httpx2HttpClientAsync() as client: + async with HttpxHttpClientAsync() as client: response = await client.call( method='GET', url=httpserver.url_for('/echo-explicit-cookie'), diff --git a/tests/unit/test_client_timeouts.py b/tests/unit/test_client_timeouts.py index f2ce6bb4..a7adaa64 100644 --- a/tests/unit/test_client_timeouts.py +++ b/tests/unit/test_client_timeouts.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock -import httpx2 +import httpx2 as httpx import impit import pytest @@ -13,8 +13,8 @@ from apify_client.http_clients import ( HttpClient, HttpClientAsync, - Httpx2HttpClient, - Httpx2HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -23,8 +23,8 @@ if TYPE_CHECKING: from _pytest.logging import LogCaptureFixture -UNSET_HTTPX2_TIMEOUT = {'connect': None, 'read': None, 'write': None, 'pool': None} -"""What HTTPX2 stores on a request built with `timeout=None`: every sub-timeout unset, not the client default.""" +UNSET_HTTPX_TIMEOUT = {'connect': None, 'read': None, 'write': None, 'pool': None} +"""What HTTPX stores on a request built with `timeout=None`: every sub-timeout unset, not the client default.""" @pytest.fixture @@ -40,7 +40,7 @@ def successful_response() -> Mock: def retryable_error(client: HttpClient | HttpClientAsync) -> Exception: if isinstance(client, (ImpitHttpClient, ImpitHttpClientAsync)): return impit.TimeoutException('timeout') - return httpx2.ReadTimeout('timeout', request=httpx2.Request('GET', 'https://example.com')) + return httpx.ReadTimeout('timeout', request=httpx.Request('GET', 'https://example.com')) @pytest.mark.parametrize( @@ -235,29 +235,29 @@ async def test_no_timeout_mapping_for_async_impit_adapter() -> None: assert client._impit_async_client.request.call_args.kwargs['timeout'] == 86_400 -def test_no_timeout_mapping_for_sync_httpx2_adapter(monkeypatch: pytest.MonkeyPatch) -> None: - """The synchronous HTTPX2 adapter maps no-timeout to every HTTPX2 sub-timeout being unset.""" - # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX2. - with Httpx2HttpClient() as client: +def test_no_timeout_mapping_for_sync_httpx_adapter(monkeypatch: pytest.MonkeyPatch) -> None: + """The synchronous HTTPX adapter maps no-timeout to every HTTPX sub-timeout being unset.""" + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. + with HttpxHttpClient() as client: send = Mock(return_value=successful_response()) - monkeypatch.setattr(client._httpx2_client, 'send', send) + monkeypatch.setattr(client._httpx_client, 'send', send) client.send_request( method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False ) - assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX2_TIMEOUT + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT -async def test_no_timeout_mapping_for_async_httpx2_adapter(monkeypatch: pytest.MonkeyPatch) -> None: - """The asynchronous HTTPX2 adapter maps no-timeout to every HTTPX2 sub-timeout being unset.""" - # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX2. - async with Httpx2HttpClientAsync() as client: +async def test_no_timeout_mapping_for_async_httpx_adapter(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX adapter maps no-timeout to every HTTPX sub-timeout being unset.""" + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. + async with HttpxHttpClientAsync() as client: send = AsyncMock(return_value=successful_response()) - monkeypatch.setattr(client._httpx2_async_client, 'send', send) + monkeypatch.setattr(client._httpx_async_client, 'send', send) await client.send_request( method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False ) - assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX2_TIMEOUT + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index 3e441189..913a5277 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -11,7 +11,7 @@ from unittest.mock import AsyncMock, Mock import brotli -import httpx2 +import httpx2 as httpx import impit import pytest @@ -22,8 +22,8 @@ HttpClient, HttpClientAsync, HttpResponse, - Httpx2HttpClient, - Httpx2HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -267,22 +267,22 @@ async def test_http_client_async_creates_async_impit_client() -> None: await client.aclose() -def test_http_client_creates_sync_httpx2_client() -> None: - """The synchronous HTTPX2 adapter creates the underlying HTTPX2 client, and the close hook closes its pool.""" - client = Httpx2HttpClient(token='test_token_123') +def test_http_client_creates_sync_httpx_client() -> None: + """The synchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" + client = HttpxHttpClient(token='test_token_123') - assert isinstance(client._httpx2_client, httpx2.Client) + assert isinstance(client._httpx_client, httpx.Client) client.close() - assert client._httpx2_client.is_closed + assert client._httpx_client.is_closed -async def test_http_client_async_creates_async_httpx2_client() -> None: - """The asynchronous HTTPX2 adapter creates the underlying HTTPX2 client, and the close hook closes its pool.""" - client = Httpx2HttpClientAsync(token='test_token_123') +async def test_http_client_async_creates_async_httpx_client() -> None: + """The asynchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" + client = HttpxHttpClientAsync(token='test_token_123') - assert isinstance(client._httpx2_async_client, httpx2.AsyncClient) + assert isinstance(client._httpx_async_client, httpx.AsyncClient) await client.aclose() - assert client._httpx2_async_client.is_closed + assert client._httpx_async_client.is_closed def test_parse_params_none() -> None: @@ -416,64 +416,64 @@ async def test_async_http_client_classifies_timeout_errors() -> None: @pytest.mark.parametrize( 'exc', [ - # Even the generic base class is transient: HTTPX2 subclasses it for every failure mode, so an + # Even the generic base class is transient: HTTPX subclasses it for every failure mode, so an # unclassified failure is safer to retry. - pytest.param(httpx2.HTTPError('unclassified failure'), id='bare HTTPError'), - pytest.param(httpx2.TimeoutException('timeout'), id='TimeoutException'), - pytest.param(httpx2.NetworkError('network error'), id='NetworkError'), - pytest.param(httpx2.RemoteProtocolError('remote protocol error'), id='RemoteProtocolError'), - pytest.param(httpx2.DecodingError('decoding error'), id='DecodingError'), + pytest.param(httpx.HTTPError('unclassified failure'), id='bare HTTPError'), + pytest.param(httpx.TimeoutException('timeout'), id='TimeoutException'), + pytest.param(httpx.NetworkError('network error'), id='NetworkError'), + pytest.param(httpx.RemoteProtocolError('remote protocol error'), id='RemoteProtocolError'), + pytest.param(httpx.DecodingError('decoding error'), id='DecodingError'), # One `ProxyError` covers both a proxy rejecting the CONNECT tunnel and a 407, so a transient case cannot # be told from a permanent one - retrying is the safer default. - pytest.param(httpx2.ProxyError('proxy error'), id='ProxyError'), + pytest.param(httpx.ProxyError('proxy error'), id='ProxyError'), ], ) -def test_httpx2_is_retryable_transport_error(exc: Exception) -> None: - """A transient HTTPX2 transport failure is classified as retryable.""" - with Httpx2HttpClient() as client: +def test_httpx_is_retryable_transport_error(exc: Exception) -> None: + """A transient HTTPX transport failure is classified as retryable.""" + with HttpxHttpClient() as client: assert client.is_retryable_transport_error(exc) @pytest.mark.parametrize( 'exc', [ - pytest.param(httpx2.LocalProtocolError('invalid header value'), id='LocalProtocolError'), - pytest.param(httpx2.UnsupportedProtocol('unsupported scheme'), id='UnsupportedProtocol'), - pytest.param(httpx2.TooManyRedirects('too many redirects'), id='TooManyRedirects'), + pytest.param(httpx.LocalProtocolError('invalid header value'), id='LocalProtocolError'), + pytest.param(httpx.UnsupportedProtocol('unsupported scheme'), id='UnsupportedProtocol'), + pytest.param(httpx.TooManyRedirects('too many redirects'), id='TooManyRedirects'), pytest.param( - httpx2.HTTPStatusError( + httpx.HTTPStatusError( 'status error', - request=httpx2.Request('GET', 'https://example.com'), - response=httpx2.Response(500), + request=httpx.Request('GET', 'https://example.com'), + response=httpx.Response(500), ), id='HTTPStatusError', ), - # HTTPX2 reports a bad URL outside the `httpx2.HTTPError` tree entirely. - pytest.param(httpx2.InvalidURL('unsupported scheme'), id='InvalidURL'), + # HTTPX reports a bad URL outside the `httpx.HTTPError` tree entirely. + pytest.param(httpx.InvalidURL('unsupported scheme'), id='InvalidURL'), pytest.param(ValueError('value error'), id='ValueError'), pytest.param(RuntimeError('runtime error'), id='RuntimeError'), pytest.param(Exception('generic exception'), id='Exception'), ], ) -def test_httpx2_is_not_retryable_transport_error(exc: Exception) -> None: - """A transport failure a retry cannot fix, and anything outside HTTPX2's hierarchy, is not retried.""" - with Httpx2HttpClient() as client: +def test_httpx_is_not_retryable_transport_error(exc: Exception) -> None: + """A transport failure a retry cannot fix, and anything outside HTTPX's hierarchy, is not retried.""" + with HttpxHttpClient() as client: assert not client.is_retryable_transport_error(exc) -def test_sync_httpx2_client_classifies_timeout_errors() -> None: - """The built-in synchronous HTTPX2 client exposes transport-neutral timeout classification.""" - with Httpx2HttpClient() as client: +def test_sync_httpx_client_classifies_timeout_errors() -> None: + """The built-in synchronous HTTPX client exposes transport-neutral timeout classification.""" + with HttpxHttpClient() as client: assert client.is_timeout_error(TimeoutError('test')) - assert client.is_timeout_error(httpx2.TimeoutException('test')) + assert client.is_timeout_error(httpx.TimeoutException('test')) assert not client.is_timeout_error(ValueError('test')) -async def test_async_httpx2_client_classifies_timeout_errors() -> None: - """The built-in asynchronous HTTPX2 client exposes transport-neutral timeout classification.""" - async with Httpx2HttpClientAsync() as client: +async def test_async_httpx_client_classifies_timeout_errors() -> None: + """The built-in asynchronous HTTPX client exposes transport-neutral timeout classification.""" + async with HttpxHttpClientAsync() as client: assert client.is_timeout_error(TimeoutError('test')) - assert client.is_timeout_error(httpx2.TimeoutException('test')) + assert client.is_timeout_error(httpx.TimeoutException('test')) assert not client.is_timeout_error(ValueError('test')) @@ -514,52 +514,52 @@ def test_transient_transport_error_is_retried() -> None: assert request.call_count == 3 -def test_httpx2_permanent_transport_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None: - """The HTTPX2 adapter feeds the same fail-fast classification into the shared pipeline.""" - with Httpx2HttpClient(token='test_token', min_delay_between_retries=timedelta(0)) as client: - send = Mock(side_effect=httpx2.UnsupportedProtocol('unsupported scheme')) - monkeypatch.setattr(client._httpx2_client, 'send', send) +def test_httpx_permanent_transport_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX adapter feeds the same fail-fast classification into the shared pipeline.""" + with HttpxHttpClient(token='test_token', min_delay_between_retries=timedelta(0)) as client: + send = Mock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) + monkeypatch.setattr(client._httpx_client, 'send', send) - with pytest.raises(httpx2.UnsupportedProtocol): + with pytest.raises(httpx.UnsupportedProtocol): client.call(method='GET', url='https://api.test.com/endpoint') send.assert_called_once() -async def test_httpx2_permanent_transport_error_is_not_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: - """The asynchronous HTTPX2 adapter applies the same policy, failing on the first attempt.""" - async with Httpx2HttpClientAsync(token='test_token', min_delay_between_retries=timedelta(0)) as client: - send = AsyncMock(side_effect=httpx2.UnsupportedProtocol('unsupported scheme')) - monkeypatch.setattr(client._httpx2_async_client, 'send', send) +async def test_httpx_permanent_transport_error_is_not_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX adapter applies the same policy, failing on the first attempt.""" + async with HttpxHttpClientAsync(token='test_token', min_delay_between_retries=timedelta(0)) as client: + send = AsyncMock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) + monkeypatch.setattr(client._httpx_async_client, 'send', send) - with pytest.raises(httpx2.UnsupportedProtocol): + with pytest.raises(httpx.UnsupportedProtocol): await client.call(method='GET', url='https://api.test.com/endpoint') send.assert_awaited_once() -def test_httpx2_transient_transport_error_is_retried(monkeypatch: pytest.MonkeyPatch) -> None: - """The HTTPX2 adapter keeps a transient transport failure inside the shared retry loop.""" - with Httpx2HttpClient(token='test_token', max_retries=2, min_delay_between_retries=timedelta(0)) as client: - send = Mock(side_effect=httpx2.TimeoutException('timeout')) - monkeypatch.setattr(client._httpx2_client, 'send', send) +def test_httpx_transient_transport_error_is_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX adapter keeps a transient transport failure inside the shared retry loop.""" + with HttpxHttpClient(token='test_token', max_retries=2, min_delay_between_retries=timedelta(0)) as client: + send = Mock(side_effect=httpx.TimeoutException('timeout')) + monkeypatch.setattr(client._httpx_client, 'send', send) - with pytest.raises(httpx2.TimeoutException): + with pytest.raises(httpx.TimeoutException): client.call(method='GET', url='https://api.test.com/endpoint') # `max_retries` attempts inside the backoff loop, plus the final one it makes after the last delay. assert send.call_count == 3 -async def test_httpx2_transient_transport_error_is_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: - """The asynchronous HTTPX2 adapter keeps a transient transport failure inside the shared retry loop too.""" - async with Httpx2HttpClientAsync( +async def test_httpx_transient_transport_error_is_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX adapter keeps a transient transport failure inside the shared retry loop too.""" + async with HttpxHttpClientAsync( token='test_token', max_retries=2, min_delay_between_retries=timedelta(0) ) as client: - send = AsyncMock(side_effect=httpx2.TimeoutException('timeout')) - monkeypatch.setattr(client._httpx2_async_client, 'send', send) + send = AsyncMock(side_effect=httpx.TimeoutException('timeout')) + monkeypatch.setattr(client._httpx_async_client, 'send', send) - with pytest.raises(httpx2.TimeoutException): + with pytest.raises(httpx.TimeoutException): await client.call(method='GET', url='https://api.test.com/endpoint') # `max_retries` attempts inside the backoff loop, plus the final one it makes after the last delay. diff --git a/tests/unit/test_pluggable_http_client.py b/tests/unit/test_pluggable_http_client.py index 37725f19..bda71bc7 100644 --- a/tests/unit/test_pluggable_http_client.py +++ b/tests/unit/test_pluggable_http_client.py @@ -356,8 +356,8 @@ def test_public_exports() -> None: 'HttpClient', 'HttpClientAsync', 'HttpResponse', - 'Httpx2HttpClient', - 'Httpx2HttpClientAsync', + 'HttpxHttpClient', + 'HttpxHttpClientAsync', 'ImpitHttpClient', 'ImpitHttpClientAsync', ): @@ -367,19 +367,19 @@ def test_public_exports() -> None: assert not hasattr(http_clients_module, 'HttpClientBase') -def test_httpx2_clients_raise_clear_error_when_extra_missing() -> None: - """Missing HTTPX2 keeps normal and star imports usable while explicit HTTPX2 access raises a clear error.""" +def test_httpx_clients_raise_clear_error_when_extra_missing() -> None: + """Missing HTTPX keeps normal and star imports usable while explicit HTTPX access raises a clear error.""" script = dedent( """ import sys - class BlockHttpx2: + class BlockHttpx: def find_spec(self, name, *_args): if name == 'httpx2' or name.startswith('httpx2.'): raise ModuleNotFoundError(f"No module named '{name}'", name='httpx2') return None - sys.meta_path.insert(0, BlockHttpx2()) + sys.meta_path.insert(0, BlockHttpx()) import apify_client.http_clients as module assert module.HttpClient is not None @@ -388,9 +388,9 @@ def find_spec(self, name, *_args): namespace = {} exec('from apify_client.http_clients import *', namespace) assert namespace['HttpClient'] is module.HttpClient - assert 'Httpx2HttpClient' not in namespace + assert 'HttpxHttpClient' not in namespace - for name in ('Httpx2HttpClient', 'Httpx2HttpClientAsync'): + for name in ('HttpxHttpClient', 'HttpxHttpClientAsync'): try: getattr(module, name) except ImportError as exc: diff --git a/uv.lock b/uv.lock index 94d84184..09dbabd2 100644 --- a/uv.lock +++ b/uv.lock @@ -84,7 +84,7 @@ dev = [ requires-dist = [ { name = "brotli", marker = "extra == 'brotli'", specifier = ">=1.0.9" }, { name = "colorama", specifier = ">=0.4.0" }, - { name = "httpx2", marker = "extra == 'httpx2'", specifier = ">=2.0.0,<3.0.0" }, + { name = "httpx2", marker = "extra == 'httpx2'", specifier = ">=2.0.0" }, { name = "impit", specifier = "~=0.13.0" }, { name = "more-itertools", specifier = ">=10.0.0" }, { name = "pydantic", extras = ["email"], specifier = ">=2.11.0" },