From 8457be280c4f051e974a924bbeeadef60b79d42d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 13:33:30 +0200 Subject: [PATCH 1/8] fix: Stop dropping user_data and other fields when adding queue requests `RequestDraft` declared only id/unique_key/url/method and leaned on `extra='allow'` for the rest, but `alias_generator=to_camel` never touches extras. So `user_data`, `no_retry`, `headers`, `payload`, `retry_count`, and `handled_at` reached the API snake_cased on `add_request` and `batch_add_requests`, which the API ignores silently. `update_request` was unaffected - it uses `Request`, which declares the full shape. The spec declares those request bodies as `RequestBase` and uses `RequestDraft` only for `unprocessedRequests` in responses, so the codegen postprocessor now reparents `RequestDraft` onto `RequestBase`. `RequestDraftDict.unique_key` and `.url` are no longer statically required: PEP 589 forbids a TypedDict subclass from redeclaring a key of its base. Pydantic still enforces both at runtime, before any HTTP call. --- scripts/postprocess_generated_models.py | 84 +++++++++++++- src/apify_client/_models.py | 2 +- src/apify_client/_typeddicts.py | 22 +--- tests/unit/test_client_request_queue.py | 107 +++++++++++++++++- .../unit/test_postprocess_generated_models.py | 74 +++++++++++- 5 files changed, 261 insertions(+), 28 deletions(-) diff --git a/scripts/postprocess_generated_models.py b/scripts/postprocess_generated_models.py index 50a51f56..5aa33c86 100644 --- a/scripts/postprocess_generated_models.py +++ b/scripts/postprocess_generated_models.py @@ -1,5 +1,9 @@ """Post-process datamodel-codegen output to fix known issues and prune the TypedDict file. +Applied to both `_models.py` and `_typeddicts.py`: +- Reparent classes whose spec schema declares the wire shape standalone instead of extending the base schema it + duplicates, so the generated class declares the full set of fields. + Applied to `_models.py`: - Fix discriminator field names that use camelCase instead of snake_case (known issue with discriminators on schemas referenced from array items). @@ -12,6 +16,7 @@ - Add `@docs_group('Models')` to every model class (plus the required import). Applied to `_typeddicts.py`: +- Drop the fields a reparented TypedDict inherits from its new base, which PEP 589 forbids it from redeclaring. - Keep only the TypedDicts actually used as resource-client method inputs (plus their transitive dependencies). The file is generated in full by datamodel-codegen; the trimming happens here. - Rename every kept class to add a `Dict` suffix so it doesn't clash with the Pydantic model name @@ -47,6 +52,16 @@ 'pricingModel': 'pricing_model', } +# Map of `{class name: base class it should inherit from}`, applied to both generated files. +# Some request-body schemas in the spec spell out only a few properties instead of extending the base schema that +# carries the rest of the wire shape. A model generated from such a schema declares those few fields and lets +# `extra='allow'` absorb everything else - but the `to_camel` alias generator only covers declared fields, so extras +# reach the API under their snake_case names, which it silently ignores. Reparenting to the base schema's class +# declares the full shape, restoring alias coverage and the TypedDict keys the type checker validates against. +BASE_CLASS_FIXES: dict[str, str] = { + 'RequestDraft': 'RequestBase', +} + # TypedDicts accepted as inputs by resource-client methods. These are the roots of the reachability # walk over `_typeddicts.py`: anything not reachable from here (directly or transitively) # is dropped so only the TypedDicts that are part of the public input surface — plus their nested @@ -102,6 +117,68 @@ def _base_names(node: ast.ClassDef) -> set[str]: return {b.id for b in node.bases if isinstance(b, ast.Name)} +def reparent_classes(content: str) -> str: + """Replace the base class of every `BASE_CLASS_FIXES` entry with the mapped one. + + The whole base list is rewritten, so re-running on already-reparented source is a no-op. Entries absent from + `content` are simply skipped - the map is shared by both generated files, and a schema does not always yield a + class in each (a root model becomes a `TypeAlias` in `_typeddicts.py`). + """ + for name, base in BASE_CLASS_FIXES.items(): + content = re.sub( + rf'^class {re.escape(name)}\([^)]*\):', + f'class {name}({base}):', + content, + flags=re.MULTILINE, + ) + return content + + +def _annotated_field_names(node: ast.ClassDef) -> set[str]: + """Return the names of every annotated field declared directly in `node`'s body.""" + return { + stmt.target.id for stmt in node.body if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name) + } + + +def drop_inherited_typeddict_fields(content: str) -> str: + """Delete the fields a reparented TypedDict now inherits, keeping only the ones its base doesn't declare. + + PEP 589 forbids a TypedDict subclass from redeclaring a key of its base - even to turn a `NotRequired` key into + a required one - so the reparented class has to give up its own copies. The keys stay required at runtime: the + Pydantic model keeps its redeclarations, which Pydantic allows. + + Each field's trailing description docstring is removed along with it. + """ + tree = ast.parse(content) + classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} + drop_line_indices: set[int] = set() + + for name, base in BASE_CLASS_FIXES.items(): + node, base_node = classes.get(name), classes.get(base) + if node is None or base_node is None: + continue + inherited = _annotated_field_names(base_node) + for index, stmt in enumerate(node.body): + if not isinstance(stmt, ast.AnnAssign) or not isinstance(stmt.target, ast.Name): + continue + if stmt.target.id not in inherited: + continue + end_line = stmt.end_lineno + following = node.body[index + 1] if index + 1 < len(node.body) else None + if following is not None and _is_string_expr(following): + end_line = following.end_lineno + assert end_line is not None # noqa: S101 + drop_line_indices.update(range(stmt.lineno - 1, end_line)) + + if not drop_line_indices: + return content + + lines = content.split('\n') + kept = [line for i, line in enumerate(lines) if i not in drop_line_indices] + return _collapse_blank_lines('\n'.join(kept)) + + def fix_discriminators(content: str) -> str: """Replace camelCase discriminator values with their snake_case equivalents.""" for camel, snake in DISCRIMINATOR_FIXES.items(): @@ -645,7 +722,8 @@ def postprocess_models(models_path: Path, literals_path: Path) -> list[Path]: Returns the list of paths that were (re)written. """ original = models_path.read_text() - fixed = fix_discriminators(original) + fixed = reparent_classes(original) + fixed = fix_discriminators(fixed) fixed = absolutize_doc_links(fixed) fixed = convert_enums_to_literals(fixed) fixed = add_docs_group_decorators(fixed, 'Models') @@ -666,7 +744,9 @@ def postprocess_models(models_path: Path, literals_path: Path) -> list[Path]: def postprocess_typeddicts(path: Path, alias_map: dict[str, dict[str, str]]) -> bool: """Apply `_typeddicts.py`-specific fixes. Returns True if the file changed.""" original = path.read_text() - pruned, kept = prune_typeddicts(original, RESOURCE_INPUT_TYPEDDICTS) + # Reparenting comes first so the new base class counts as a dependency of the input surface and survives pruning. + reparented = drop_inherited_typeddict_fields(reparent_classes(original)) + pruned, kept = prune_typeddicts(reparented, RESOURCE_INPUT_TYPEDDICTS) renamed = rename_with_dict_suffix(pruned, kept) flattened = flatten_empty_typeddicts(renamed) camelized = add_camel_case_typeddicts(flattened, alias_map) diff --git a/src/apify_client/_models.py b/src/apify_client/_models.py index 7e39f283..a1de1d3c 100644 --- a/src/apify_client/_models.py +++ b/src/apify_client/_models.py @@ -2486,7 +2486,7 @@ class Request(RequestBase): @docs_group('Models') -class RequestDraft(BaseModel): +class RequestDraft(RequestBase): """A request that failed to be processed during a request queue operation and can be retried.""" model_config = ConfigDict( diff --git a/src/apify_client/_typeddicts.py b/src/apify_client/_typeddicts.py index adbad385..864b9a1b 100644 --- a/src/apify_client/_typeddicts.py +++ b/src/apify_client/_typeddicts.py @@ -112,41 +112,23 @@ class RequestCamelDict(RequestBaseCamelDict): @docs_group('Typed dicts') -class RequestDraftDict(TypedDict): +class RequestDraftDict(RequestBaseDict): """A request that failed to be processed during a request queue operation and can be retried.""" id: NotRequired[str] """ A unique identifier assigned to the request. """ - unique_key: str - """ - A unique key used for request de-duplication. Requests with the same unique key are considered identical. - """ - url: str - """ - The URL of the request. - """ - method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']] @docs_group('Typed dicts') -class RequestDraftCamelDict(TypedDict): +class RequestDraftCamelDict(RequestBaseCamelDict): """A request that failed to be processed during a request queue operation and can be retried.""" id: NotRequired[str] """ A unique identifier assigned to the request. """ - uniqueKey: str - """ - A unique key used for request de-duplication. Requests with the same unique key are considered identical. - """ - url: str - """ - The URL of the request. - """ - method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']] @docs_group('Typed dicts') diff --git a/tests/unit/test_client_request_queue.py b/tests/unit/test_client_request_queue.py index 8c34b025..e0a48fc0 100644 --- a/tests/unit/test_client_request_queue.py +++ b/tests/unit/test_client_request_queue.py @@ -17,7 +17,7 @@ from pytest_httpserver import HTTPServer from werkzeug.wrappers import Request - from apify_client._typeddicts import RequestDraftDict + from apify_client._typeddicts import RequestDict, RequestDraftDict # The Apify API limit on the payload size of a batch-add request, which the client's batching must respect. _API_MAX_PAYLOAD_SIZE_BYTES = 9 * 1024 * 1024 @@ -119,8 +119,11 @@ def _make_large_requests() -> list[RequestDraftDict]: ] -def _payload_capturing_handler(payloads: list[bytes]) -> Callable[[Request], Response]: - """Return a handler that records each POST body and responds with an empty batch result. +def _payload_capturing_handler( + payloads: list[bytes], + response_content: str = _EMPTY_BATCH_RESPONSE_CONTENT, +) -> Callable[[Request], Response]: + """Return a handler that records each request body and responds with `response_content`. Bodies below the client's compression threshold arrive uncompressed, so the recorded payload is decompressed only when the request says it was encoded. @@ -129,7 +132,7 @@ def _payload_capturing_handler(payloads: list[bytes]) -> Callable[[Request], Res def handler(request: Request) -> Response: body = request.get_data() payloads.append(gzip.decompress(body) if request.headers.get('Content-Encoding') == 'gzip' else body) - return Response(_EMPTY_BATCH_RESPONSE_CONTENT, status=200, content_type='application/json') + return Response(response_content, status=200, content_type='application/json') return handler @@ -253,3 +256,99 @@ def test_batch_processed_partially_sync(httpserver: HTTPServer) -> None: assert requests[0]['unique_key'] in {request.unique_key for request in batch_response.processed_requests} assert len(batch_response.unprocessed_requests) == 1 assert batch_response.unprocessed_requests[0].unique_key == requests[1]['unique_key'] + + +_REQUEST_REGISTRATION_RESPONSE_CONTENT = ( + '{"data": {"requestId": "YiKoxjkaS9gjGTqhF", "wasAlreadyPresent": false, "wasAlreadyHandled": false}}' +) + +_FULL_REQUEST_DICT: RequestDict = { + 'id': 'YiKoxjkaS9gjGTqhF', + 'unique_key': 'http://example.com/1', + 'url': 'http://example.com/1', + 'method': 'GET', + 'user_data': {'label': 'DETAIL'}, + 'no_retry': True, + 'retry_count': 2, + 'loaded_url': 'http://example.com/1/final', + 'headers': {'X-Test': 'value'}, + 'payload': 'body', + 'error_messages': ['boom'], + 'handled_at': '2019-06-16T10:23:31.607Z', +} + + +async def test_add_request_matches_update_request_casing_async(httpserver: HTTPServer) -> None: + """The same snake_case request dict reaches the API camelCased on add_request just as on update_request.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*')).respond_with_handler( + _payload_capturing_handler(payloads, _REQUEST_REGISTRATION_RESPONSE_CONTENT) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + await rq_client.add_request(_FULL_REQUEST_DICT) + await rq_client.update_request(_FULL_REQUEST_DICT) + + added, updated = (json.loads(payload) for payload in payloads) + assert added == updated + assert added['userData'] == {'label': 'DETAIL'} + assert [key for key in added if '_' in key] == [] + + +def test_add_request_matches_update_request_casing_sync(httpserver: HTTPServer) -> None: + """The same snake_case request dict reaches the API camelCased on add_request just as on update_request.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*')).respond_with_handler( + _payload_capturing_handler(payloads, _REQUEST_REGISTRATION_RESPONSE_CONTENT) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + rq_client.add_request(_FULL_REQUEST_DICT) + rq_client.update_request(_FULL_REQUEST_DICT) + + added, updated = (json.loads(payload) for payload in payloads) + assert added == updated + assert added['userData'] == {'label': 'DETAIL'} + assert [key for key in added if '_' in key] == [] + + +async def test_batch_add_requests_camel_cases_every_field_async(httpserver: HTTPServer) -> None: + """Every field of a snake_case request dict is camelCased in the batch-add payload.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + await rq_client.batch_add_requests(requests=[_FULL_REQUEST_DICT]) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request['userData'] == {'label': 'DETAIL'} + assert [key for key in sent_request if '_' in key] == [] + + +def test_batch_add_requests_camel_cases_every_field_sync(httpserver: HTTPServer) -> None: + """Every field of a snake_case request dict is camelCased in the batch-add payload.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + rq_client.batch_add_requests(requests=[_FULL_REQUEST_DICT]) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request['userData'] == {'label': 'DETAIL'} + assert [key for key in sent_request if '_' in key] == [] diff --git a/tests/unit/test_postprocess_generated_models.py b/tests/unit/test_postprocess_generated_models.py index c652ba5a..36334452 100644 --- a/tests/unit/test_postprocess_generated_models.py +++ b/tests/unit/test_postprocess_generated_models.py @@ -7,11 +7,13 @@ add_docs_group_decorators, build_alias_map, convert_enums_to_literals, + drop_inherited_typeddict_fields, fix_discriminators, + reparent_classes, split_literals_to_file, ) -from apify_client._models import Request +from apify_client._models import Request, RequestBase, RequestDraft # -- fix_discriminators ------------------------------------------------------- @@ -678,3 +680,73 @@ def test_add_camel_case_typeddicts_camel_validates_with_pydantic() -> None: 'user_data': {'tag': 'x'}, } assert Request.model_validate(camel_payload) == Request.model_validate(snake_payload) + + +def test_reparent_classes_swaps_base_class() -> None: + """A mapped class gets the configured base in place of the generated one.""" + content = 'class RequestDraft(BaseModel):\n url: str\n' + result = reparent_classes(content) + assert result == 'class RequestDraft(RequestBase):\n url: str\n' + + +def test_reparent_classes_is_idempotent() -> None: + """Re-running on already-reparented source leaves it unchanged.""" + content = 'class RequestDraft(RequestBase):\n url: str\n' + assert reparent_classes(content) == content + + +def test_reparent_classes_skips_absent_class() -> None: + """Source without any mapped class passes through unchanged.""" + content = 'class Foo(BaseModel):\n name: str\n' + assert reparent_classes(content) == content + + +def test_reparent_classes_leaves_name_prefixed_classes() -> None: + """A class whose name merely starts with a mapped name keeps its own base.""" + content = 'class RequestDraft(TypedDict):\n url: str\n\n\nclass RequestDraftDelete(TypedDict):\n id: str\n' + result = reparent_classes(content) + assert 'class RequestDraftDelete(TypedDict):' in result + + +def test_drop_inherited_typeddict_fields_removes_shadowed_keys() -> None: + """A reparented TypedDict keeps only the keys its new base does not declare, docstrings included.""" + content = textwrap.dedent("""\ + class RequestBase(TypedDict): + unique_key: NotRequired[str] + url: NotRequired[str] + + class RequestDraft(RequestBase): + id: NotRequired[str] + \""" + A unique identifier. + \""" + unique_key: str + \""" + A unique key. + \""" + url: str + """) + result = drop_inherited_typeddict_fields(content) + assert ' id: NotRequired[str]' in result + assert 'A unique identifier.' in result + assert ' unique_key: str' not in result + assert 'A unique key.' not in result + assert ' url: str' not in result + + +def test_drop_inherited_typeddict_fields_leaves_unreparented_source() -> None: + """Source whose mapped class has no base to inherit from passes through unchanged.""" + content = 'class RequestDraft(TypedDict):\n unique_key: str\n' + assert drop_inherited_typeddict_fields(content) == content + + +def test_reparented_draft_serializes_camel_case() -> None: + """`RequestDraft` inherits the full wire shape, so every field it accepts is aliased on serialization.""" + payload = {'unique_key': 'GET|abc', 'url': 'https://example.com', 'user_data': {'tag': 'x'}, 'no_retry': True} + assert issubclass(RequestDraft, RequestBase) + assert RequestDraft.model_validate(payload).model_dump(by_alias=True, exclude_none=True) == { + 'uniqueKey': 'GET|abc', + 'url': 'https://example.com', + 'userData': {'tag': 'x'}, + 'noRetry': True, + } From fa77d88d45056a880e7e9f05baa4148faf51c85d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 14:51:24 +0200 Subject: [PATCH 2/8] fix: Serialize request queue request bodies in JSON mode --- .../_resource_clients/request_queue.py | 14 +- tests/unit/test_client_request_queue.py | 134 ++++++++++++++++++ 2 files changed, 141 insertions(+), 7 deletions(-) diff --git a/src/apify_client/_resource_clients/request_queue.py b/src/apify_client/_resource_clients/request_queue.py index a9539bfb..59527db8 100644 --- a/src/apify_client/_resource_clients/request_queue.py +++ b/src/apify_client/_resource_clients/request_queue.py @@ -81,7 +81,7 @@ def _serialize_requests( return [ json.dumps( (request if isinstance(request, RequestDraft) else RequestDraft.model_validate(request)).model_dump( - by_alias=True, exclude_none=True + mode='json', by_alias=True, exclude_none=True, fallback=str ), ensure_ascii=False, allow_nan=False, @@ -253,7 +253,7 @@ def add_request( response = self._http_client.call( url=self._build_url('requests'), method='POST', - json=request.model_dump(by_alias=True, exclude_none=True), + json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str), params=request_params, timeout=timeout, ) @@ -315,7 +315,7 @@ def update_request( response = self._http_client.call( url=self._build_url(f'requests/{request.id}'), method='PUT', - json=request.model_dump(by_alias=True, exclude_none=True), + json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str), params=request_params, timeout=timeout, ) @@ -504,7 +504,7 @@ def batch_delete_requests( else RequestDraftDelete.model_validate( request, ) - ).model_dump(by_alias=True, exclude_none=True) + ).root.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str) for request in requests ] @@ -780,7 +780,7 @@ async def add_request( response = await self._http_client.call( url=self._build_url('requests'), method='POST', - json=request.model_dump(by_alias=True, exclude_none=True), + json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str), params=request_params, timeout=timeout, ) @@ -840,7 +840,7 @@ async def update_request( response = await self._http_client.call( url=self._build_url(f'requests/{request.id}'), method='PUT', - json=request.model_dump(by_alias=True, exclude_none=True), + json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str), params=request_params, timeout=timeout, ) @@ -1082,7 +1082,7 @@ async def batch_delete_requests( else RequestDraftDelete.model_validate( request, ) - ).model_dump(by_alias=True, exclude_none=True) + ).root.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str) for request in requests ] diff --git a/tests/unit/test_client_request_queue.py b/tests/unit/test_client_request_queue.py index e0a48fc0..9ef4056f 100644 --- a/tests/unit/test_client_request_queue.py +++ b/tests/unit/test_client_request_queue.py @@ -9,6 +9,7 @@ from werkzeug.wrappers import Response from apify_client import ApifyClient, ApifyClientAsync +from apify_client._models import RequestDraftDelete from apify_client.errors import ApifyApiError if TYPE_CHECKING: @@ -278,6 +279,13 @@ def test_batch_processed_partially_sync(httpserver: HTTPServer) -> None: } +class Unserializable: + """A `user_data` value of a type no JSON serializer can encode without a stringification fallback.""" + + def __str__(self) -> str: + return 'unserializable' + + async def test_add_request_matches_update_request_casing_async(httpserver: HTTPServer) -> None: """The same snake_case request dict reaches the API camelCased on add_request just as on update_request.""" server_url = httpserver.url_for('/').removesuffix('/') @@ -296,6 +304,7 @@ async def test_add_request_matches_update_request_casing_async(httpserver: HTTPS assert added == updated assert added['userData'] == {'label': 'DETAIL'} assert [key for key in added if '_' in key] == [] + assert added['handledAt'] == '2019-06-16T10:23:31.607000Z' def test_add_request_matches_update_request_casing_sync(httpserver: HTTPServer) -> None: @@ -316,6 +325,7 @@ def test_add_request_matches_update_request_casing_sync(httpserver: HTTPServer) assert added == updated assert added['userData'] == {'label': 'DETAIL'} assert [key for key in added if '_' in key] == [] + assert added['handledAt'] == '2019-06-16T10:23:31.607000Z' async def test_batch_add_requests_camel_cases_every_field_async(httpserver: HTTPServer) -> None: @@ -334,6 +344,7 @@ async def test_batch_add_requests_camel_cases_every_field_async(httpserver: HTTP (sent_request,) = json.loads(payloads[0]) assert sent_request['userData'] == {'label': 'DETAIL'} assert [key for key in sent_request if '_' in key] == [] + assert sent_request['handledAt'] == '2019-06-16T10:23:31.607000Z' def test_batch_add_requests_camel_cases_every_field_sync(httpserver: HTTPServer) -> None: @@ -352,3 +363,126 @@ def test_batch_add_requests_camel_cases_every_field_sync(httpserver: HTTPServer) (sent_request,) = json.loads(payloads[0]) assert sent_request['userData'] == {'label': 'DETAIL'} assert [key for key in sent_request if '_' in key] == [] + assert sent_request['handledAt'] == '2019-06-16T10:23:31.607000Z' + + +async def test_add_request_stringifies_unserializable_user_data_async(httpserver: HTTPServer) -> None: + """A `user_data` value JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*')).respond_with_handler( + _payload_capturing_handler(payloads, _REQUEST_REGISTRATION_RESPONSE_CONTENT) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + await rq_client.add_request( + {'unique_key': 'http://example.com/1', 'url': 'http://example.com/1', 'user_data': {'tag': Unserializable()}} + ) + + assert json.loads(payloads[0])['userData'] == {'tag': 'unserializable'} + + +def test_add_request_stringifies_unserializable_user_data_sync(httpserver: HTTPServer) -> None: + """A `user_data` value JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*')).respond_with_handler( + _payload_capturing_handler(payloads, _REQUEST_REGISTRATION_RESPONSE_CONTENT) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + rq_client.add_request( + {'unique_key': 'http://example.com/1', 'url': 'http://example.com/1', 'user_data': {'tag': Unserializable()}} + ) + + assert json.loads(payloads[0])['userData'] == {'tag': 'unserializable'} + + +async def test_batch_add_requests_stringifies_unserializable_user_data_async(httpserver: HTTPServer) -> None: + """A `user_data` value JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + await rq_client.batch_add_requests( + requests=[ + { + 'unique_key': 'http://example.com/1', + 'url': 'http://example.com/1', + 'user_data': {'tag': Unserializable()}, + } + ] + ) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request['userData'] == {'tag': 'unserializable'} + + +def test_batch_add_requests_stringifies_unserializable_user_data_sync(httpserver: HTTPServer) -> None: + """A `user_data` value JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + rq_client.batch_add_requests( + requests=[ + { + 'unique_key': 'http://example.com/1', + 'url': 'http://example.com/1', + 'user_data': {'tag': Unserializable()}, + } + ] + ) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request['userData'] == {'tag': 'unserializable'} + + +async def test_batch_delete_requests_stringifies_unserializable_extra_async(httpserver: HTTPServer) -> None: + """An extra field JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='DELETE').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + request = RequestDraftDelete.model_validate({'id': 'YiKoxjkaS9gjGTqhF', 'weird': Unserializable()}) + await rq_client.batch_delete_requests(requests=[request]) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request == {'id': 'YiKoxjkaS9gjGTqhF', 'weird': 'unserializable'} + + +def test_batch_delete_requests_stringifies_unserializable_extra_sync(httpserver: HTTPServer) -> None: + """An extra field JSON cannot represent is stringified instead of aborting the call.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + payloads = list[bytes]() + httpserver.expect_request(re.compile(r'.*'), method='DELETE').respond_with_handler( + _payload_capturing_handler(payloads) + ) + rq_client = client.request_queue(request_queue_id='whatever') + + request = RequestDraftDelete.model_validate({'id': 'YiKoxjkaS9gjGTqhF', 'weird': Unserializable()}) + rq_client.batch_delete_requests(requests=[request]) + + (sent_request,) = json.loads(payloads[0]) + assert sent_request == {'id': 'YiKoxjkaS9gjGTqhF', 'weird': 'unserializable'} From 808c90e411418e52dbd73f7643984658fe44d6c9 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 18:01:26 +0200 Subject: [PATCH 3/8] test: Cover the reparented request draft shape and its required keys --- tests/unit/test_client_request_queue.py | 27 +++++++++++++++++++ .../unit/test_postprocess_generated_models.py | 15 +++++++++++ 2 files changed, 42 insertions(+) diff --git a/tests/unit/test_client_request_queue.py b/tests/unit/test_client_request_queue.py index 9ef4056f..6cc1eae4 100644 --- a/tests/unit/test_client_request_queue.py +++ b/tests/unit/test_client_request_queue.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING import pytest +from pydantic import ValidationError from werkzeug.wrappers import Response from apify_client import ApifyClient, ApifyClientAsync @@ -486,3 +487,29 @@ def test_batch_delete_requests_stringifies_unserializable_extra_sync(httpserver: (sent_request,) = json.loads(payloads[0]) assert sent_request == {'id': 'YiKoxjkaS9gjGTqhF', 'weird': 'unserializable'} + + +async def test_add_request_requires_unique_key_and_url_async(httpserver: HTTPServer) -> None: + """`RequestDraftDict` marks no key required, so Pydantic is the only guard left on `unique_key` and `url`.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + rq_client = client.request_queue(request_queue_id='whatever') + + with pytest.raises(ValidationError) as exc_info: + await rq_client.add_request({'method': 'GET'}) + + assert {error['loc'][0] for error in exc_info.value.errors()} == {'uniqueKey', 'url'} + assert httpserver.log == [] + + +def test_add_request_requires_unique_key_and_url_sync(httpserver: HTTPServer) -> None: + """`RequestDraftDict` marks no key required, so Pydantic is the only guard left on `unique_key` and `url`.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + rq_client = client.request_queue(request_queue_id='whatever') + + with pytest.raises(ValidationError) as exc_info: + rq_client.add_request({'method': 'GET'}) + + assert {error['loc'][0] for error in exc_info.value.errors()} == {'uniqueKey', 'url'} + assert httpserver.log == [] diff --git a/tests/unit/test_postprocess_generated_models.py b/tests/unit/test_postprocess_generated_models.py index 36334452..14c0e4ed 100644 --- a/tests/unit/test_postprocess_generated_models.py +++ b/tests/unit/test_postprocess_generated_models.py @@ -14,6 +14,12 @@ ) from apify_client._models import Request, RequestBase, RequestDraft +from apify_client._typeddicts import ( + RequestBaseCamelDict, + RequestBaseDict, + RequestDraftCamelDict, + RequestDraftDict, +) # -- fix_discriminators ------------------------------------------------------- @@ -705,6 +711,7 @@ def test_reparent_classes_leaves_name_prefixed_classes() -> None: """A class whose name merely starts with a mapped name keeps its own base.""" content = 'class RequestDraft(TypedDict):\n url: str\n\n\nclass RequestDraftDelete(TypedDict):\n id: str\n' result = reparent_classes(content) + assert 'class RequestDraft(RequestBase):' in result assert 'class RequestDraftDelete(TypedDict):' in result @@ -740,6 +747,14 @@ def test_drop_inherited_typeddict_fields_leaves_unreparented_source() -> None: assert drop_inherited_typeddict_fields(content) == content +def test_reparented_draft_typeddicts_declare_full_shape() -> None: + """The reparented request TypedDicts expose every `RequestBase` key, snake_case and camelCase alike.""" + assert set(RequestBaseDict.__annotations__) < set(RequestDraftDict.__annotations__) + assert set(RequestBaseCamelDict.__annotations__) < set(RequestDraftCamelDict.__annotations__) + assert 'user_data' in RequestDraftDict.__annotations__ + assert 'userData' in RequestDraftCamelDict.__annotations__ + + def test_reparented_draft_serializes_camel_case() -> None: """`RequestDraft` inherits the full wire shape, so every field it accepts is aliased on serialization.""" payload = {'unique_key': 'GET|abc', 'url': 'https://example.com', 'user_data': {'tag': 'x'}, 'no_retry': True} From 6c033915e9076a3b985cc1c230f5ed72132b2e77 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 18:01:46 +0200 Subject: [PATCH 4/8] docs: Note that added queue requests must carry unique_key and url --- src/apify_client/_resource_clients/request_queue.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/apify_client/_resource_clients/request_queue.py b/src/apify_client/_resource_clients/request_queue.py index 59527db8..21099e65 100644 --- a/src/apify_client/_resource_clients/request_queue.py +++ b/src/apify_client/_resource_clients/request_queue.py @@ -238,7 +238,7 @@ def add_request( https://docs.apify.com/api/v2#/reference/request-queues/request-collection/add-request Args: - request: The request to add to the queue. + request: The request to add to the queue. Must carry a `unique_key` and a `url`. forefront: Whether to add the request to the head or the end of the queue. timeout: Timeout for the API HTTP request. @@ -417,7 +417,7 @@ def batch_add_requests( https://docs.apify.com/api/v2#/reference/request-queues/batch-request-operations/add-requests Args: - requests: List of requests to be added to the queue. + requests: List of requests to be added to the queue. Each must carry a `unique_key` and a `url`. forefront: Whether to add requests to the front of the queue. max_parallel: Specifies the maximum number of parallel tasks for API calls. This is only applicable to the async client. For the sync client, this value must be set to 1, as parallel execution @@ -765,7 +765,7 @@ async def add_request( https://docs.apify.com/api/v2#/reference/request-queues/request-collection/add-request Args: - request: The request to add to the queue. + request: The request to add to the queue. Must carry a `unique_key` and a `url`. forefront: Whether to add the request to the head or the end of the queue. timeout: Timeout for the API HTTP request. @@ -990,7 +990,7 @@ async def batch_add_requests( https://docs.apify.com/api/v2#/reference/request-queues/batch-request-operations/add-requests Args: - requests: List of requests to be added to the queue. + requests: List of requests to be added to the queue. Each must carry a `unique_key` and a `url`. forefront: Whether to add requests to the front of the queue. max_parallel: Specifies the maximum number of parallel tasks for API calls. This is only applicable to the async client. For the sync client, this value must be set to 1, as parallel execution From e7c92bf6ded42633b9bbeb20a70e7d629ecf2dc3 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 18:01:58 +0200 Subject: [PATCH 5/8] chore: Clarify why the codegen reparents a generated model --- scripts/postprocess_generated_models.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/postprocess_generated_models.py b/scripts/postprocess_generated_models.py index 5aa33c86..cf706eb3 100644 --- a/scripts/postprocess_generated_models.py +++ b/scripts/postprocess_generated_models.py @@ -53,11 +53,12 @@ } # Map of `{class name: base class it should inherit from}`, applied to both generated files. -# Some request-body schemas in the spec spell out only a few properties instead of extending the base schema that -# carries the rest of the wire shape. A model generated from such a schema declares those few fields and lets -# `extra='allow'` absorb everything else - but the `to_camel` alias generator only covers declared fields, so extras -# reach the API under their snake_case names, which it silently ignores. Reparenting to the base schema's class -# declares the full shape, restoring alias coverage and the TypedDict keys the type checker validates against. +# Some schemas in the spec spell out only a few properties instead of extending the base schema that carries the rest +# of the wire shape. When a resource client sends such a model as a request body, the generated class declares those +# few fields and lets `extra='allow'` absorb everything else - but the `to_camel` alias generator only covers declared +# fields, so extras reach the API under their snake_case names, which it silently ignores. Reparenting to the base +# schema's class declares the full shape, restoring alias coverage and the TypedDict keys the type checker validates +# against. BASE_CLASS_FIXES: dict[str, str] = { 'RequestDraft': 'RequestBase', } @@ -120,9 +121,11 @@ def _base_names(node: ast.ClassDef) -> set[str]: def reparent_classes(content: str) -> str: """Replace the base class of every `BASE_CLASS_FIXES` entry with the mapped one. - The whole base list is rewritten, so re-running on already-reparented source is a no-op. Entries absent from - `content` are simply skipped - the map is shared by both generated files, and a schema does not always yield a - class in each (a root model becomes a `TypeAlias` in `_typeddicts.py`). + The whole base list is rewritten, so re-running on already-reparented source is a no-op. A mapped class absent + from `content` is skipped - the map is shared by both generated files, and a schema does not always yield a class + in each (a root model becomes a `TypeAlias` in `_typeddicts.py`). The mapped base is not looked up: an entry + naming a base that the file no longer defines yields source that fails to import, which is preferable to + silently leaving the class unreparented. """ for name, base in BASE_CLASS_FIXES.items(): content = re.sub( From 804b05df1d593b6a9a0521cfcab0484739c47103 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 18:37:22 +0200 Subject: [PATCH 6/8] chore: Tighten the comments and docstrings added for the reparenting fix --- scripts/postprocess_generated_models.py | 34 ++++++++++--------------- tests/unit/test_client_request_queue.py | 2 +- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/scripts/postprocess_generated_models.py b/scripts/postprocess_generated_models.py index cf706eb3..f3d728d8 100644 --- a/scripts/postprocess_generated_models.py +++ b/scripts/postprocess_generated_models.py @@ -1,8 +1,8 @@ """Post-process datamodel-codegen output to fix known issues and prune the TypedDict file. Applied to both `_models.py` and `_typeddicts.py`: -- Reparent classes whose spec schema declares the wire shape standalone instead of extending the base schema it - duplicates, so the generated class declares the full set of fields. +- Reparent classes whose schema spells out the wire shape standalone instead of extending the base schema it + duplicates, so the generated class declares every field. Applied to `_models.py`: - Fix discriminator field names that use camelCase instead of snake_case (known issue with discriminators on schemas @@ -52,13 +52,10 @@ 'pricingModel': 'pricing_model', } -# Map of `{class name: base class it should inherit from}`, applied to both generated files. -# Some schemas in the spec spell out only a few properties instead of extending the base schema that carries the rest -# of the wire shape. When a resource client sends such a model as a request body, the generated class declares those -# few fields and lets `extra='allow'` absorb everything else - but the `to_camel` alias generator only covers declared -# fields, so extras reach the API under their snake_case names, which it silently ignores. Reparenting to the base -# schema's class declares the full shape, restoring alias coverage and the TypedDict keys the type checker validates -# against. +# Map of `{class name: base class it should inherit from}`, applied to both generated files. A schema that spells out +# only a few properties instead of extending the base schema carrying the rest generates a class whose `extra='allow'` +# absorbs the other fields - and `to_camel` doesn't alias extras, so sending it as a request body puts snake_case keys +# on the wire. Reparenting declares the full shape. BASE_CLASS_FIXES: dict[str, str] = { 'RequestDraft': 'RequestBase', } @@ -121,11 +118,9 @@ def _base_names(node: ast.ClassDef) -> set[str]: def reparent_classes(content: str) -> str: """Replace the base class of every `BASE_CLASS_FIXES` entry with the mapped one. - The whole base list is rewritten, so re-running on already-reparented source is a no-op. A mapped class absent - from `content` is skipped - the map is shared by both generated files, and a schema does not always yield a class - in each (a root model becomes a `TypeAlias` in `_typeddicts.py`). The mapped base is not looked up: an entry - naming a base that the file no longer defines yields source that fails to import, which is preferable to - silently leaving the class unreparented. + Rewrites the whole base list, so re-running is a no-op. A mapped class absent from `content` is skipped, since a + schema doesn't always yield a class in both files. The base is not looked up: naming one the file doesn't define + yields source that fails to import, which beats silently skipping the fix. """ for name, base in BASE_CLASS_FIXES.items(): content = re.sub( @@ -145,13 +140,10 @@ def _annotated_field_names(node: ast.ClassDef) -> set[str]: def drop_inherited_typeddict_fields(content: str) -> str: - """Delete the fields a reparented TypedDict now inherits, keeping only the ones its base doesn't declare. + """Delete the fields a reparented TypedDict now inherits, along with their description docstrings. - PEP 589 forbids a TypedDict subclass from redeclaring a key of its base - even to turn a `NotRequired` key into - a required one - so the reparented class has to give up its own copies. The keys stay required at runtime: the - Pydantic model keeps its redeclarations, which Pydantic allows. - - Each field's trailing description docstring is removed along with it. + PEP 589 forbids redeclaring a base's key, even to turn a `NotRequired` one into a required one. The keys stay + required at runtime, where the Pydantic model keeps its own redeclarations. """ tree = ast.parse(content) classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} @@ -747,7 +739,7 @@ def postprocess_models(models_path: Path, literals_path: Path) -> list[Path]: def postprocess_typeddicts(path: Path, alias_map: dict[str, dict[str, str]]) -> bool: """Apply `_typeddicts.py`-specific fixes. Returns True if the file changed.""" original = path.read_text() - # Reparenting comes first so the new base class counts as a dependency of the input surface and survives pruning. + # Reparenting comes first so the new base counts as a dependency of the input surface and survives pruning. reparented = drop_inherited_typeddict_fields(reparent_classes(original)) pruned, kept = prune_typeddicts(reparented, RESOURCE_INPUT_TYPEDDICTS) renamed = rename_with_dict_suffix(pruned, kept) diff --git a/tests/unit/test_client_request_queue.py b/tests/unit/test_client_request_queue.py index 6cc1eae4..a07f737e 100644 --- a/tests/unit/test_client_request_queue.py +++ b/tests/unit/test_client_request_queue.py @@ -281,7 +281,7 @@ def test_batch_processed_partially_sync(httpserver: HTTPServer) -> None: class Unserializable: - """A `user_data` value of a type no JSON serializer can encode without a stringification fallback.""" + """A value no JSON serializer can encode without a stringification fallback.""" def __str__(self) -> str: return 'unserializable' From a6b25eae9e9aa277cdc64063964fa9f57d9c36c1 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 19:02:02 +0200 Subject: [PATCH 7/8] test: Cover the request queue write paths against the live API --- tests/integration/test_request_queue.py | 202 +++++++++++++++++++++++- 1 file changed, 200 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_request_queue.py b/tests/integration/test_request_queue.py index c09e2d7d..d67c25cc 100644 --- a/tests/integration/test_request_queue.py +++ b/tests/integration/test_request_queue.py @@ -3,8 +3,10 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterator -from datetime import timedelta -from typing import TYPE_CHECKING +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any + +import pytest from .._utils import ( collect_iterate_until_present, @@ -28,17 +30,75 @@ RequestRegistration, UnlockRequestsResult, ) +from apify_client.errors import ApifyApiError if TYPE_CHECKING: from apify_client import ApifyClient, ApifyClientAsync from apify_client._resource_clients.request_queue import RequestQueueClient, RequestQueueClientAsync from apify_client._typeddicts import ( RequestDict, + RequestDraftCamelDict, RequestDraftDeleteDict, RequestDraftDict, ) +# The wire format the API declares for `handled_at`, and the instant it denotes. The client validates the string +# into a datetime, so it only sends ISO 8601 back out while it serializes in JSON mode. +HANDLED_AT_ISO = '2019-06-16T10:23:31.607Z' +HANDLED_AT = datetime(2019, 6, 16, 10, 23, 31, 607000, tzinfo=UTC) + +# Every request field beyond `id`/`unique_key`/`url`, snake_cased. The API declares its write bodies with +# `additionalProperties: false`, so each of these has to reach it camelCased to be stored at all. +ALL_REQUEST_FIELDS: RequestDict = { + 'method': 'POST', + 'user_data': {'label': 'DETAIL', 'depth': 2}, + 'no_retry': True, + 'retry_count': 3, + 'headers': {'X-Test': 'yes'}, + 'payload': '{"a": 1}', + 'loaded_url': 'https://example.com/loaded', + 'error_messages': ['boom'], + 'handled_at': HANDLED_AT_ISO, +} + + +async def fetch_stored_request( + rq_client: RequestQueueClient | RequestQueueClientAsync, + request_id: str, +) -> Request: + """Poll until `request_id` is readable back from the queue, then return it.""" + + async def get_request() -> Request | None: + return await maybe_await(rq_client.get_request(request_id)) + + stored = await poll_until_condition(get_request, lambda request: request is not None) + assert isinstance(stored, Request) + return stored + + +def non_identity_fields(request: Request) -> dict[str, Any]: + """Return a stored request's fields without the ones identifying it, so two write paths can be compared.""" + dumped = request.model_dump(by_alias=True) + for key in ('id', 'uniqueKey', 'url'): + dumped.pop(key, None) + return dumped + + +def assert_all_fields_stored(request: Request) -> None: + """Assert the stored request carries every value of `ALL_REQUEST_FIELDS`.""" + assert request.method == 'POST' + assert request.user_data is not None + assert request.user_data.model_dump() == {'label': 'DETAIL', 'depth': 2} + assert request.no_retry is True + assert request.retry_count == 3 + assert request.headers == {'X-Test': 'yes'} + assert request.payload == '{"a": 1}' + assert str(request.loaded_url) == 'https://example.com/loaded' + assert request.error_messages == ['boom'] + assert request.handled_at == HANDLED_AT + + async def ensure_queue_is_populated( rq_client: RequestQueueClient | RequestQueueClientAsync, *, @@ -578,6 +638,144 @@ async def get_added_request() -> Request | None: await maybe_await(rq_client.delete()) +async def test_request_queue_add_request_round_trips_all_fields(client: ApifyClient | ApifyClientAsync) -> None: + """Every field of a snake_cased request survives `add_request` and comes back from `get_request`.""" + rq = await maybe_await(client.request_queues().get_or_create(name=get_random_resource_name('rq'))) + assert isinstance(rq, RequestQueue) + rq_client = client.request_queue(rq.id) + + try: + request_data: RequestDraftDict = { + 'unique_key': 'round-trip', + 'url': 'https://example.com/round-trip', + **ALL_REQUEST_FIELDS, + } + add_result = await maybe_await(rq_client.add_request(request_data)) + assert isinstance(add_result, RequestRegistration) + + stored = await fetch_stored_request(rq_client, add_result.request_id) + assert_all_fields_stored(stored) + finally: + await maybe_await(rq_client.delete()) + + +async def test_request_queue_batch_add_requests_round_trips_all_fields( + client: ApifyClient | ApifyClientAsync, +) -> None: + """Every field of a snake_cased request survives `batch_add_requests` and comes back from `get_request`.""" + rq = await maybe_await(client.request_queues().get_or_create(name=get_random_resource_name('rq'))) + assert isinstance(rq, RequestQueue) + rq_client = client.request_queue(rq.id) + + try: + requests_to_add: list[RequestDraftDict] = [ + { + 'unique_key': 'batch-round-trip', + 'url': 'https://example.com/batch-round-trip', + **ALL_REQUEST_FIELDS, + } + ] + batch_result = await maybe_await(rq_client.batch_add_requests(requests_to_add)) + assert isinstance(batch_result, BatchAddResult) + assert len(batch_result.unprocessed_requests) == 0 + assert len(batch_result.processed_requests) == 1 + + stored = await fetch_stored_request(rq_client, batch_result.processed_requests[0].request_id) + assert_all_fields_stored(stored) + finally: + await maybe_await(rq_client.delete()) + + +async def test_request_queue_add_and_update_request_store_identical_fields( + client: ApifyClient | ApifyClientAsync, +) -> None: + """The same field dict stored through `add_request` and through `update_request` lands identically.""" + rq = await maybe_await(client.request_queues().get_or_create(name=get_random_resource_name('rq'))) + assert isinstance(rq, RequestQueue) + rq_client = client.request_queue(rq.id) + + try: + added = await maybe_await( + rq_client.add_request( + {'unique_key': 'parity-add', 'url': 'https://example.com/parity-add', **ALL_REQUEST_FIELDS} + ) + ) + assert isinstance(added, RequestRegistration) + + seeded = await maybe_await( + rq_client.add_request({'unique_key': 'parity-update', 'url': 'https://example.com/parity-update'}) + ) + assert isinstance(seeded, RequestRegistration) + updated = await maybe_await( + rq_client.update_request( + { + 'id': seeded.request_id, + 'unique_key': 'parity-update', + 'url': 'https://example.com/parity-update', + **ALL_REQUEST_FIELDS, + } + ) + ) + assert isinstance(updated, RequestRegistration) + + from_add = await fetch_stored_request(rq_client, added.request_id) + from_update = await fetch_stored_request(rq_client, seeded.request_id) + assert non_identity_fields(from_add) == non_identity_fields(from_update) + assert_all_fields_stored(from_add) + assert_all_fields_stored(from_update) + finally: + await maybe_await(rq_client.delete()) + + +async def test_request_queue_add_request_accepts_camel_cased_fields(client: ApifyClient | ApifyClientAsync) -> None: + """A camelCased request dict stores the same fields as its snake_cased equivalent.""" + rq = await maybe_await(client.request_queues().get_or_create(name=get_random_resource_name('rq'))) + assert isinstance(rq, RequestQueue) + rq_client = client.request_queue(rq.id) + + try: + camel_request: RequestDraftCamelDict = { + 'uniqueKey': 'camel', + 'url': 'https://example.com/camel', + 'method': 'POST', + 'userData': {'label': 'DETAIL', 'depth': 2}, + 'noRetry': True, + 'retryCount': 3, + 'headers': {'X-Test': 'yes'}, + 'payload': '{"a": 1}', + 'loadedUrl': 'https://example.com/loaded', + 'errorMessages': ['boom'], + 'handledAt': HANDLED_AT_ISO, + } + add_result = await maybe_await(rq_client.add_request(camel_request)) + assert isinstance(add_result, RequestRegistration) + + stored = await fetch_stored_request(rq_client, add_result.request_id) + assert_all_fields_stored(stored) + finally: + await maybe_await(rq_client.delete()) + + +async def test_request_queue_add_request_rejects_undeclared_fields(client: ApifyClient | ApifyClientAsync) -> None: + """The API refuses a body key its schema does not declare, so a mis-cased field takes the whole write down.""" + rq = await maybe_await(client.request_queues().get_or_create(name=get_random_resource_name('rq'))) + assert isinstance(rq, RequestQueue) + rq_client = client.request_queue(rq.id) + + try: + # `model_validate` keeps the undeclared key as a model extra, which is serialized verbatim. + draft = RequestDraft.model_validate( + {'unique_key': 'undeclared', 'url': 'https://example.com/undeclared', 'undeclared_field': 'value'} + ) + with pytest.raises(ApifyApiError, match='not allowed by the schema'): + await maybe_await(rq_client.add_request(draft)) + + with pytest.raises(ApifyApiError, match='not allowed by the schema'): + await maybe_await(rq_client.batch_add_requests([draft])) + finally: + await maybe_await(rq_client.delete()) + + async def test_request_queue_collection_iterate(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test paginated iteration over user request queues.""" created_ids: list[str] = [] From ada23066f57d11ed24f87fd32915a2be0b60dd6c Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 19:05:18 +0200 Subject: [PATCH 8/8] test: Fix a spelling error in an integration test docstring --- tests/integration/test_request_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_request_queue.py b/tests/integration/test_request_queue.py index d67c25cc..8be9454b 100644 --- a/tests/integration/test_request_queue.py +++ b/tests/integration/test_request_queue.py @@ -757,7 +757,7 @@ async def test_request_queue_add_request_accepts_camel_cased_fields(client: Apif async def test_request_queue_add_request_rejects_undeclared_fields(client: ApifyClient | ApifyClientAsync) -> None: - """The API refuses a body key its schema does not declare, so a mis-cased field takes the whole write down.""" + """The API refuses a body key its schema does not declare, so a snake_cased field takes the whole write down.""" rq = await maybe_await(client.request_queues().get_or_create(name=get_random_resource_name('rq'))) assert isinstance(rq, RequestQueue) rq_client = client.request_queue(rq.id)