diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9370cfb --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +src/unstract/api_deployments/_sdk_docstudio/** linguist-generated=true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 304fbda..4f9df87 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,14 +14,14 @@ jobs: matrix: python-version: ["3.11", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} - name: Install uv - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 with: version: "0.6.14" enable-cache: true @@ -37,3 +37,31 @@ jobs: - name: Tests (pytest) run: uv run pytest tests/ -v + + sdk-drift: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + with: + version: "0.6.14" + enable-cache: true + + # The generated tree is committed, so an edit to it reviews like any + # other change and then disappears on the next regeneration. Same for a + # spec change that never had the generator run over it. + - name: Regenerate from the committed spec + run: ./tools/gen_sdk.sh + + # `git add -N` first: a diff alone cannot see a file the generator has + # newly created, which is exactly what a spec growing an endpoint does. + - name: Fail if the committed SDK is not what the spec generates + run: | + git add -N -- src/unstract/api_deployments/_sdk_docstudio + git diff --exit-code -- src/unstract/api_deployments/_sdk_docstudio diff --git a/.gitignore b/.gitignore index 8946793..51c2792 100644 --- a/.gitignore +++ b/.gitignore @@ -114,6 +114,8 @@ celerybeat.pid # Environments .env .venv +# Built inside the repo by tools/gen_sdk.sh. +.gen-venv/ env/ venv/ ENV/ diff --git a/README.md b/README.md index eedc645..fc8c92a 100644 --- a/README.md +++ b/README.md @@ -65,15 +65,28 @@ except APIDeploymentsClientException as e: `api_url`: The URL of the Unstract API deployment. `api_key`: Your raw API key. **Do not** include the `"Bearer "` prefix — the client adds it automatically. -`api_timeout`: Set a timeout for API requests, e.g., `api_timeout=10`. +`api_timeout`: Backend execution mode sent with the request (see `timeout` on `structure_file`). `0` or below queues the execution and returns immediately; above it the call runs synchronously and the value bounds how long the backend waits. This is not a socket timeout — pass `transport_timeout` for that. `logging_level`: Set logging verbosity (e.g., "`DEBUG`"). `include_metadata`: If set to `True`, the response will include additional metadata (cost, tokens consumed and context) for each call made by the Prompt Studio exported tool. +`transport_timeout`: Socket timeout in seconds (keyword-only). Left unset, a stalled connection blocks forever, which is what earlier releases did. + +## Closing the client + +The client reuses connections between calls, so release them when you are done +with it — either by calling `close()`, or by using it as a context manager: + +```python +with APIDeploymentsClient(api_url="url", api_key="your_api_key") as adc: + response = adc.structure_file([""]) +``` + +A long-lived client can be left open; one built per job should be closed. ## Retry Configuration The client includes built-in exponential backoff retry with the following behavior: -- **Async mode** (`api_timeout=0`): POST requests are retried on transient failures (5xx, 429) and connection errors, since the server returns immediately after queuing. +- **Async mode** (`api_timeout` of `0` or below): POST requests are retried on transient failures (5xx, 429) and connection errors, since the server returns immediately after queuing. - **Sync mode** (`api_timeout > 0`, the default): POST requests are **not** retried, because the server blocks during processing — a failure may mean the request was processed but the response was lost. - **Status polling** (`check_execution_status`): GET requests are always retried, as they are idempotent. @@ -100,16 +113,24 @@ client = APIDeploymentsClient( The retry logic uses exponential backoff with full jitter and respects the `Retry-After` header on 429 responses. -## Unstract CLI +## Internals + +`unstract.api_deployments._sdk_docstudio` is generated from the deployment API's +OpenAPI spec by `tools/gen_sdk.sh` and is an implementation detail of the +transport. `APIDeploymentsClient` is the supported surface — import from it, not +from the generated tree, which is regenerated wholesale whenever the spec moves. + +## Cloning an organization -Installing `unstract-client` also provides the `unstract` command: +Installing `unstract-client` also provides a clone command. This package no +longer installs an `unstract` console script, so invoke it as a module: ```bash pip install unstract-client -unstract --help +python -m unstract.clone --help ``` -### `unstract clone` +### `python -m unstract.clone clone` Clones an organization's resources to another org, on the same or a different deployment (e.g. promote **dev** → **QA** → **prod**). Covers adapters, @@ -124,7 +145,7 @@ so keys never land in shell history: export UNSTRACT_SRC_PLATFORM_KEY="" export UNSTRACT_TGT_PLATFORM_KEY="" -unstract clone \ +python -m unstract.clone clone \ --source-url https://dev.example.com --source-org org_dev123 \ --target-url https://qa.example.com --target-org org_qa456 \ --dry-run @@ -152,14 +173,14 @@ failed run can be resumed by re-running the same command. #### Compatibility -`unstract clone` is capability-probed: each phase checks for its endpoint on the +Cloning is capability-probed: each phase checks for its endpoint on the source and target, and clones only what both orgs support. A capability missing on either side is reported and skipped — the run never fails because of a version difference. Cloning a newer source into an older target therefore drops the entity types the target lacks (listed in the end-of-run report). - Run the source and target on the same (or a newer-target) Unstract build. -- Use `unstract-client >= 1.4.0`, the first release that ships `unstract clone`. +- Use `unstract-client >= 1.4.0`, the first release that ships the clone command. ## Questions and Feedback diff --git a/pyproject.toml b/pyproject.toml index 0cb4dff..694877c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,13 @@ authors = [ {name = "Zipstack Inc", email = "devsupport@zipstack.com"}, ] dependencies = [ + # The transport layer is generated against httpx; attrs backs its models. + # Upper-bounded because the generated code is written against one minor + # series: a bump has to be regenerated and re-tested, not resolved into. + "httpx>=0.27,<0.29", + "attrs>=23.2", + # Still the transport for the `unstract.clone` subpackage, and the source of + # the exception classes callers catch by name around the deployment client. "requests>=2.32.3", "tenacity>=8.2.0", "click>=8.1", @@ -27,9 +34,6 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] -[project.scripts] -unstract = "unstract.cli:main" - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" @@ -64,6 +68,9 @@ lint = [ [tool.ruff] line-length = 88 +# Generated and vendored code is overwritten wholesale by its refresh script, so +# a lint finding there can never be fixed in place. +extend-exclude = ["src/unstract/api_deployments/_sdk_docstudio", "tests/baseline"] [tool.ruff.lint] select = ["E", "F", "W", "I"] diff --git a/specs/docstudio-oss.json b/specs/docstudio-oss.json new file mode 100644 index 0000000..c7f8ebb --- /dev/null +++ b/specs/docstudio-oss.json @@ -0,0 +1,722 @@ +{ + "components": { + "schemas": { + "AcknowledgedResponse": { + "description": "The execution's result was handed to an earlier call and discarded.", + "properties": { + "message": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "required": [ + "message", + "status" + ], + "type": "object" + }, + "ErrorDetail": { + "description": "One problem found with the request.", + "properties": { + "attr": { + "description": "The request field the problem belongs to, when it belongs to one.", + "nullable": true, + "type": "string" + }, + "code": { + "description": "Machine-readable problem identifier.", + "type": "string" + }, + "detail": { + "description": "Human-readable description.", + "type": "string" + } + }, + "required": [ + "attr", + "code", + "detail" + ], + "type": "object" + }, + "ErrorResponse": { + "description": "The body of a rejected request.\n\nProduced by the project-wide exception handler, so its shape is the same\nfor every failure listed against an operation.", + "properties": { + "errors": { + "items": { + "$ref": "#/components/schemas/ErrorDetail" + }, + "type": "array" + }, + "type": { + "$ref": "#/components/schemas/ErrorType" + } + }, + "required": [ + "errors", + "type" + ], + "type": "object" + }, + "ErrorType": { + "description": "* `validation_error` - validation_error\n* `client_error` - client_error\n* `server_error` - server_error", + "enum": [ + "validation_error", + "client_error", + "server_error" + ], + "type": "string" + }, + "ExecuteRequest": { + "description": "The documents to run, and the options that shape the result.\n\nSupply `files`, `presigned_urls`, or both.", + "properties": { + "custom_data": { + "nullable": true + }, + "files": { + "items": { + "format": "binary", + "type": "string" + }, + "type": "array" + }, + "hitl_packet_id": { + "description": "Groups documents reviewed together into one packet. Requires the enterprise manual-review capability; an installation without it rejects the request with 400.", + "nullable": true, + "type": "string" + }, + "hitl_queue_name": { + "description": "Document class name for the manual review queue. Requires the enterprise manual-review capability; an installation without it rejects the request with 400.", + "nullable": true, + "type": "string" + }, + "include_extracted_text": { + "default": false, + "type": "boolean" + }, + "include_metadata": { + "default": false, + "type": "boolean" + }, + "include_metrics": { + "default": false, + "type": "boolean" + }, + "llm_profile_id": { + "nullable": true, + "type": "string" + }, + "presigned_urls": { + "items": { + "format": "uri", + "type": "string" + }, + "type": "array" + }, + "tags": { + "default": "", + "description": "Comma-separated list of tag names (EX:'tag1,tag2-name,tag3_name')", + "type": "string" + }, + "timeout": { + "default": -1, + "maximum": 300, + "minimum": -1, + "type": "integer" + }, + "use_file_history": { + "default": false, + "type": "boolean" + } + }, + "type": "object" + }, + "ExecuteResponse": { + "properties": { + "message": { + "$ref": "#/components/schemas/ExecutionMessage" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "ExecutionMessage": { + "description": "The execution's identity and, once it has finished, its per-file\nresults.", + "properties": { + "error": { + "nullable": true, + "type": "string" + }, + "execution_id": { + "type": "string" + }, + "execution_status": { + "type": "string" + }, + "result": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status_api": { + "nullable": true, + "type": "string" + } + }, + "required": [ + "execution_id", + "execution_status" + ], + "type": "object" + }, + "FileResult": { + "description": "One input document's outcome.\n\nEvery key is present on every item; the ones that depend on the request\noptions or on the outcome are sent as `null` when they do not apply.", + "properties": { + "error": { + "nullable": true, + "type": "string" + }, + "extracted_text": { + "description": "The document's full extracted text. Sent only when the request set `include_extracted_text`.", + "nullable": true, + "type": "string" + }, + "file": { + "type": "string" + }, + "file_execution_id": { + "nullable": true, + "type": "string" + }, + "metadata": { + "nullable": true + }, + "result": { + "nullable": true + }, + "status": { + "type": "string" + } + }, + "required": [ + "file" + ], + "type": "object" + }, + "StatusResponse": { + "properties": { + "message": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status": { + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + } + }, + "securitySchemes": { + "deploymentKey": { + "description": "The API deployment's own key.", + "scheme": "bearer", + "type": "http" + } + } + }, + "info": { + "title": "Unstract API", + "version": "v1" + }, + "openapi": "3.0.3", + "paths": { + "/deployment/api/{org_name}/{api_name}/": { + "get": { + "description": "Read the result of a previously started execution.\n\nThis read is one-shot: the first call that observes a completed execution acknowledges it and the stored result is discarded, so every later call for that execution answers 406. Poll while the execution is pending, and keep the payload of the call that returns it \u2014 it cannot be fetched again.\n\nA still-running execution answers 422 carrying its current `status`, so a polling loop should treat 422 as the normal reply and stop on 200. Clients that raise on any non-2xx need to allow for that.", + "operationId": "status", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "in": "query", + "name": "execution_id", + "required": true, + "schema": { + "minLength": 1, + "type": "string" + } + }, + { + "in": "query", + "name": "include_extracted_text", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metadata", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metrics", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "examples": { + "AuthenticationFailed": { + "value": { + "errors": [ + { + "attr": null, + "code": "authentication_failed", + "detail": "Incorrect authentication credentials." + } + ], + "type": "client_error" + } + }, + "NotAuthenticated": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_authenticated", + "detail": "Authentication credentials were not provided." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No usable API key was supplied for the deployment." + }, + "403": { + "content": { + "application/json": { + "examples": { + "PermissionDenied": { + "value": { + "errors": [ + { + "attr": null, + "code": "permission_denied", + "detail": "You do not have permission to perform this action." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request was refused as unauthorized." + }, + "404": { + "content": { + "application/json": { + "examples": { + "NotFound": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_found", + "detail": "Not found." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No active deployment, or a referenced document, was found." + }, + "406": { + "content": { + "application/json": { + "examples": { + "NotAcceptable": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_acceptable", + "detail": "Could not satisfy the request Accept header." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/AcknowledgedResponse" + } + } + }, + "description": "The result was already consumed by an earlier call." + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "The execution is still running, or it finished with an error; read `status` to tell them apart." + }, + "500": { + "content": { + "application/json": { + "examples": { + "APIException": { + "value": { + "errors": [ + { + "attr": null, + "code": "error", + "detail": "A server error occurred." + } + ], + "type": "server_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "The execution could not be completed; the body carries its last known state." + } + }, + "security": [ + { + "deploymentKey": [] + } + ], + "tags": [ + "deployment" + ] + }, + "post": { + "description": "Execute an API deployment against one or more documents.\n\nSupply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or both \u2014 a request carrying neither is rejected, and the two together may not exceed 32 documents.\n\nWith the default `timeout` of -1 the call returns as soon as the execution is queued; read the outcome from the status endpoint.", + "operationId": "execute", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/ExecuteRequest" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "examples": { + "AuthenticationFailed": { + "value": { + "errors": [ + { + "attr": null, + "code": "authentication_failed", + "detail": "Incorrect authentication credentials." + } + ], + "type": "client_error" + } + }, + "NotAuthenticated": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_authenticated", + "detail": "Authentication credentials were not provided." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No usable API key was supplied for the deployment." + }, + "403": { + "content": { + "application/json": { + "examples": { + "PermissionDenied": { + "value": { + "errors": [ + { + "attr": null, + "code": "permission_denied", + "detail": "You do not have permission to perform this action." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request was refused as unauthorized." + }, + "404": { + "content": { + "application/json": { + "examples": { + "NotFound": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_found", + "detail": "Not found." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No active deployment, or a referenced document, was found." + }, + "413": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "A referenced document is larger than the limit." + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "The execution finished with an error." + }, + "429": { + "content": { + "application/json": { + "examples": { + "Throttled": { + "value": { + "errors": [ + { + "attr": null, + "code": "throttled", + "detail": "Request was throttled." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Too many concurrent executions; retry later." + }, + "500": { + "content": { + "application/json": { + "examples": { + "APIException": { + "value": { + "errors": [ + { + "attr": null, + "code": "error", + "detail": "A server error occurred." + } + ], + "type": "server_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "The deployment could not be run; the body carries the execution that failed." + }, + "502": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "A referenced document could not be fetched." + }, + "504": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Fetching a referenced document timed out." + } + }, + "security": [ + { + "deploymentKey": [] + } + ], + "tags": [ + "deployment" + ] + } + } + }, + "tags": [ + { + "description": "Run an API deployment against one or more documents and poll the result.", + "name": "deployment" + } + ] +} diff --git a/src/unstract/api_deployments/_sdk_docstudio/__init__.py b/src/unstract/api_deployments/_sdk_docstudio/__init__.py new file mode 100644 index 0000000..ad2a46e --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/__init__.py @@ -0,0 +1,9 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""A client library for accessing Unstract API""" + +from .client import AuthenticatedClient, Client + +__all__ = ( + "AuthenticatedClient", + "Client", +) diff --git a/src/unstract/api_deployments/_sdk_docstudio/api/__init__.py b/src/unstract/api_deployments/_sdk_docstudio/api/__init__.py new file mode 100644 index 0000000..d8d42ef --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/api/__init__.py @@ -0,0 +1,2 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""Contains methods for accessing the API""" diff --git a/src/unstract/api_deployments/_sdk_docstudio/api/deployment/__init__.py b/src/unstract/api_deployments/_sdk_docstudio/api/deployment/__init__.py new file mode 100644 index 0000000..c7e8df6 --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/api/deployment/__init__.py @@ -0,0 +1,2 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""Contains endpoint functions for accessing the API""" diff --git a/src/unstract/api_deployments/_sdk_docstudio/api/deployment/execute.py b/src/unstract/api_deployments/_sdk_docstudio/api/deployment/execute.py new file mode 100644 index 0000000..fabe9ab --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/api/deployment/execute.py @@ -0,0 +1,279 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from http import HTTPStatus +from typing import Any +from urllib.parse import quote + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.error_response import ErrorResponse +from ...models.execute_request import ExecuteRequest +from ...models.execute_response import ExecuteResponse +from ...types import UNSET, Response, Unset + + +def _get_kwargs( + org_name: str, + api_name: str, + *, + body: ExecuteRequest | Unset = UNSET, +) -> dict[str, Any]: + headers: dict[str, Any] = {} + + _kwargs: dict[str, Any] = { + "method": "post", + "url": "/deployment/api/{org_name}/{api_name}/".format( + org_name=quote(str(org_name), safe=""), + api_name=quote(str(api_name), safe=""), + ), + } + + if not isinstance(body, Unset): + _kwargs["files"] = body.to_multipart() + + headers["Content-Type"] = "multipart/form-data; boundary=+++" + + _kwargs["headers"] = headers + return _kwargs + + +def _parse_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> ErrorResponse | ExecuteResponse | None: + if response.status_code == 200: + response_200 = ExecuteResponse.from_dict(response.json()) + + return response_200 + + if response.status_code == 400: + response_400 = ErrorResponse.from_dict(response.json()) + + return response_400 + + if response.status_code == 401: + response_401 = ErrorResponse.from_dict(response.json()) + + return response_401 + + if response.status_code == 403: + response_403 = ErrorResponse.from_dict(response.json()) + + return response_403 + + if response.status_code == 404: + response_404 = ErrorResponse.from_dict(response.json()) + + return response_404 + + if response.status_code == 413: + response_413 = ErrorResponse.from_dict(response.json()) + + return response_413 + + if response.status_code == 422: + response_422 = ExecuteResponse.from_dict(response.json()) + + return response_422 + + if response.status_code == 429: + response_429 = ErrorResponse.from_dict(response.json()) + + return response_429 + + if response.status_code == 500: + response_500 = ExecuteResponse.from_dict(response.json()) + + return response_500 + + if response.status_code == 502: + response_502 = ErrorResponse.from_dict(response.json()) + + return response_502 + + if response.status_code == 504: + response_504 = ErrorResponse.from_dict(response.json()) + + return response_504 + + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Response[ErrorResponse | ExecuteResponse]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + body: ExecuteRequest | Unset = UNSET, +) -> Response[ErrorResponse | ExecuteResponse]: + """Execute an API deployment against one or more documents. + + Supply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or + both — a request carrying neither is rejected, and the two together may not exceed 32 documents. + + With the default `timeout` of -1 the call returns as soon as the execution is queued; read the + outcome from the status endpoint. + + Args: + org_name (str): + api_name (str): + body (ExecuteRequest | Unset): The documents to run, and the options that shape the + result. + + Supply `files`, `presigned_urls`, or both. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[ErrorResponse | ExecuteResponse] + """ + + kwargs = _get_kwargs( + org_name=org_name, + api_name=api_name, + body=body, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + body: ExecuteRequest | Unset = UNSET, +) -> ErrorResponse | ExecuteResponse | None: + """Execute an API deployment against one or more documents. + + Supply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or + both — a request carrying neither is rejected, and the two together may not exceed 32 documents. + + With the default `timeout` of -1 the call returns as soon as the execution is queued; read the + outcome from the status endpoint. + + Args: + org_name (str): + api_name (str): + body (ExecuteRequest | Unset): The documents to run, and the options that shape the + result. + + Supply `files`, `presigned_urls`, or both. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + ErrorResponse | ExecuteResponse + """ + + return sync_detailed( + org_name=org_name, + api_name=api_name, + client=client, + body=body, + ).parsed + + +async def asyncio_detailed( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + body: ExecuteRequest | Unset = UNSET, +) -> Response[ErrorResponse | ExecuteResponse]: + """Execute an API deployment against one or more documents. + + Supply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or + both — a request carrying neither is rejected, and the two together may not exceed 32 documents. + + With the default `timeout` of -1 the call returns as soon as the execution is queued; read the + outcome from the status endpoint. + + Args: + org_name (str): + api_name (str): + body (ExecuteRequest | Unset): The documents to run, and the options that shape the + result. + + Supply `files`, `presigned_urls`, or both. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[ErrorResponse | ExecuteResponse] + """ + + kwargs = _get_kwargs( + org_name=org_name, + api_name=api_name, + body=body, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + body: ExecuteRequest | Unset = UNSET, +) -> ErrorResponse | ExecuteResponse | None: + """Execute an API deployment against one or more documents. + + Supply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or + both — a request carrying neither is rejected, and the two together may not exceed 32 documents. + + With the default `timeout` of -1 the call returns as soon as the execution is queued; read the + outcome from the status endpoint. + + Args: + org_name (str): + api_name (str): + body (ExecuteRequest | Unset): The documents to run, and the options that shape the + result. + + Supply `files`, `presigned_urls`, or both. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + ErrorResponse | ExecuteResponse + """ + + return ( + await asyncio_detailed( + org_name=org_name, + api_name=api_name, + client=client, + body=body, + ) + ).parsed diff --git a/src/unstract/api_deployments/_sdk_docstudio/api/deployment/status.py b/src/unstract/api_deployments/_sdk_docstudio/api/deployment/status.py new file mode 100644 index 0000000..5946b09 --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/api/deployment/status.py @@ -0,0 +1,301 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from http import HTTPStatus +from typing import Any +from urllib.parse import quote + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.acknowledged_response import AcknowledgedResponse +from ...models.error_response import ErrorResponse +from ...models.status_response import StatusResponse +from ...types import UNSET, Response, Unset + + +def _get_kwargs( + org_name: str, + api_name: str, + *, + execution_id: str, + include_extracted_text: bool | Unset = False, + include_metadata: bool | Unset = False, + include_metrics: bool | Unset = False, +) -> dict[str, Any]: + + params: dict[str, Any] = {} + + params["execution_id"] = execution_id + + params["include_extracted_text"] = include_extracted_text + + params["include_metadata"] = include_metadata + + params["include_metrics"] = include_metrics + + params = {k: v for k, v in params.items() if v is not UNSET and v is not None} + + _kwargs: dict[str, Any] = { + "method": "get", + "url": "/deployment/api/{org_name}/{api_name}/".format( + org_name=quote(str(org_name), safe=""), + api_name=quote(str(api_name), safe=""), + ), + "params": params, + } + + return _kwargs + + +def _parse_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> AcknowledgedResponse | ErrorResponse | StatusResponse | None: + if response.status_code == 200: + response_200 = StatusResponse.from_dict(response.json()) + + return response_200 + + if response.status_code == 400: + response_400 = ErrorResponse.from_dict(response.json()) + + return response_400 + + if response.status_code == 401: + response_401 = ErrorResponse.from_dict(response.json()) + + return response_401 + + if response.status_code == 403: + response_403 = ErrorResponse.from_dict(response.json()) + + return response_403 + + if response.status_code == 404: + response_404 = ErrorResponse.from_dict(response.json()) + + return response_404 + + if response.status_code == 406: + response_406 = AcknowledgedResponse.from_dict(response.json()) + + return response_406 + + if response.status_code == 422: + response_422 = StatusResponse.from_dict(response.json()) + + return response_422 + + if response.status_code == 500: + response_500 = StatusResponse.from_dict(response.json()) + + return response_500 + + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Response[AcknowledgedResponse | ErrorResponse | StatusResponse]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + execution_id: str, + include_extracted_text: bool | Unset = False, + include_metadata: bool | Unset = False, + include_metrics: bool | Unset = False, +) -> Response[AcknowledgedResponse | ErrorResponse | StatusResponse]: + """Read the result of a previously started execution. + + This read is one-shot: the first call that observes a completed execution acknowledges it and the + stored result is discarded, so every later call for that execution answers 406. Poll while the + execution is pending, and keep the payload of the call that returns it — it cannot be fetched again. + + A still-running execution answers 422 carrying its current `status`, so a polling loop should treat + 422 as the normal reply and stop on 200. Clients that raise on any non-2xx need to allow for that. + + Args: + org_name (str): + api_name (str): + execution_id (str): + include_extracted_text (bool | Unset): Default: False. + include_metadata (bool | Unset): Default: False. + include_metrics (bool | Unset): Default: False. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[AcknowledgedResponse | ErrorResponse | StatusResponse] + """ + + kwargs = _get_kwargs( + org_name=org_name, + api_name=api_name, + execution_id=execution_id, + include_extracted_text=include_extracted_text, + include_metadata=include_metadata, + include_metrics=include_metrics, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + execution_id: str, + include_extracted_text: bool | Unset = False, + include_metadata: bool | Unset = False, + include_metrics: bool | Unset = False, +) -> AcknowledgedResponse | ErrorResponse | StatusResponse | None: + """Read the result of a previously started execution. + + This read is one-shot: the first call that observes a completed execution acknowledges it and the + stored result is discarded, so every later call for that execution answers 406. Poll while the + execution is pending, and keep the payload of the call that returns it — it cannot be fetched again. + + A still-running execution answers 422 carrying its current `status`, so a polling loop should treat + 422 as the normal reply and stop on 200. Clients that raise on any non-2xx need to allow for that. + + Args: + org_name (str): + api_name (str): + execution_id (str): + include_extracted_text (bool | Unset): Default: False. + include_metadata (bool | Unset): Default: False. + include_metrics (bool | Unset): Default: False. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + AcknowledgedResponse | ErrorResponse | StatusResponse + """ + + return sync_detailed( + org_name=org_name, + api_name=api_name, + client=client, + execution_id=execution_id, + include_extracted_text=include_extracted_text, + include_metadata=include_metadata, + include_metrics=include_metrics, + ).parsed + + +async def asyncio_detailed( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + execution_id: str, + include_extracted_text: bool | Unset = False, + include_metadata: bool | Unset = False, + include_metrics: bool | Unset = False, +) -> Response[AcknowledgedResponse | ErrorResponse | StatusResponse]: + """Read the result of a previously started execution. + + This read is one-shot: the first call that observes a completed execution acknowledges it and the + stored result is discarded, so every later call for that execution answers 406. Poll while the + execution is pending, and keep the payload of the call that returns it — it cannot be fetched again. + + A still-running execution answers 422 carrying its current `status`, so a polling loop should treat + 422 as the normal reply and stop on 200. Clients that raise on any non-2xx need to allow for that. + + Args: + org_name (str): + api_name (str): + execution_id (str): + include_extracted_text (bool | Unset): Default: False. + include_metadata (bool | Unset): Default: False. + include_metrics (bool | Unset): Default: False. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[AcknowledgedResponse | ErrorResponse | StatusResponse] + """ + + kwargs = _get_kwargs( + org_name=org_name, + api_name=api_name, + execution_id=execution_id, + include_extracted_text=include_extracted_text, + include_metadata=include_metadata, + include_metrics=include_metrics, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + execution_id: str, + include_extracted_text: bool | Unset = False, + include_metadata: bool | Unset = False, + include_metrics: bool | Unset = False, +) -> AcknowledgedResponse | ErrorResponse | StatusResponse | None: + """Read the result of a previously started execution. + + This read is one-shot: the first call that observes a completed execution acknowledges it and the + stored result is discarded, so every later call for that execution answers 406. Poll while the + execution is pending, and keep the payload of the call that returns it — it cannot be fetched again. + + A still-running execution answers 422 carrying its current `status`, so a polling loop should treat + 422 as the normal reply and stop on 200. Clients that raise on any non-2xx need to allow for that. + + Args: + org_name (str): + api_name (str): + execution_id (str): + include_extracted_text (bool | Unset): Default: False. + include_metadata (bool | Unset): Default: False. + include_metrics (bool | Unset): Default: False. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + AcknowledgedResponse | ErrorResponse | StatusResponse + """ + + return ( + await asyncio_detailed( + org_name=org_name, + api_name=api_name, + client=client, + execution_id=execution_id, + include_extracted_text=include_extracted_text, + include_metadata=include_metadata, + include_metrics=include_metrics, + ) + ).parsed diff --git a/src/unstract/api_deployments/_sdk_docstudio/client.py b/src/unstract/api_deployments/_sdk_docstudio/client.py new file mode 100644 index 0000000..5b50486 --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/client.py @@ -0,0 +1,283 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +import ssl +from typing import Any + +import httpx +from attrs import define, evolve, field + + +@define +class Client: + """A class for keeping track of data related to the API + + The following are accepted as keyword arguments and will be used to construct httpx Clients internally: + + ``base_url``: The base URL for the API, all requests are made to a relative path to this URL + + ``cookies``: A dictionary of cookies to be sent with every request + + ``headers``: A dictionary of headers to be sent with every request + + ``timeout``: The maximum amount of a time a request can take. API functions will raise + httpx.TimeoutException if this is exceeded. + + ``verify_ssl``: Whether or not to verify the SSL certificate of the API server. This should be True in production, + but can be set to False for testing purposes. + + ``follow_redirects``: Whether or not to follow redirects. Default value is False. + + ``httpx_args``: A dictionary of additional arguments to be passed to the ``httpx.Client`` and ``httpx.AsyncClient`` constructor. + + + Attributes: + raise_on_unexpected_status: Whether or not to raise an errors.UnexpectedStatus if the API returns a + status code that was not documented in the source OpenAPI document. Can also be provided as a keyword + argument to the constructor. + """ + + raise_on_unexpected_status: bool = field(default=False, kw_only=True) + _base_url: str = field(alias="base_url") + _cookies: dict[str, str] = field(factory=dict, kw_only=True, alias="cookies") + _headers: dict[str, str] = field(factory=dict, kw_only=True, alias="headers") + _timeout: httpx.Timeout | None = field(default=None, kw_only=True, alias="timeout") + _verify_ssl: str | bool | ssl.SSLContext = field( + default=True, kw_only=True, alias="verify_ssl" + ) + _follow_redirects: bool = field( + default=False, kw_only=True, alias="follow_redirects" + ) + _httpx_args: dict[str, Any] = field(factory=dict, kw_only=True, alias="httpx_args") + _client: httpx.Client | None = field(default=None, init=False) + _async_client: httpx.AsyncClient | None = field(default=None, init=False) + + def with_headers(self, headers: dict[str, str]) -> "Client": + """Get a new client matching this one with additional headers""" + if self._client is not None: + self._client.headers.update(headers) + if self._async_client is not None: + self._async_client.headers.update(headers) + return evolve(self, headers={**self._headers, **headers}) + + def with_cookies(self, cookies: dict[str, str]) -> "Client": + """Get a new client matching this one with additional cookies""" + if self._client is not None: + self._client.cookies.update(cookies) + if self._async_client is not None: + self._async_client.cookies.update(cookies) + return evolve(self, cookies={**self._cookies, **cookies}) + + def with_timeout(self, timeout: httpx.Timeout) -> "Client": + """Get a new client matching this one with a new timeout configuration""" + if self._client is not None: + self._client.timeout = timeout + if self._async_client is not None: + self._async_client.timeout = timeout + return evolve(self, timeout=timeout) + + def set_httpx_client(self, client: httpx.Client) -> "Client": + """Manually set the underlying httpx.Client + + **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout. + """ + self._client = client + return self + + def get_httpx_client(self) -> httpx.Client: + """Get the underlying httpx.Client, constructing a new one if not previously set""" + if self._client is None: + self._client = httpx.Client( + base_url=self._base_url, + cookies=self._cookies, + headers=self._headers, + timeout=self._timeout, + verify=self._verify_ssl, + follow_redirects=self._follow_redirects, + **self._httpx_args, + ) + return self._client + + def __enter__(self) -> "Client": + """Enter a context manager for self.client—you cannot enter twice (see httpx docs)""" + self.get_httpx_client().__enter__() + return self + + def __exit__(self, *args: Any, **kwargs: Any) -> None: + """Exit a context manager for internal httpx.Client (see httpx docs)""" + self.get_httpx_client().__exit__(*args, **kwargs) + + def set_async_httpx_client(self, async_client: httpx.AsyncClient) -> "Client": + """Manually set the underlying httpx.AsyncClient + + **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout. + """ + self._async_client = async_client + return self + + def get_async_httpx_client(self) -> httpx.AsyncClient: + """Get the underlying httpx.AsyncClient, constructing a new one if not previously set""" + if self._async_client is None: + self._async_client = httpx.AsyncClient( + base_url=self._base_url, + cookies=self._cookies, + headers=self._headers, + timeout=self._timeout, + verify=self._verify_ssl, + follow_redirects=self._follow_redirects, + **self._httpx_args, + ) + return self._async_client + + async def __aenter__(self) -> "Client": + """Enter a context manager for underlying httpx.AsyncClient—you cannot enter twice (see httpx docs)""" + await self.get_async_httpx_client().__aenter__() + return self + + async def __aexit__(self, *args: Any, **kwargs: Any) -> None: + """Exit a context manager for underlying httpx.AsyncClient (see httpx docs)""" + await self.get_async_httpx_client().__aexit__(*args, **kwargs) + + +@define +class AuthenticatedClient: + """A Client which has been authenticated for use on secured endpoints + + The following are accepted as keyword arguments and will be used to construct httpx Clients internally: + + ``base_url``: The base URL for the API, all requests are made to a relative path to this URL + + ``cookies``: A dictionary of cookies to be sent with every request + + ``headers``: A dictionary of headers to be sent with every request + + ``timeout``: The maximum amount of a time a request can take. API functions will raise + httpx.TimeoutException if this is exceeded. + + ``verify_ssl``: Whether or not to verify the SSL certificate of the API server. This should be True in production, + but can be set to False for testing purposes. + + ``follow_redirects``: Whether or not to follow redirects. Default value is False. + + ``httpx_args``: A dictionary of additional arguments to be passed to the ``httpx.Client`` and ``httpx.AsyncClient`` constructor. + + + Attributes: + raise_on_unexpected_status: Whether or not to raise an errors.UnexpectedStatus if the API returns a + status code that was not documented in the source OpenAPI document. Can also be provided as a keyword + argument to the constructor. + token: The token to use for authentication + prefix: The prefix to use for the Authorization header + auth_header_name: The name of the Authorization header + """ + + raise_on_unexpected_status: bool = field(default=False, kw_only=True) + _base_url: str = field(alias="base_url") + _cookies: dict[str, str] = field(factory=dict, kw_only=True, alias="cookies") + _headers: dict[str, str] = field(factory=dict, kw_only=True, alias="headers") + _timeout: httpx.Timeout | None = field(default=None, kw_only=True, alias="timeout") + _verify_ssl: str | bool | ssl.SSLContext = field( + default=True, kw_only=True, alias="verify_ssl" + ) + _follow_redirects: bool = field( + default=False, kw_only=True, alias="follow_redirects" + ) + _httpx_args: dict[str, Any] = field(factory=dict, kw_only=True, alias="httpx_args") + _client: httpx.Client | None = field(default=None, init=False) + _async_client: httpx.AsyncClient | None = field(default=None, init=False) + + token: str + prefix: str = "Bearer" + auth_header_name: str = "Authorization" + + def with_headers(self, headers: dict[str, str]) -> "AuthenticatedClient": + """Get a new client matching this one with additional headers""" + if self._client is not None: + self._client.headers.update(headers) + if self._async_client is not None: + self._async_client.headers.update(headers) + return evolve(self, headers={**self._headers, **headers}) + + def with_cookies(self, cookies: dict[str, str]) -> "AuthenticatedClient": + """Get a new client matching this one with additional cookies""" + if self._client is not None: + self._client.cookies.update(cookies) + if self._async_client is not None: + self._async_client.cookies.update(cookies) + return evolve(self, cookies={**self._cookies, **cookies}) + + def with_timeout(self, timeout: httpx.Timeout) -> "AuthenticatedClient": + """Get a new client matching this one with a new timeout configuration""" + if self._client is not None: + self._client.timeout = timeout + if self._async_client is not None: + self._async_client.timeout = timeout + return evolve(self, timeout=timeout) + + def set_httpx_client(self, client: httpx.Client) -> "AuthenticatedClient": + """Manually set the underlying httpx.Client + + **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout. + """ + self._client = client + return self + + def get_httpx_client(self) -> httpx.Client: + """Get the underlying httpx.Client, constructing a new one if not previously set""" + if self._client is None: + self._headers[self.auth_header_name] = ( + f"{self.prefix} {self.token}" if self.prefix else self.token + ) + self._client = httpx.Client( + base_url=self._base_url, + cookies=self._cookies, + headers=self._headers, + timeout=self._timeout, + verify=self._verify_ssl, + follow_redirects=self._follow_redirects, + **self._httpx_args, + ) + return self._client + + def __enter__(self) -> "AuthenticatedClient": + """Enter a context manager for self.client—you cannot enter twice (see httpx docs)""" + self.get_httpx_client().__enter__() + return self + + def __exit__(self, *args: Any, **kwargs: Any) -> None: + """Exit a context manager for internal httpx.Client (see httpx docs)""" + self.get_httpx_client().__exit__(*args, **kwargs) + + def set_async_httpx_client( + self, async_client: httpx.AsyncClient + ) -> "AuthenticatedClient": + """Manually set the underlying httpx.AsyncClient + + **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout. + """ + self._async_client = async_client + return self + + def get_async_httpx_client(self) -> httpx.AsyncClient: + """Get the underlying httpx.AsyncClient, constructing a new one if not previously set""" + if self._async_client is None: + self._headers[self.auth_header_name] = ( + f"{self.prefix} {self.token}" if self.prefix else self.token + ) + self._async_client = httpx.AsyncClient( + base_url=self._base_url, + cookies=self._cookies, + headers=self._headers, + timeout=self._timeout, + verify=self._verify_ssl, + follow_redirects=self._follow_redirects, + **self._httpx_args, + ) + return self._async_client + + async def __aenter__(self) -> "AuthenticatedClient": + """Enter a context manager for underlying httpx.AsyncClient—you cannot enter twice (see httpx docs)""" + await self.get_async_httpx_client().__aenter__() + return self + + async def __aexit__(self, *args: Any, **kwargs: Any) -> None: + """Exit a context manager for underlying httpx.AsyncClient (see httpx docs)""" + await self.get_async_httpx_client().__aexit__(*args, **kwargs) diff --git a/src/unstract/api_deployments/_sdk_docstudio/errors.py b/src/unstract/api_deployments/_sdk_docstudio/errors.py new file mode 100644 index 0000000..0d1fb6b --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/errors.py @@ -0,0 +1,17 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""Contains shared errors types that can be raised from API functions""" + + +class UnexpectedStatus(Exception): + """Raised by api functions when the response status an undocumented status and Client.raise_on_unexpected_status is True""" + + def __init__(self, status_code: int, content: bytes): + self.status_code = status_code + self.content = content + + super().__init__( + f"Unexpected status code: {status_code}\n\nResponse content:\n{content.decode(errors='ignore')}" + ) + + +__all__ = ["UnexpectedStatus"] diff --git a/src/unstract/api_deployments/_sdk_docstudio/models/__init__.py b/src/unstract/api_deployments/_sdk_docstudio/models/__init__.py new file mode 100644 index 0000000..e04b133 --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/models/__init__.py @@ -0,0 +1,24 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""Contains all the data models used in inputs/outputs""" + +from .acknowledged_response import AcknowledgedResponse +from .error_detail import ErrorDetail +from .error_response import ErrorResponse +from .error_type import ErrorType +from .execute_request import ExecuteRequest +from .execute_response import ExecuteResponse +from .execution_message import ExecutionMessage +from .file_result import FileResult +from .status_response import StatusResponse + +__all__ = ( + "AcknowledgedResponse", + "ErrorDetail", + "ErrorResponse", + "ErrorType", + "ExecuteRequest", + "ExecuteResponse", + "ExecutionMessage", + "FileResult", + "StatusResponse", +) diff --git a/src/unstract/api_deployments/_sdk_docstudio/models/acknowledged_response.py b/src/unstract/api_deployments/_sdk_docstudio/models/acknowledged_response.py new file mode 100644 index 0000000..bb49a51 --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/models/acknowledged_response.py @@ -0,0 +1,71 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="AcknowledgedResponse") + + +@_attrs_define +class AcknowledgedResponse: + """The execution's result was handed to an earlier call and discarded. + + Attributes: + message (str): + status (str): + """ + + message: str + status: str + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + message = self.message + + status = self.status + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "message": message, + "status": status, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + message = d.pop("message") + + status = d.pop("status") + + acknowledged_response = cls( + message=message, + status=status, + ) + + acknowledged_response.additional_properties = d + return acknowledged_response + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/_sdk_docstudio/models/error_detail.py b/src/unstract/api_deployments/_sdk_docstudio/models/error_detail.py new file mode 100644 index 0000000..b1ed1ec --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/models/error_detail.py @@ -0,0 +1,86 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="ErrorDetail") + + +@_attrs_define +class ErrorDetail: + """One problem found with the request. + + Attributes: + attr (None | str): The request field the problem belongs to, when it belongs to one. + code (str): Machine-readable problem identifier. + detail (str): Human-readable description. + """ + + attr: None | str + code: str + detail: str + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + attr: None | str + attr = self.attr + + code = self.code + + detail = self.detail + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "attr": attr, + "code": code, + "detail": detail, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + + def _parse_attr(data: object) -> None | str: + if data is None: + return data + return cast(None | str, data) + + attr = _parse_attr(d.pop("attr")) + + code = d.pop("code") + + detail = d.pop("detail") + + error_detail = cls( + attr=attr, + code=code, + detail=detail, + ) + + error_detail.additional_properties = d + return error_detail + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/_sdk_docstudio/models/error_response.py b/src/unstract/api_deployments/_sdk_docstudio/models/error_response.py new file mode 100644 index 0000000..f1efa2c --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/models/error_response.py @@ -0,0 +1,92 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.error_type import ErrorType, check_error_type + +if TYPE_CHECKING: + from ..models.error_detail import ErrorDetail + + +T = TypeVar("T", bound="ErrorResponse") + + +@_attrs_define +class ErrorResponse: + """The body of a rejected request. + + Produced by the project-wide exception handler, so its shape is the same + for every failure listed against an operation. + + Attributes: + errors (list[ErrorDetail]): + type_ (ErrorType): * `validation_error` - validation_error + * `client_error` - client_error + * `server_error` - server_error + """ + + errors: list[ErrorDetail] + type_: ErrorType + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + errors = [] + for errors_item_data in self.errors: + errors_item = errors_item_data.to_dict() + errors.append(errors_item) + + type_: str = self.type_ + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "errors": errors, + "type": type_, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.error_detail import ErrorDetail + + d = dict(src_dict) + errors = [] + _errors = d.pop("errors") + for errors_item_data in _errors: + errors_item = ErrorDetail.from_dict(errors_item_data) + + errors.append(errors_item) + + type_ = check_error_type(d.pop("type")) + + error_response = cls( + errors=errors, + type_=type_, + ) + + error_response.additional_properties = d + return error_response + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/_sdk_docstudio/models/error_type.py b/src/unstract/api_deployments/_sdk_docstudio/models/error_type.py new file mode 100644 index 0000000..837eb20 --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/models/error_type.py @@ -0,0 +1,18 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from typing import Literal + +ErrorType = Literal["client_error", "server_error", "validation_error"] + +ERROR_TYPE_VALUES: set[ErrorType] = { + "client_error", + "server_error", + "validation_error", +} + + +def check_error_type(value: str) -> ErrorType: + if value in ERROR_TYPE_VALUES: + return value + raise TypeError( + f"Unexpected value {value!r}. Expected one of {ERROR_TYPE_VALUES!r}" + ) diff --git a/src/unstract/api_deployments/_sdk_docstudio/models/execute_request.py b/src/unstract/api_deployments/_sdk_docstudio/models/execute_request.py new file mode 100644 index 0000000..db0e7f9 --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/models/execute_request.py @@ -0,0 +1,328 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from io import BytesIO +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from .. import types +from ..types import UNSET, File, FileTypes, Unset + +T = TypeVar("T", bound="ExecuteRequest") + + +@_attrs_define +class ExecuteRequest: + """The documents to run, and the options that shape the result. + + Supply `files`, `presigned_urls`, or both. + + Attributes: + custom_data (Any | Unset): + files (list[File] | Unset): + hitl_packet_id (None | str | Unset): Groups documents reviewed together into one packet. Requires the enterprise + manual-review capability; an installation without it rejects the request with 400. + hitl_queue_name (None | str | Unset): Document class name for the manual review queue. Requires the enterprise + manual-review capability; an installation without it rejects the request with 400. + include_extracted_text (bool | Unset): Default: False. + include_metadata (bool | Unset): Default: False. + include_metrics (bool | Unset): Default: False. + llm_profile_id (None | str | Unset): + presigned_urls (list[str] | Unset): + tags (str | Unset): Comma-separated list of tag names (EX:'tag1,tag2-name,tag3_name') Default: ''. + timeout (int | Unset): Default: -1. + use_file_history (bool | Unset): Default: False. + """ + + custom_data: Any | Unset = UNSET + files: list[File] | Unset = UNSET + hitl_packet_id: None | str | Unset = UNSET + hitl_queue_name: None | str | Unset = UNSET + include_extracted_text: bool | Unset = False + include_metadata: bool | Unset = False + include_metrics: bool | Unset = False + llm_profile_id: None | str | Unset = UNSET + presigned_urls: list[str] | Unset = UNSET + tags: str | Unset = "" + timeout: int | Unset = -1 + use_file_history: bool | Unset = False + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + custom_data = self.custom_data + + files: list[FileTypes] | Unset = UNSET + if not isinstance(self.files, Unset): + files = [] + for files_item_data in self.files: + files_item = files_item_data.to_tuple() + + files.append(files_item) + + hitl_packet_id: None | str | Unset + if isinstance(self.hitl_packet_id, Unset): + hitl_packet_id = UNSET + else: + hitl_packet_id = self.hitl_packet_id + + hitl_queue_name: None | str | Unset + if isinstance(self.hitl_queue_name, Unset): + hitl_queue_name = UNSET + else: + hitl_queue_name = self.hitl_queue_name + + include_extracted_text = self.include_extracted_text + + include_metadata = self.include_metadata + + include_metrics = self.include_metrics + + llm_profile_id: None | str | Unset + if isinstance(self.llm_profile_id, Unset): + llm_profile_id = UNSET + else: + llm_profile_id = self.llm_profile_id + + presigned_urls: list[str] | Unset = UNSET + if not isinstance(self.presigned_urls, Unset): + presigned_urls = self.presigned_urls + + tags = self.tags + + timeout = self.timeout + + use_file_history = self.use_file_history + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if custom_data is not UNSET: + field_dict["custom_data"] = custom_data + if files is not UNSET: + field_dict["files"] = files + if hitl_packet_id is not UNSET: + field_dict["hitl_packet_id"] = hitl_packet_id + if hitl_queue_name is not UNSET: + field_dict["hitl_queue_name"] = hitl_queue_name + if include_extracted_text is not UNSET: + field_dict["include_extracted_text"] = include_extracted_text + if include_metadata is not UNSET: + field_dict["include_metadata"] = include_metadata + if include_metrics is not UNSET: + field_dict["include_metrics"] = include_metrics + if llm_profile_id is not UNSET: + field_dict["llm_profile_id"] = llm_profile_id + if presigned_urls is not UNSET: + field_dict["presigned_urls"] = presigned_urls + if tags is not UNSET: + field_dict["tags"] = tags + if timeout is not UNSET: + field_dict["timeout"] = timeout + if use_file_history is not UNSET: + field_dict["use_file_history"] = use_file_history + + return field_dict + + def to_multipart(self) -> types.RequestFiles: + files: types.RequestFiles = [] + + if not isinstance(self.custom_data, Unset): + files.append( + ("custom_data", (None, str(self.custom_data).encode(), "text/plain")) + ) + + if not isinstance(self.files, Unset): + for files_item_element in self.files: + files.append(("files", files_item_element.to_tuple())) + + if not isinstance(self.hitl_packet_id, Unset): + if isinstance(self.hitl_packet_id, str): + files.append( + ( + "hitl_packet_id", + (None, str(self.hitl_packet_id).encode(), "text/plain"), + ) + ) + else: + files.append( + ( + "hitl_packet_id", + (None, str(self.hitl_packet_id).encode(), "text/plain"), + ) + ) + + if not isinstance(self.hitl_queue_name, Unset): + if isinstance(self.hitl_queue_name, str): + files.append( + ( + "hitl_queue_name", + (None, str(self.hitl_queue_name).encode(), "text/plain"), + ) + ) + else: + files.append( + ( + "hitl_queue_name", + (None, str(self.hitl_queue_name).encode(), "text/plain"), + ) + ) + + if not isinstance(self.include_extracted_text, Unset): + files.append( + ( + "include_extracted_text", + (None, str(self.include_extracted_text).encode(), "text/plain"), + ) + ) + + if not isinstance(self.include_metadata, Unset): + files.append( + ( + "include_metadata", + (None, str(self.include_metadata).encode(), "text/plain"), + ) + ) + + if not isinstance(self.include_metrics, Unset): + files.append( + ( + "include_metrics", + (None, str(self.include_metrics).encode(), "text/plain"), + ) + ) + + if not isinstance(self.llm_profile_id, Unset): + if isinstance(self.llm_profile_id, str): + files.append( + ( + "llm_profile_id", + (None, str(self.llm_profile_id).encode(), "text/plain"), + ) + ) + else: + files.append( + ( + "llm_profile_id", + (None, str(self.llm_profile_id).encode(), "text/plain"), + ) + ) + + if not isinstance(self.presigned_urls, Unset): + for presigned_urls_item_element in self.presigned_urls: + files.append( + ( + "presigned_urls", + (None, str(presigned_urls_item_element).encode(), "text/plain"), + ) + ) + + if not isinstance(self.tags, Unset): + files.append(("tags", (None, str(self.tags).encode(), "text/plain"))) + + if not isinstance(self.timeout, Unset): + files.append(("timeout", (None, str(self.timeout).encode(), "text/plain"))) + + if not isinstance(self.use_file_history, Unset): + files.append( + ( + "use_file_history", + (None, str(self.use_file_history).encode(), "text/plain"), + ) + ) + + for prop_name, prop in self.additional_properties.items(): + files.append((prop_name, (None, str(prop).encode(), "text/plain"))) + + return files + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + custom_data = d.pop("custom_data", UNSET) + + _files = d.pop("files", UNSET) + files: list[File] | Unset = UNSET + if _files is not UNSET: + files = [] + for files_item_data in _files: + files_item = File(payload=BytesIO(files_item_data)) + + files.append(files_item) + + def _parse_hitl_packet_id(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + hitl_packet_id = _parse_hitl_packet_id(d.pop("hitl_packet_id", UNSET)) + + def _parse_hitl_queue_name(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + hitl_queue_name = _parse_hitl_queue_name(d.pop("hitl_queue_name", UNSET)) + + include_extracted_text = d.pop("include_extracted_text", UNSET) + + include_metadata = d.pop("include_metadata", UNSET) + + include_metrics = d.pop("include_metrics", UNSET) + + def _parse_llm_profile_id(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + llm_profile_id = _parse_llm_profile_id(d.pop("llm_profile_id", UNSET)) + + presigned_urls = cast(list[str], d.pop("presigned_urls", UNSET)) + + tags = d.pop("tags", UNSET) + + timeout = d.pop("timeout", UNSET) + + use_file_history = d.pop("use_file_history", UNSET) + + execute_request = cls( + custom_data=custom_data, + files=files, + hitl_packet_id=hitl_packet_id, + hitl_queue_name=hitl_queue_name, + include_extracted_text=include_extracted_text, + include_metadata=include_metadata, + include_metrics=include_metrics, + llm_profile_id=llm_profile_id, + presigned_urls=presigned_urls, + tags=tags, + timeout=timeout, + use_file_history=use_file_history, + ) + + execute_request.additional_properties = d + return execute_request + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/_sdk_docstudio/models/execute_response.py b/src/unstract/api_deployments/_sdk_docstudio/models/execute_response.py new file mode 100644 index 0000000..035eb5a --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/models/execute_response.py @@ -0,0 +1,69 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.execution_message import ExecutionMessage + + +T = TypeVar("T", bound="ExecuteResponse") + + +@_attrs_define +class ExecuteResponse: + """ + Attributes: + message (ExecutionMessage): The execution's identity and, once it has finished, its per-file + results. + """ + + message: ExecutionMessage + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + message = self.message.to_dict() + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "message": message, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.execution_message import ExecutionMessage + + d = dict(src_dict) + message = ExecutionMessage.from_dict(d.pop("message")) + + execute_response = cls( + message=message, + ) + + execute_response.additional_properties = d + return execute_response + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/_sdk_docstudio/models/execution_message.py b/src/unstract/api_deployments/_sdk_docstudio/models/execution_message.py new file mode 100644 index 0000000..d4dd69f --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/models/execution_message.py @@ -0,0 +1,159 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.file_result import FileResult + + +T = TypeVar("T", bound="ExecutionMessage") + + +@_attrs_define +class ExecutionMessage: + """The execution's identity and, once it has finished, its per-file + results. + + Attributes: + execution_id (str): + execution_status (str): + error (None | str | Unset): + result (list[FileResult] | None | Unset): + status_api (None | str | Unset): + """ + + execution_id: str + execution_status: str + error: None | str | Unset = UNSET + result: list[FileResult] | None | Unset = UNSET + status_api: None | str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + execution_id = self.execution_id + + execution_status = self.execution_status + + error: None | str | Unset + if isinstance(self.error, Unset): + error = UNSET + else: + error = self.error + + result: list[dict[str, Any]] | None | Unset + if isinstance(self.result, Unset): + result = UNSET + elif isinstance(self.result, list): + result = [] + for result_type_0_item_data in self.result: + result_type_0_item = result_type_0_item_data.to_dict() + result.append(result_type_0_item) + + else: + result = self.result + + status_api: None | str | Unset + if isinstance(self.status_api, Unset): + status_api = UNSET + else: + status_api = self.status_api + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "execution_id": execution_id, + "execution_status": execution_status, + } + ) + if error is not UNSET: + field_dict["error"] = error + if result is not UNSET: + field_dict["result"] = result + if status_api is not UNSET: + field_dict["status_api"] = status_api + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.file_result import FileResult + + d = dict(src_dict) + execution_id = d.pop("execution_id") + + execution_status = d.pop("execution_status") + + def _parse_error(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + error = _parse_error(d.pop("error", UNSET)) + + def _parse_result(data: object) -> list[FileResult] | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, list): + raise TypeError() + result_type_0 = [] + _result_type_0 = data + for result_type_0_item_data in _result_type_0: + result_type_0_item = FileResult.from_dict(result_type_0_item_data) + + result_type_0.append(result_type_0_item) + + return result_type_0 + except (TypeError, ValueError, AttributeError, KeyError): + pass + return cast(list[FileResult] | None | Unset, data) + + result = _parse_result(d.pop("result", UNSET)) + + def _parse_status_api(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + status_api = _parse_status_api(d.pop("status_api", UNSET)) + + execution_message = cls( + execution_id=execution_id, + execution_status=execution_status, + error=error, + result=result, + status_api=status_api, + ) + + execution_message.additional_properties = d + return execution_message + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/_sdk_docstudio/models/file_result.py b/src/unstract/api_deployments/_sdk_docstudio/models/file_result.py new file mode 100644 index 0000000..744d30f --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/models/file_result.py @@ -0,0 +1,156 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="FileResult") + + +@_attrs_define +class FileResult: + """One input document's outcome. + + Every key is present on every item; the ones that depend on the request + options or on the outcome are sent as `null` when they do not apply. + + Attributes: + file (str): + error (None | str | Unset): + extracted_text (None | str | Unset): The document's full extracted text. Sent only when the request set + `include_extracted_text`. + file_execution_id (None | str | Unset): + metadata (Any | Unset): + result (Any | Unset): + status (str | Unset): + """ + + file: str + error: None | str | Unset = UNSET + extracted_text: None | str | Unset = UNSET + file_execution_id: None | str | Unset = UNSET + metadata: Any | Unset = UNSET + result: Any | Unset = UNSET + status: str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + file = self.file + + error: None | str | Unset + if isinstance(self.error, Unset): + error = UNSET + else: + error = self.error + + extracted_text: None | str | Unset + if isinstance(self.extracted_text, Unset): + extracted_text = UNSET + else: + extracted_text = self.extracted_text + + file_execution_id: None | str | Unset + if isinstance(self.file_execution_id, Unset): + file_execution_id = UNSET + else: + file_execution_id = self.file_execution_id + + metadata = self.metadata + + result = self.result + + status = self.status + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "file": file, + } + ) + if error is not UNSET: + field_dict["error"] = error + if extracted_text is not UNSET: + field_dict["extracted_text"] = extracted_text + if file_execution_id is not UNSET: + field_dict["file_execution_id"] = file_execution_id + if metadata is not UNSET: + field_dict["metadata"] = metadata + if result is not UNSET: + field_dict["result"] = result + if status is not UNSET: + field_dict["status"] = status + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + file = d.pop("file") + + def _parse_error(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + error = _parse_error(d.pop("error", UNSET)) + + def _parse_extracted_text(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + extracted_text = _parse_extracted_text(d.pop("extracted_text", UNSET)) + + def _parse_file_execution_id(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + file_execution_id = _parse_file_execution_id(d.pop("file_execution_id", UNSET)) + + metadata = d.pop("metadata", UNSET) + + result = d.pop("result", UNSET) + + status = d.pop("status", UNSET) + + file_result = cls( + file=file, + error=error, + extracted_text=extracted_text, + file_execution_id=file_execution_id, + metadata=metadata, + result=result, + status=status, + ) + + file_result.additional_properties = d + return file_result + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/_sdk_docstudio/models/status_response.py b/src/unstract/api_deployments/_sdk_docstudio/models/status_response.py new file mode 100644 index 0000000..8073788 --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/models/status_response.py @@ -0,0 +1,109 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.file_result import FileResult + + +T = TypeVar("T", bound="StatusResponse") + + +@_attrs_define +class StatusResponse: + """ + Attributes: + status (str): + message (list[FileResult] | None | Unset): + """ + + status: str + message: list[FileResult] | None | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + status = self.status + + message: list[dict[str, Any]] | None | Unset + if isinstance(self.message, Unset): + message = UNSET + elif isinstance(self.message, list): + message = [] + for message_type_0_item_data in self.message: + message_type_0_item = message_type_0_item_data.to_dict() + message.append(message_type_0_item) + + else: + message = self.message + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "status": status, + } + ) + if message is not UNSET: + field_dict["message"] = message + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.file_result import FileResult + + d = dict(src_dict) + status = d.pop("status") + + def _parse_message(data: object) -> list[FileResult] | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, list): + raise TypeError() + message_type_0 = [] + _message_type_0 = data + for message_type_0_item_data in _message_type_0: + message_type_0_item = FileResult.from_dict(message_type_0_item_data) + + message_type_0.append(message_type_0_item) + + return message_type_0 + except (TypeError, ValueError, AttributeError, KeyError): + pass + return cast(list[FileResult] | None | Unset, data) + + message = _parse_message(d.pop("message", UNSET)) + + status_response = cls( + status=status, + message=message, + ) + + status_response.additional_properties = d + return status_response + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/_sdk_docstudio/types.py b/src/unstract/api_deployments/_sdk_docstudio/types.py new file mode 100644 index 0000000..3ca05bc --- /dev/null +++ b/src/unstract/api_deployments/_sdk_docstudio/types.py @@ -0,0 +1,55 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""Contains some shared types for properties""" + +from collections.abc import Mapping, MutableMapping +from http import HTTPStatus +from typing import IO, BinaryIO, Generic, Literal, TypeVar + +from attrs import define + + +class Unset: + def __bool__(self) -> Literal[False]: + return False + + +UNSET: Unset = Unset() + +# The types that `httpx.Client(files=)` can accept, copied from that library. +FileContent = IO[bytes] | bytes | str +FileTypes = ( + # (filename, file (or bytes), content_type) + tuple[str | None, FileContent, str | None] + # (filename, file (or bytes), content_type, headers) + | tuple[str | None, FileContent, str | None, Mapping[str, str]] +) +RequestFiles = list[tuple[str, FileTypes]] + + +@define +class File: + """Contains information for file uploads""" + + payload: BinaryIO + file_name: str | None = None + mime_type: str | None = None + + def to_tuple(self) -> FileTypes: + """Return a tuple representation that httpx will accept for multipart/form-data""" + return self.file_name, self.payload, self.mime_type + + +T = TypeVar("T") + + +@define +class Response(Generic[T]): + """A response from an endpoint""" + + status_code: HTTPStatus + content: bytes + headers: MutableMapping[str, str] + parsed: T | None + + +__all__ = ["UNSET", "File", "FileTypes", "RequestFiles", "Response", "Unset"] diff --git a/src/unstract/api_deployments/client.py b/src/unstract/api_deployments/client.py index e39731d..156bff5 100644 --- a/src/unstract/api_deployments/client.py +++ b/src/unstract/api_deployments/client.py @@ -7,14 +7,34 @@ APIDeploymentsClient class. """ +import json import logging import ntpath import os +import threading import time -from urllib.parse import urlparse - -import requests -from requests.exceptions import ConnectionError, JSONDecodeError, Timeout +from typing import Any +from urllib.parse import parse_qs, urljoin, urlparse + +import attrs +import httpx + +# `requests` is still the transport for the `unstract.clone` subpackage, and it +# supplies the exception classes callers catch by name around these calls. The +# httpx equivalents are not subclasses of those, so they are translated at the +# transport seam. +from requests.exceptions import ( + ConnectionError, + ConnectTimeout, + ContentDecodingError, + InvalidHeader, + InvalidURL, + MissingSchema, + ProxyError, + ReadTimeout, + Timeout, + TooManyRedirects, +) from tenacity import ( RetryCallState, Retrying, @@ -25,9 +45,128 @@ ) from tenacity.wait import wait_base +from unstract.api_deployments._sdk_docstudio import AuthenticatedClient +from unstract.api_deployments._sdk_docstudio.api.deployment import execute, status +from unstract.api_deployments._sdk_docstudio.models import ExecuteRequest +from unstract.api_deployments._sdk_docstudio.types import UNSET, File, Unset from unstract.api_deployments.utils import UnstractUtils +def _translate_transport_errors(fn, *args, **kwargs): + """Re-raise httpx transport failures as their ``requests`` equivalents. + + Callers document and catch the ``requests`` classes, and the retry policy + keys off them too, so the class chosen here decides whether a failure is + retried. Every branch is ordered before the base class it derives from. + ``RequestError`` is the catch-all for the transport subtree, which is where + a novel failure appears. Failures no retry can fix are pulled out above it: + ``UnsupportedProtocol`` and ``LocalProtocolError``. httpx puts three families + outside the subtree: ``InvalidURL``, translated here because ``requests`` + raised its own, and ``StreamError`` and ``CookieConflict``, which propagate + as themselves. + """ + try: + return fn(*args, **kwargs) + except httpx.ConnectTimeout as e: + # ConnectTimeout is both a ConnectionError and a Timeout; the plain + # Timeout httpx implies would stop matching half the callers. + raise ConnectTimeout(str(e)) from e + except httpx.ReadTimeout as e: + raise ReadTimeout(str(e)) from e + except (httpx.WriteTimeout, httpx.PoolTimeout) as e: + # Neither had a Timeout equivalent: a send that failed and a pool that + # could not hand out a connection both surfaced as ConnectionError. + raise ConnectionError(str(e)) from e + except httpx.TimeoutException as e: + raise Timeout(str(e)) from e + except httpx.UnsupportedProtocol as e: + # A URL rejected before any socket is opened. Deliberately not a + # ConnectionError: retrying a malformed URL cannot start working. + raise MissingSchema(str(e)) from e + except httpx.ProxyError as e: + raise ProxyError(str(e)) from e + except httpx.ConnectError as e: + raise ConnectionError(str(e)) from e + except httpx.TooManyRedirects as e: + raise TooManyRedirects(str(e)) from e + except httpx.DecodingError as e: + raise ContentDecodingError(str(e)) from e + except httpx.InvalidURL as e: + raise InvalidURL(str(e)) from e + except httpx.LocalProtocolError as e: + # The request cannot be written as composed -- an api_key carrying a + # newline is the everyday cause. Deliberately not a ConnectionError: + # re-sending the identical request cannot start working. + raise InvalidHeader(str(e)) from e + except httpx.RequestError as e: + raise ConnectionError(str(e)) from e + + +def _query_value(url: str, key: str) -> str: + """Read one required query parameter out of a URL, absolute or relative. + + Empty is not a usable value here: it polls for an execution the service + cannot identify and reports whatever it makes of a blank id. + """ + parsed = urlparse(url) + value = parse_qs(parsed.query).get(key, [""])[0] + if not value: + # Only the path is reported: the query is the service's to shape, and + # the documented usage prints this exception straight to a log. + raise APIDeploymentsClientException( + f"No {key} in the query of {parsed.path!r}. The status endpoint the " + "service returned carries it; pass that endpoint unmodified." + ) + return value + + +def _forwarded_query(url: str) -> dict[str, str]: + """Everything else the service put on the status endpoint. + + Today it sends only the execution id, but the endpoint is the service's + instruction for reaching this execution: dropping a parameter it decided to + add -- a region hint, a cursor -- polls somewhere the execution is not. + """ + return { + key: values[-1] + for key, values in parse_qs(urlparse(url).query, keep_blank_values=True).items() + if key != "execution_id" + } + + +#: How much of an unparseable error body is worth reporting to a caller. +_ERROR_TEXT_LIMIT = 500 + + +def _error_text(body: Any, response) -> str: + """The reason a non-2xx carries, for a body that is not the endpoint's own + envelope. + + A refused request answers through the API's exception handler with + ``{"type", "errors": [{"code", "detail", "attr"}]}``. Anything in front of + the service -- a proxy, a gateway -- answers however it likes, so a single + readable string is looked for before falling back to the body itself. + Neither is the envelope the result fields are read out of, so without this + the reason is dropped and the caller sees an empty error next to a bare + status code. + """ + if isinstance(body, dict): + errors = body.get("errors") + if isinstance(errors, list): + details = [ + str(item["detail"]) + for item in errors + if isinstance(item, dict) and item.get("detail") + ] + if details: + return "; ".join(details) + for key in ("message", "detail", "error"): + value = body.get(key) + if isinstance(value, str) and value: + return value + return (response.text or "").strip()[:_ERROR_TEXT_LIMIT] + + class APIDeploymentsClientException(Exception): """A class to handle exceptions raised by the APIClient class.""" @@ -76,6 +215,18 @@ def __call__(self, retry_state: RetryCallState) -> float: return self._exp_jitter(retry_state) +#: Request fields this client generates itself. Any other field the generated +#: builder writes is reset to UNSET (body) or filtered out (query) before the +#: request goes out, so a new parameter must be added here to be sent at all. +#: This bounds what the client generates, not the whole outgoing query: the +#: status endpoint's own query is forwarded on top by design (see +#: ``_forwarded_query``). +_EXECUTE_SEND_ONLY = frozenset( + {"timeout", "include_metadata", "files", "additional_properties"} +) +_STATUS_SEND_ONLY = frozenset({"execution_id", "include_metadata"}) + + class APIDeploymentsClient: """A class to invoke APIs deployed on the Unstract platform.""" @@ -104,18 +255,27 @@ def __init__( max_delay: float = 60.0, backoff_factor: float = 2.0, jitter: float = 1.0, + *, + transport_timeout: float | None = None, ): """Initializes the APIClient class. Args: api_key (str): The API key to authenticate the API request. - api_timeout (int): The timeout to wait for the API response. + api_timeout (int): Backend execution mode sent with the request — + see ``timeout`` on ``structure_file``. ``0`` or below queues the + execution and returns; above it the call runs synchronously and + the value bounds how long the backend waits. logging_level (str): The logging level to log messages. max_retries (int): Maximum number of retry attempts for failed requests. initial_delay (float): Initial delay in seconds before the first retry. max_delay (float): Maximum delay in seconds between retries. backoff_factor (float): Multiplier applied to delay for each retry. jitter (float): Maximum additive jitter in seconds added to each delay. + transport_timeout (float | None): Socket timeout in seconds. Unset + means a stalled connection blocks forever, which is what the + released client did; ``api_timeout`` cannot serve here because + it is an execution mode, not a socket timeout. """ if logging_level == "": logging_level = os.getenv("UNSTRACT_API_CLIENT_LOGGING_LEVEL", "INFO") @@ -147,6 +307,9 @@ def __init__( self.max_delay = max_delay self.backoff_factor = backoff_factor self.jitter = jitter + self.transport_timeout = transport_timeout + self._transport_client = None + self._transport_lock = threading.Lock() def _is_retryable_status(self, status_code: int) -> bool: """Checks whether a status code should trigger a retry. @@ -169,6 +332,143 @@ def __save_base_url(self, full_url: str): self.base_url = parsed_url.scheme + "://" + parsed_url.netloc self.logger.debug("Base URL: " + self.base_url) + @property + def _transport(self): + """The HTTP client, built on first use. + + Untimed by default, matching the previous behaviour. ``api_timeout`` is + a backend execution mode (0 selects async execution), never a socket + timeout; feeding it to the transport fails deep in the connection layer + for the negative values the API accepts. ``transport_timeout`` is the + way to bound a stalled connection. + + Built under a lock: two threads racing the first call would otherwise + each build a pool and one would be dropped still holding its sockets. + """ + if self._transport_client is None: + with self._transport_lock: + if self._transport_client is None: + self._transport_client = AuthenticatedClient( + base_url=self.base_url, + token=self.api_key, + verify_ssl=self.verify, + timeout=httpx.Timeout(self.transport_timeout), + raise_on_unexpected_status=False, + # The previous transport followed redirects. Without this + # a 30x from a load balancer is read as a terminal result + # with no status, which a poll loop reports as a + # finished-and-empty job. + follow_redirects=True, + ) + return self._transport_client + + def close(self) -> None: + """Release the pooled connections this client holds. + + The transport is kept between calls so connections are reused; nothing + else releases its sockets, and a client built per job would otherwise + accumulate pools until each instance is collected. Safe to call more + than once, and the next request builds a fresh transport. + """ + with self._transport_lock: + transport, self._transport_client = self._transport_client, None + if transport is not None: + transport.get_httpx_client().close() + + def __enter__(self) -> "APIDeploymentsClient": + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.close() + + @property + def _deployment_route(self) -> tuple[str, str]: + """Organisation and API name, from the deployment URL's last two + segments.""" + segments = urlparse(self.api_url).path.strip("/").split("/") + if len(segments) < 2: + raise APIDeploymentsClientException( + f"Cannot derive organisation and API name from api_url: {self.api_url}" + ) + return segments[-2], segments[-1] + + def _spec_route(self) -> str: + """The path the spec routes a poll to, or ``""`` when the deployment URL + carries no organisation and API name to build one from. + + Built through the generated builder so it follows the spec rather than a + copy of it. + """ + try: + org_name, api_name = self._deployment_route + except APIDeploymentsClientException: + return "" + return status._get_kwargs(org_name, api_name, execution_id="")["url"] + + def _status_url(self, endpoint: str) -> str: + """Absolute URL to poll, under the deployment's own path prefix. + + ``base_url`` is scheme and host only, so a deployment served under a path + prefix would execute -- the execute call sends the caller's URL verbatim + -- and then never poll. The prefix is whatever precedes the spec route + inside the deployment URL. Where the two do not line up there is no + prefix to derive, and the path the service returned is used as it came: + a guessed path polls nothing, and the execution behind it has already + been paid for. + + A deployment URL with no organisation and API name in it -- an ingress + rewrite short enough to have neither -- has no route to line up against + and takes that same branch. The released client polled those, and the + execution has already been submitted by the time this runs. + + Only the path is taken. A scheme and host in the reply would otherwise + decide where the deployment key is sent, and the reply is not the thing + that gets to choose that. + """ + path = self._spec_route() + route = path.rstrip("/") + prefix = urlparse(self.api_url).path.rstrip("/") + if route and prefix.endswith(route): + return self.base_url + prefix[: -len(route)] + path + # Joined rather than concatenated: the query travels as params. + return urljoin( + self.base_url, + urlparse(endpoint)._replace(scheme="", netloc="", query="").geturl(), + ) + + def _send(self, method: str, url: str, **kwargs) -> httpx.Response: + """Issue one request, translating transport failures on the way out. + + Translation happens here rather than around the retry loop, so + the retry policy still sees the exception types it is configured + to retry. + + The credential is read per request rather than captured with the + transport, so assigning ``api_key`` takes effect on the next call the + way it did when every call built its own header. + """ + kwargs["headers"] = { + **(kwargs.get("headers") or {}), + "Authorization": f"Bearer {self.api_key}", + } + return _translate_transport_errors( + self._transport.get_httpx_client().request, method, url, **kwargs + ) + + @staticmethod + def _read_body(response): + """Read the JSON body directly, never the generated response model. + + A model is only built for the statuses the spec declares, and error + bodies are typed loosely, so an undeclared status or any error response + has no usable model. ``None`` means there is nothing to read: either the + body was not JSON, or it was the JSON literal ``null``. + """ + try: + return response.json() + except ValueError: + return None + @staticmethod def _rewind_files(files): """Rewinds file objects so they can be re-sent on retry.""" @@ -180,7 +480,7 @@ def _rewind_files(files): if hasattr(file_obj[1], "seek"): file_obj[1].seek(0) - def _request_with_retry(self, method: str, url: str, **kwargs) -> requests.Response: + def _request_with_retry(self, method: str, url: str, **kwargs) -> httpx.Response: """Makes an HTTP request with exponential backoff retry logic. Uses ``tenacity`` with additive jitter and Retry-After support. @@ -188,10 +488,10 @@ def _request_with_retry(self, method: str, url: str, **kwargs) -> requests.Respo Args: method (str): The HTTP method (e.g., "GET", "POST"). url (str): The request URL. - **kwargs: Additional keyword arguments passed to requests.request(). + **kwargs: Additional keyword arguments passed to the transport. Returns: - requests.Response: The response from the request. + The response from the request. Raises: ConnectionError: If a connection error persists after all retries. @@ -267,13 +567,50 @@ def _retry_error_callback(retry_state: RetryCallState): reraise=False, ) - return retrier(requests.request, method, url, **kwargs) + return retrier(self._send, method, url, **kwargs) - def structure_file(self, file_paths: list[str]) -> dict: + def structure_file( + self, + file_paths: list[str], + *, + timeout: int | Unset = UNSET, + include_metadata: bool | Unset = UNSET, + include_metrics: bool | Unset = UNSET, + include_extracted_text: bool | Unset = UNSET, + use_file_history: bool | Unset = UNSET, + tags: str | Unset = UNSET, + llm_profile_id: str | None | Unset = UNSET, + hitl_queue_name: str | None | Unset = UNSET, + hitl_packet_id: str | None | Unset = UNSET, + presigned_urls: list[str] | Unset = UNSET, + custom_data: Any | Unset = UNSET, + ) -> dict: """Invokes the API deployed on the Unstract platform. + The keyword arguments are the request parameters the deployment accepts, + named as the API names them. One left unset is not sent at all, so the + server picks its own default; ``timeout`` and ``include_metadata`` fall + back to the values given at construction. + Args: file_paths (list[str]): The file path to the file to be uploaded. + timeout (int): Execution mode — ``0`` or below queues the execution + and returns immediately; above it the call runs synchronously. + include_metadata (bool): Include metadata in the result. + include_metrics (bool): Include metrics in the result. + include_extracted_text (bool): Include the extracted text. + use_file_history (bool): Reuse a previous result for the same file. + tags (str): Comma-separated tag names. + llm_profile_id (str): LLM profile to override the deployment's. + hitl_queue_name (str): Human-in-the-loop queue to route the file to. + hitl_packet_id (str): Human-in-the-loop packet to attach the file to. + presigned_urls (list[str]): URLs to fetch the inputs from. + custom_data (Any): Arbitrary JSON. The service returns it under + each result item's ``metadata.custom_data``, which is server + behaviour: the spec carries the field on the request only, so + the round trip is not declared and nothing here pins it. + Anything that is not already a string is serialised to JSON + before it is sent. Returns: dict: The response from the API. @@ -281,63 +618,114 @@ def structure_file(self, file_paths: list[str]) -> dict: self.logger.debug("Invoking API: " + self.api_url) self.logger.debug("File paths: " + str(file_paths)) - headers = { - "Authorization": "Bearer " + self.api_key, + requested = { + "timeout": timeout, + "include_metadata": include_metadata, + "include_metrics": include_metrics, + "include_extracted_text": include_extracted_text, + "use_file_history": use_file_history, + "tags": tags, + "llm_profile_id": llm_profile_id, + "hitl_queue_name": hitl_queue_name, + "hitl_packet_id": hitl_packet_id, + "presigned_urls": presigned_urls, + "custom_data": custom_data, } - - form_data = { + # ``None`` is dropped with ``UNSET``: these are optional overrides, and a + # form field carries no null, so one would go out as the string "None" + # for the service to look up. + requested = { + k: v + for k, v in requested.items() + if not isinstance(v, Unset) and v is not None + } + if "custom_data" in requested and not isinstance(requested["custom_data"], str): + # A form field carries text, and the generated encoder writes + # ``str(value)`` -- a Python repr, which the server's JSON field + # cannot parse. Strings are passed through, so a caller already + # serialising its own payload is unaffected. + requested["custom_data"] = json.dumps(requested["custom_data"]) + params = { "timeout": self.api_timeout, "include_metadata": self.include_metadata, + **requested, } + send_only = _EXECUTE_SEND_ONLY | requested.keys() - files = [] - + handles = [] try: for file_path in file_paths: - record = ( - "files", - ( - ntpath.basename(file_path), - open(file_path, "rb"), - "application/octet-stream", - ), - ) - files.append(record) - except FileNotFoundError as e: - raise APIDeploymentsClientException("File not found: " + str(e)) - - if self.api_timeout == 0: - # Async mode: server returns immediately after queuing. - # A 5xx means queuing failed — safe to retry. - response = self._request_with_retry( - "POST", - self.api_url, - headers=headers, - data=form_data, - files=files, - verify=self.verify, - ) - else: - # Sync mode: server blocks during processing. - # A 5xx may mean it processed but response was lost — don't retry - # to avoid duplicate executions. - response = requests.post( - self.api_url, - headers=headers, - data=form_data, - files=files, - verify=self.verify, + handles.append(open(file_path, "rb")) + except OSError as e: + # Every open failure, not just a missing file: a directory or an + # unreadable path would otherwise leave the handles opened so far + # held by the traceback, and reach the caller as a builtin rather + # than the exception this class documents. + for handle in handles: + handle.close() + reason = ( + "File not found" + if isinstance(e, FileNotFoundError) + else "Cannot read file" ) + raise APIDeploymentsClientException(f"{reason}: {e}") from e + + body = ExecuteRequest( + files=[ + File( + payload=handle, + file_name=ntpath.basename(file_path), + mime_type="application/octet-stream", + ) + for file_path, handle in zip(file_paths, handles) + ], + **params, + ) + # Only the fields this client sets are sent. Every other field carries the + # spec's declared default, and sending a default is not the same as + # omitting it: it pins a value the server would otherwise choose, and the + # two diverge the moment the server's own default changes. + for field in attrs.fields(ExecuteRequest): + if field.name not in send_only: + setattr(body, field.name, UNSET) + + # Placeholders: the generated builder only spends these on the URL, and + # the URL is discarded below in favour of the caller's own. Deriving a + # route here would reject deployment URLs the released client posted to. + request_kwargs = execute._get_kwargs("", "", body=body) + # The generated builder pins a fixed multipart boundary in the header. An + # uploaded file containing those bytes would break the encoding, so let + # the transport pick a random boundary instead. + request_kwargs.get("headers", {}).pop("Content-Type", None) + method = request_kwargs.pop("method") + request_kwargs.pop("url") + # The deployment URL is the caller's, sent back verbatim. Rebuilding it + # from the spec's path template drops any prefix the deployment is + # served under, which no route template can express. + url = self.api_url + + try: + if params["timeout"] <= 0: + # Zero and below only queue the execution, so a 5xx means + # queuing failed and retrying cannot duplicate work. ``-1`` is + # the API's own default for this, so it has to take this branch + # too. + response = self._request_with_retry(method, url, **request_kwargs) + else: + # The request runs the execution, so a 5xx may mean it ran and + # the response was lost: a retry would execute it twice. + response = self._send(method, url, **request_kwargs) + finally: + for handle in handles: + handle.close() self.logger.debug(response.status_code) self.logger.debug(response.text) # The returned object is wrapped in a "message" key. # Let's simplify the response. obj_to_return = {} - try: - response_data = response.json() - response_message = response_data.get("message", {}) - except JSONDecodeError: + response_data = self._read_body(response) + if response_data is None: self.logger.error( "Failed to decode JSON response. Raw response: %s", response.text, @@ -351,17 +739,13 @@ def structure_file(self, file_paths: list[str]) -> dict: "extraction_result": "", } return obj_to_return - if response.status_code == 401: - obj_to_return = { - "status_code": response.status_code, - "pending": False, - "execution_status": "", - "error": response_data.get("errors", [{}])[0].get( - "detail", "Unauthorized" - ), - "extraction_result": "", - } - return obj_to_return + # An error body carries no success envelope, and the shapes the API + # answers errors with put a string or a list where this reads a mapping. + response_message = ( + response_data.get("message") if isinstance(response_data, dict) else None + ) + if not isinstance(response_message, dict): + response_message = {} # If the execution status is pending, extract the execution ID from # the response and return it in the response. @@ -373,6 +757,16 @@ def structure_file(self, file_paths: list[str]) -> dict: error_message = response_message.get("error", "") extraction_result = response_message.get("result", "") status_api_endpoint = response_message.get("status_api") + if ( + not error_message + and not response_message + and not 200 <= response.status_code < 300 + ): + # Only a refused request needs this. A body carrying the endpoint's + # own envelope has already been read for its reason above, and a + # non-2xx that still carries one is reporting an execution state + # rather than refusing the request. + error_message = _error_text(response_data, response) obj_to_return = { "status_code": response.status_code, @@ -399,37 +793,80 @@ def structure_file(self, file_paths: list[str]) -> dict: return obj_to_return - def check_execution_status(self, status_check_api_endpoint: str) -> dict: + def check_execution_status( + self, + status_check_api_endpoint: str, + *, + include_metadata: bool | Unset = UNSET, + include_metrics: bool | Unset = UNSET, + include_extracted_text: bool | Unset = UNSET, + ) -> dict: """Checks the status of the execution. + The keyword arguments are the query parameters the endpoint accepts, + named as the API names them. One left unset is not sent at all, so the + server picks its own default; ``include_metadata`` falls back to the + value given at construction. + Args: status_check_api_endpoint (str): The API endpoint to check the status of the execution. + include_metadata (bool): Include metadata in the result. + include_metrics (bool): Include metrics in the result. + include_extracted_text (bool): Include the extracted text. Returns: dict: The response from the API. """ - headers = { - "Authorization": "Bearer " + self.api_key, + self.logger.debug( + "Checking execution status via endpoint: " + status_check_api_endpoint + ) + requested = { + "include_metadata": include_metadata, + "include_metrics": include_metrics, + "include_extracted_text": include_extracted_text, } - status_call_url = self.base_url + status_check_api_endpoint - self.logger.debug("Checking execution status via endpoint: " + status_call_url) + requested = {k: v for k, v in requested.items() if not isinstance(v, Unset)} + params = {"include_metadata": self.include_metadata, **requested} + + # Placeholders: the generated builder only spends these on the URL, and + # ``_status_url`` derives its own. Requiring a route here would refuse to + # poll deployment URLs the released client polled, after the execution + # behind them has already been submitted. + request_kwargs = status._get_kwargs( + "", + "", + execution_id=_query_value(status_check_api_endpoint, "execution_id"), + **params, + ) + # The generated builder writes every declared query parameter, including + # ones this client has never sent. Keep only what was asked for. + send_only = _STATUS_SEND_ONLY | requested.keys() + # Booleans are spelled the way urlencoding a Python bool spells them, + # which is what the released client sent. The service reads either, but + # traffic diffed against the previous release should show no change. + request_kwargs["params"] = { + **_forwarded_query(status_check_api_endpoint), + **{ + k: str(v) if isinstance(v, bool) else v + for k, v in request_kwargs["params"].items() + if k in send_only + }, + } + request_kwargs.pop("url") response = self._request_with_retry( - "GET", - status_call_url, - headers=headers, - params={"include_metadata": self.include_metadata}, - verify=self.verify, + request_kwargs.pop("method"), + self._status_url(status_check_api_endpoint), + **request_kwargs, ) self.logger.debug(response.status_code) self.logger.debug(response.text) obj_to_return = {} - try: - response_data = response.json() - except JSONDecodeError: + response_data = self._read_body(response) + if response_data is None: self.logger.error( "Failed to decode JSON response. Raw response: %s", response.text, @@ -445,9 +882,24 @@ def check_execution_status(self, status_check_api_endpoint: str) -> dict: return obj_to_return # Construct response object - execution_status = response_data.get("status", "") - error_message = response_data.get("error", "") - extraction_result = response_data.get("message", "") + body = response_data if isinstance(response_data, dict) else {} + execution_status = body.get("status", "") + error_message = body.get("error", "") + extraction_result = body.get("message", "") + if not error_message and not 200 <= response.status_code < 300: + if "status" in body: + # The endpoint's own envelope. A non-2xx here reports an + # execution state, not a refused request, and the only reason it + # carries is a message that is text where a result would be a + # list. Left under both keys a caller reads the error back as an + # extraction. + if isinstance(extraction_result, str) and extraction_result: + error_message, extraction_result = extraction_result, "" + else: + # A refused request answers in another shape entirely, and the + # reason lives outside the fields read above. Without this a + # failed poll is indistinguishable from a finished-and-empty one. + error_message = _error_text(response_data, response) obj_to_return = { "status_code": response.status_code, diff --git a/src/unstract/cli.py b/src/unstract/cli.py deleted file mode 100644 index 9634111..0000000 --- a/src/unstract/cli.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Top-level ``unstract`` command group. - -Subcommands live in their own subpackages and are registered here so a -single console script (``unstract``) fronts all of them. ``unstract.clone`` -keeps its own group + ``main`` so ``python -m unstract.clone`` still works. -""" - -from __future__ import annotations - -from typing import Any - -import click - -from unstract.clone.cli import clone_cmd - - -@click.group(name="unstract") -def cli() -> None: - """Unstract command-line tools.""" - - -cli.add_command(clone_cmd, name="clone") - - -def main(argv: list[str] | None = None) -> Any: - return cli(args=argv, standalone_mode=True) - - -if __name__ == "__main__": - main() diff --git a/src/unstract/clone/cli.py b/src/unstract/clone/cli.py index 43f3a09..e957713 100644 --- a/src/unstract/clone/cli.py +++ b/src/unstract/clone/cli.py @@ -1,8 +1,6 @@ """Click-based CLI for ``unstract.clone``. -Single ``clone`` command, registered on the top-level ``unstract`` group -(``unstract.cli``) — the canonical invocation is ``unstract clone``. The -local group here only backs ``python -m unstract.clone``. +Single ``clone`` command, invoked as ``python -m unstract.clone clone``. Platform keys can be passed via flags (``--source-key`` / ``--target-key``) or env vars (``UNSTRACT_SRC_PLATFORM_KEY`` / ``UNSTRACT_TGT_PLATFORM_KEY``) diff --git a/tests/baseline/client_1_5_3.py b/tests/baseline/client_1_5_3.py new file mode 100644 index 0000000..dd76c91 --- /dev/null +++ b/tests/baseline/client_1_5_3.py @@ -0,0 +1,475 @@ +# Vendored from the released unstract-client 1.5.3 wheel on PyPI. DO NOT EDIT. +# Refresh with tools/refresh_baseline.sh when the parity baseline is intentionally moved. +"""This module provides an API client to invoke APIs deployed on the Unstract +platform. + +Classes: + APIDeploymentsClient: A class to invoke APIs deployed on the Unstract platform. + APIDeploymentsClientException: A class to handle exceptions raised by the + APIDeploymentsClient class. +""" + +import logging +import ntpath +import os +import time +from urllib.parse import urlparse + +import requests +from requests.exceptions import ConnectionError, JSONDecodeError, Timeout +from tenacity import ( + RetryCallState, + Retrying, + retry_if_exception_type, + retry_if_result, + stop_after_attempt, + wait_exponential_jitter, +) +from tenacity.wait import wait_base + +from unstract.api_deployments.utils import UnstractUtils + + +class APIDeploymentsClientException(Exception): + """A class to handle exceptions raised by the APIClient class.""" + + def __init__(self, message): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + def error_message(self): + return self.value + + +class _WaitRetryAfterOrExponentialJitter(wait_base): + """Wait strategy that respects Retry-After on 429, else exponential jitter. + + For 429 responses with a valid ``Retry-After`` header the server-requested + delay is used. In every other case the strategy delegates to + ``wait_exponential_jitter`` (additive jitter). + """ + + def __init__( + self, + initial: float, + max: float, + exp_base: float, + jitter: float, + ) -> None: + super().__init__() + self._exp_jitter = wait_exponential_jitter( + initial=initial, max=max, exp_base=exp_base, jitter=jitter + ) + + def __call__(self, retry_state: RetryCallState) -> float: + outcome = retry_state.outcome + if outcome and not outcome.failed: + response = outcome.result() + if response is not None and getattr(response, "status_code", None) == 429: + retry_after = response.headers.get("Retry-After") + if retry_after is not None: + try: + return float(retry_after) + except (ValueError, TypeError): + pass + return self._exp_jitter(retry_state) + + +class APIDeploymentsClient: + """A class to invoke APIs deployed on the Unstract platform.""" + + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + logger = logging.getLogger(__name__) + log_stream_handler = logging.StreamHandler() + log_stream_handler.setFormatter(formatter) + logger.addHandler(log_stream_handler) + + api_key = "" + api_timeout = 300 + in_progress_statuses = ["PENDING", "EXECUTING", "READY", "QUEUED", "INITIATED"] + + def __init__( + self, + api_url: str, + api_key: str, + api_timeout: int = 300, + logging_level: str = "INFO", + include_metadata: bool = False, + verify: bool = True, + max_retries: int = 4, + initial_delay: float = 2.0, + max_delay: float = 60.0, + backoff_factor: float = 2.0, + jitter: float = 1.0, + ): + """Initializes the APIClient class. + + Args: + api_key (str): The API key to authenticate the API request. + api_timeout (int): The timeout to wait for the API response. + logging_level (str): The logging level to log messages. + max_retries (int): Maximum number of retry attempts for failed requests. + initial_delay (float): Initial delay in seconds before the first retry. + max_delay (float): Maximum delay in seconds between retries. + backoff_factor (float): Multiplier applied to delay for each retry. + jitter (float): Maximum additive jitter in seconds added to each delay. + """ + if logging_level == "": + logging_level = os.getenv("UNSTRACT_API_CLIENT_LOGGING_LEVEL", "INFO") + if logging_level == "DEBUG": + self.logger.setLevel(logging.DEBUG) + elif logging_level == "INFO": + self.logger.setLevel(logging.INFO) + elif logging_level == "WARNING": + self.logger.setLevel(logging.WARNING) + elif logging_level == "ERROR": + self.logger.setLevel(logging.ERROR) + + # self.logger.setLevel(logging_level) + self.logger.debug("Logging level set to: " + logging_level) + + if api_key == "": + self.api_key = os.getenv("UNSTRACT_API_DEPLOYMENT_KEY", "") + else: + self.api_key = api_key + self.logger.debug("API key set to: " + UnstractUtils.redact_key(self.api_key)) + + self.api_timeout = api_timeout + self.api_url = api_url + self.__save_base_url(api_url) + self.include_metadata = include_metadata + self.verify = verify + self.max_retries = max_retries + self.initial_delay = initial_delay + self.max_delay = max_delay + self.backoff_factor = backoff_factor + self.jitter = jitter + + def _is_retryable_status(self, status_code: int) -> bool: + """Checks whether a status code should trigger a retry. + + Args: + status_code (int): The HTTP status code to check. + + Returns: + bool: True if the request should be retried. + """ + return status_code >= 500 or status_code == 429 + + def __save_base_url(self, full_url: str): + """Extracts the base URL from the full URL and saves it. + + Args: + full_url (str): The full URL of the API. + """ + parsed_url = urlparse(full_url) + self.base_url = parsed_url.scheme + "://" + parsed_url.netloc + self.logger.debug("Base URL: " + self.base_url) + + @staticmethod + def _rewind_files(files): + """Rewinds file objects so they can be re-sent on retry.""" + for file_tuple in files: + file_obj = file_tuple[1] + if hasattr(file_obj, "seek"): + file_obj.seek(0) + elif isinstance(file_obj, tuple) and len(file_obj) >= 2: + if hasattr(file_obj[1], "seek"): + file_obj[1].seek(0) + + def _request_with_retry(self, method: str, url: str, **kwargs) -> requests.Response: + """Makes an HTTP request with exponential backoff retry logic. + + Uses ``tenacity`` with additive jitter and Retry-After support. + + Args: + method (str): The HTTP method (e.g., "GET", "POST"). + url (str): The request URL. + **kwargs: Additional keyword arguments passed to requests.request(). + + Returns: + requests.Response: The response from the request. + + Raises: + ConnectionError: If a connection error persists after all retries. + Timeout: If a timeout persists after all retries. + """ + files = kwargs.get("files") + + def _before_sleep(retry_state: RetryCallState): + attempt = retry_state.attempt_number + delay = retry_state.next_action.sleep + outcome = retry_state.outcome + if outcome.failed: + exc = outcome.exception() + self.logger.warning( + "%s during request to %s. Retrying in %.1fs (attempt %d/%d).", + type(exc).__name__, + url, + delay, + attempt, + self.max_retries, + ) + else: + response = outcome.result() + self.logger.warning( + "Request to %s returned %d. Retrying in %.1fs (attempt %d/%d).", + url, + response.status_code, + delay, + attempt, + self.max_retries, + ) + # Rewind file objects before next attempt + if files: + self._rewind_files(files) + + def _retry_error_callback(retry_state: RetryCallState): + outcome = retry_state.outcome + if outcome.failed: + exc = outcome.exception() + self.logger.warning( + "%s during request to %s. Retries exhausted (%d/%d).", + type(exc).__name__, + url, + self.max_retries, + self.max_retries, + ) + raise exc + response = outcome.result() + self.logger.warning( + "Request to %s returned %d. Retries exhausted (%d/%d).", + url, + response.status_code, + self.max_retries, + self.max_retries, + ) + return response + + retrier = Retrying( + stop=stop_after_attempt(self.max_retries + 1), + wait=_WaitRetryAfterOrExponentialJitter( + initial=self.initial_delay, + max=self.max_delay, + exp_base=self.backoff_factor, + jitter=self.jitter, + ), + retry=( + retry_if_result(lambda r: self._is_retryable_status(r.status_code)) + | retry_if_exception_type((ConnectionError, Timeout)) + ), + before_sleep=_before_sleep, + retry_error_callback=_retry_error_callback, + sleep=time.sleep, + reraise=False, + ) + + return retrier(requests.request, method, url, **kwargs) + + def structure_file(self, file_paths: list[str]) -> dict: + """Invokes the API deployed on the Unstract platform. + + Args: + file_paths (list[str]): The file path to the file to be uploaded. + + Returns: + dict: The response from the API. + """ + self.logger.debug("Invoking API: " + self.api_url) + self.logger.debug("File paths: " + str(file_paths)) + + headers = { + "Authorization": "Bearer " + self.api_key, + } + + form_data = { + "timeout": self.api_timeout, + "include_metadata": self.include_metadata, + } + + files = [] + + try: + for file_path in file_paths: + record = ( + "files", + ( + ntpath.basename(file_path), + open(file_path, "rb"), + "application/octet-stream", + ), + ) + files.append(record) + except FileNotFoundError as e: + raise APIDeploymentsClientException("File not found: " + str(e)) + + if self.api_timeout == 0: + # Async mode: server returns immediately after queuing. + # A 5xx means queuing failed — safe to retry. + response = self._request_with_retry( + "POST", + self.api_url, + headers=headers, + data=form_data, + files=files, + verify=self.verify, + ) + else: + # Sync mode: server blocks during processing. + # A 5xx may mean it processed but response was lost — don't retry + # to avoid duplicate executions. + response = requests.post( + self.api_url, + headers=headers, + data=form_data, + files=files, + verify=self.verify, + ) + self.logger.debug(response.status_code) + self.logger.debug(response.text) + # The returned object is wrapped in a "message" key. + # Let's simplify the response. + obj_to_return = {} + + try: + response_data = response.json() + response_message = response_data.get("message", {}) + except JSONDecodeError: + self.logger.error( + "Failed to decode JSON response. Raw response: %s", + response.text, + exc_info=True, + ) + obj_to_return = { + "status_code": response.status_code, + "pending": False, + "execution_status": "", + "error": "Invalid JSON response from API", + "extraction_result": "", + } + return obj_to_return + if response.status_code == 401: + obj_to_return = { + "status_code": response.status_code, + "pending": False, + "execution_status": "", + "error": response_data.get("errors", [{}])[0].get( + "detail", "Unauthorized" + ), + "extraction_result": "", + } + return obj_to_return + + # If the execution status is pending, extract the execution ID from + # the response and return it in the response. + # Later, users can use the execution ID to check the status of the execution. + # The returned object is wrapped in a "message" key. + # Let's simplify the response. + # Construct response object + execution_status = response_message.get("execution_status", "") + error_message = response_message.get("error", "") + extraction_result = response_message.get("result", "") + status_api_endpoint = response_message.get("status_api") + + obj_to_return = { + "status_code": response.status_code, + "pending": False, + "execution_status": execution_status, + "error": error_message, + "extraction_result": extraction_result, + } + + # Check if the status is pending or if it's successful but lacks a result. + # The POST endpoint returns 200 for successful queuing (including + # PENDING/EXECUTING) and 422 only on setup errors — guard against + # incorrectly polling after an error response. + if 200 <= response.status_code < 300: + if execution_status in self.in_progress_statuses or ( + execution_status == "SUCCESS" and not extraction_result + ): + obj_to_return.update( + { + "status_check_api_endpoint": status_api_endpoint, + "pending": True, + } + ) + + return obj_to_return + + def check_execution_status(self, status_check_api_endpoint: str) -> dict: + """Checks the status of the execution. + + Args: + status_check_api_endpoint (str): + The API endpoint to check the status of the execution. + + Returns: + dict: The response from the API. + """ + + headers = { + "Authorization": "Bearer " + self.api_key, + } + status_call_url = self.base_url + status_check_api_endpoint + self.logger.debug("Checking execution status via endpoint: " + status_call_url) + response = self._request_with_retry( + "GET", + status_call_url, + headers=headers, + params={"include_metadata": self.include_metadata}, + verify=self.verify, + ) + self.logger.debug(response.status_code) + self.logger.debug(response.text) + + obj_to_return = {} + + try: + response_data = response.json() + except JSONDecodeError: + self.logger.error( + "Failed to decode JSON response. Raw response: %s", + response.text, + exc_info=True, + ) + obj_to_return = { + "status_code": response.status_code, + "pending": False, + "execution_status": "", + "error": "Invalid JSON response from API", + "extraction_result": "", + } + return obj_to_return + + # Construct response object + execution_status = response_data.get("status", "") + error_message = response_data.get("error", "") + extraction_result = response_data.get("message", "") + + obj_to_return = { + "status_code": response.status_code, + "pending": False, + "execution_status": execution_status, + "error": error_message, + "extraction_result": extraction_result, + } + + # If the execution status is pending, extract the execution ID from the response + # and return it in the response. + # Later, users can use the execution ID to check the status of the execution. + if obj_to_return["execution_status"] in self.in_progress_statuses: + obj_to_return["pending"] = True + elif self._is_retryable_status(response.status_code): + obj_to_return["pending"] = True + self.logger.warning( + "Status check returned %d after retries; " + "marking as pending to continue polling.", + response.status_code, + ) + + return obj_to_return diff --git a/tests/test_cli_top_level.py b/tests/test_cli_top_level.py deleted file mode 100644 index e55d26d..0000000 --- a/tests/test_cli_top_level.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Tests for the top-level ``unstract`` command group (``unstract.cli``).""" - -from __future__ import annotations - -from click.testing import CliRunner - -from unstract.cli import cli -from unstract.clone.report import CloneReport, Endpoint - - -def test_clone_invocation_via_top_level_group(monkeypatch): - captured: dict = {} - - def fake_clone(source, target, options=None): - captured["source"] = source - captured["target"] = target - return CloneReport( - source=Endpoint( - base_url=source.base_url, organization_id=source.organization_id - ), - target=Endpoint( - base_url=target.base_url, organization_id=target.organization_id - ), - ) - - # The clone command's callback resolves run_clone from unstract.clone.cli. - monkeypatch.setattr("unstract.clone.cli.run_clone", fake_clone) - - result = CliRunner().invoke( - cli, - [ - "clone", - "--source-url", - "http://src", - "--source-org", - "src", - "--source-key", - "sk", - "--target-url", - "http://tgt", - "--target-org", - "tgt", - "--target-key", - "tk", - ], - ) - - assert result.exit_code == 0, result.output - assert captured["source"].organization_id == "src" - assert captured["target"].organization_id == "tgt" diff --git a/tests/test_compat.py b/tests/test_compat.py new file mode 100644 index 0000000..8cd95fa --- /dev/null +++ b/tests/test_compat.py @@ -0,0 +1,1895 @@ +"""Parity tests against the last released client. + +The transport underneath ``APIDeploymentsClient`` changed; its published +behaviour must not. These tests pin the seams where that could silently break: +the constructor and method signatures, what goes out on the wire, which +exceptions come back out, and the exact dict each method returns — the last one +by running the released client side by side over the same responses. + +This suite exists for the transport migration, not forever: once the released +client it compares against is old enough that no caller is upgrading from it, +it should be dropped or re-baselined deliberately (``tools/refresh_baseline.sh``) +rather than edited case by case until it passes. + +Differences from the baseline that are accepted rather than fixed are asserted +here explicitly, each in the test that would otherwise be blind to it — the +``User-Agent``, the per-part ``Content-Type`` on form fields, the error text now +reported for non-2xx bodies the baseline dropped, the retry boundary moving from +a timeout of exactly zero to at or below zero, and the exception class a +malformed deployment URL raises. +""" + +import ast +import hashlib +import importlib.util +import inspect +import io +import json +import re +import socket +import threading +import tomllib +from pathlib import Path +from unittest.mock import MagicMock, patch +from urllib.parse import parse_qs, urlparse + +import httpx +import pytest +import requests +from requests.exceptions import ( + ConnectionError, + ConnectTimeout, + ContentDecodingError, + InvalidHeader, + MissingSchema, + ProxyError, + ReadTimeout, + RequestException, + Timeout, + TooManyRedirects, +) + +from unstract import api_deployments +from unstract.api_deployments.client import ( + _EXECUTE_SEND_ONLY, + _STATUS_SEND_ONLY, + APIDeploymentsClient, + APIDeploymentsClientException, +) +from unstract.api_deployments._sdk_docstudio.types import UNSET + +BASELINE_VERSION = "1.5.3" +BASELINE_PATH = Path(__file__).parent / "baseline" / "client_1_5_3.py" +BASELINE_SHA256 = "45201bb0de000e8f3a0e65f40cb0b08fec389514f7a17c8bb3410a3dc59229df" +SPEC_PATH = Path(__file__).parents[1] / "specs" / "docstudio-oss.json" +#: The one place the vendored spec's provenance is recorded. +GENERATOR_PATH = Path(__file__).parents[1] / "tools" / "gen_sdk.sh" + +API_URL = "https://api.example.com/deployment/api/testorg/testapi/" +STATUS_ENDPOINT = "/deployment/api/testorg/testapi/?execution_id=exec-123" + +#: Operations the facade wraps. The spec declares exactly these, and a new one +#: has to be added here deliberately rather than arriving unnoticed. +WRAPPED_OPERATIONS = frozenset({"execute", "status"}) + +#: Every accepted divergence from the baseline, named as the module docstring +#: names it. A divergence pinned by a test but missing from that list is only +#: findable by reading all of them. +ACCEPTED_DIVERGENCES = ( + "User-Agent", + "Content-Type", + "error text", + "retry boundary", + "exception class", +) + + +def test_the_accepted_divergences_are_all_listed_up_front(): + """Each one is asserted in the test that would otherwise be blind to it, but + that only makes it deliberate — the list is what makes it findable.""" + for phrase in ACCEPTED_DIVERGENCES: + assert phrase in (__doc__ or ""), phrase + + +def _load_baseline(): + """Import the vendored released client under its own module name.""" + spec = importlib.util.spec_from_file_location("baseline_client", BASELINE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +baseline = _load_baseline() + + +@pytest.fixture +def sample_file(tmp_path): + path = tmp_path / "sample.txt" + path.write_bytes(b"hello") + return str(path) + + +def _client(**kwargs): + kwargs.setdefault("api_url", API_URL) + kwargs.setdefault("api_key", "test-key") + kwargs.setdefault("logging_level", "ERROR") + kwargs.setdefault("max_retries", 0) + return APIDeploymentsClient(**kwargs) + + +def _baseline_client(**kwargs): + kwargs.setdefault("api_url", API_URL) + kwargs.setdefault("api_key", "test-key") + kwargs.setdefault("logging_level", "ERROR") + kwargs.setdefault("max_retries", 0) + return baseline.APIDeploymentsClient(**kwargs) + + +def _httpx_response(status_code=200, json_data=None, text=None): + if text is not None: + return httpx.Response(status_code, text=text) + return httpx.Response(status_code, json=json_data) + + +def _requests_response(status_code=200, json_data=None, text=None): + response = MagicMock() + response.status_code = status_code + if text is not None: + response.text = text + response.json.side_effect = requests.exceptions.JSONDecodeError( + "no json", text, 0 + ) + else: + response.text = json.dumps(json_data) + response.json.return_value = json_data + response.headers = {} + return response + + +# -------------------------------------------------------------------------- +# Transport error translation +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("raised", "expected"), + [ + (httpx.ConnectTimeout("connect timed out"), ConnectTimeout), + (httpx.ReadTimeout("read timed out"), ReadTimeout), + # Neither had a Timeout equivalent: a send that failed and a pool that + # could not hand out a connection both surfaced as ConnectionError. + (httpx.WriteTimeout("write timed out"), ConnectionError), + (httpx.PoolTimeout("pool timed out"), ConnectionError), + (httpx.ConnectError("refused"), ConnectionError), + (httpx.ReadError("reset"), ConnectionError), + (httpx.WriteError("broken pipe"), ConnectionError), + (httpx.ProtocolError("bad framing"), ConnectionError), + (httpx.ProxyError("proxy exploded"), ProxyError), + (httpx.UnsupportedProtocol("no scheme"), MissingSchema), + (httpx.TooManyRedirects("looping"), TooManyRedirects), + (httpx.DecodingError("bad gzip"), ContentDecodingError), + # A request that cannot be written as composed -- an api_key carrying a + # newline is the everyday cause -- is a client-side fault, not the + # server-side framing failure its parent ProtocolError stands for. + (httpx.LocalProtocolError("illegal header value"), InvalidHeader), + ], +) +def test_transport_errors_are_translated(raised, expected): + """Callers catch the ``requests`` classes; httpx's are not subclasses. + + ``ConnectTimeout`` is the case that makes ordering load-bearing, and it is + also both a ``ConnectionError`` and a ``Timeout`` — the plain ``Timeout`` + that httpx's hierarchy implies would stop matching half the callers. The + exact class matters too: a caller catching ``ReadTimeout`` sees nothing if + a broader ``Timeout`` is raised in its place. + """ + client = _client() + with patch.object( + client._transport.get_httpx_client(), "request", side_effect=raised + ): + with pytest.raises(expected) as caught: + client._send("get", "/anything") + assert type(caught.value) is expected + + +@pytest.mark.parametrize( + ("api_url", "expected"), + [ + ("::::", MissingSchema), + ("/deployment/api/testorg/testapi/", MissingSchema), + # Parity: the released client raised this one too. + ("http://[bad", ValueError), + ], +) +def test_a_malformed_deployment_url_raises_the_class_this_client_chose( + api_url, expected +): + """A deliberate divergence, pinned here so it stays deliberate. + + The released client answered the first two with ``InvalidSchema``. Both + reach the transport here and come back as ``MissingSchema``, which names + what is actually wrong -- a different class, but the same + ``RequestException`` a caller wrapping construction already catches. + """ + with pytest.raises(expected) as caught: + client = _client(api_url=api_url) + client.check_execution_status(STATUS_ENDPOINT) + # MissingSchema is itself a ValueError, so the parity row can only tell the + # two apart by the exact class. + assert type(caught.value) is expected + + +def _httpx_request_errors(): + """Every httpx request failure, discovered rather than listed. + + A hand-written list is exactly as complete as it was the day it was + written; this one grows when httpx does. + """ + found, stack = [], [httpx.RequestError] + while stack: + cls = stack.pop() + found.append(cls) + stack.extend(cls.__subclasses__()) + return sorted(found, key=lambda cls: cls.__name__) + + +@pytest.mark.parametrize("cls", _httpx_request_errors(), ids=lambda cls: cls.__name__) +def test_no_httpx_failure_escapes_untranslated(cls): + """An httpx class reaching a caller is a class no caller catches.""" + client = _client() + with patch.object( + client._transport.get_httpx_client(), "request", side_effect=cls("boom") + ): + with pytest.raises(RequestException): + client._send("get", "/anything") + + +@pytest.mark.parametrize( + ("raised", "retried"), + [ + (httpx.PoolTimeout("pool timed out"), True), + (httpx.ProxyError("proxy exploded"), True), + # Retrying these cannot start working: the URL stays malformed, the + # redirect chain stays a loop, the body stays undecodable, the header + # stays illegal. + (httpx.UnsupportedProtocol("no scheme"), False), + (httpx.TooManyRedirects("looping"), False), + (httpx.DecodingError("bad gzip"), False), + (httpx.LocalProtocolError("illegal header value"), False), + ], +) +def test_translation_decides_what_gets_retried(raised, retried): + client = _client(max_retries=2, initial_delay=0, max_delay=0, jitter=0) + with patch.object( + client._transport.get_httpx_client(), "request", side_effect=raised + ) as request: + with pytest.raises(RequestException): + client._request_with_retry("get", "/anything") + assert (request.call_count > 1) is retried + + +def test_a_connect_timeout_is_still_a_connection_error(): + client = _client() + with patch.object( + client._transport.get_httpx_client(), + "request", + side_effect=httpx.ConnectTimeout("connect timed out"), + ): + with pytest.raises(ConnectionError): + client._send("get", "/anything") + with patch.object( + client._transport.get_httpx_client(), + "request", + side_effect=httpx.ConnectTimeout("connect timed out"), + ): + with pytest.raises(Timeout): + client._send("get", "/anything") + + +def test_translated_errors_keep_the_original_cause(): + client = _client() + original = httpx.ConnectError("refused") + with patch.object( + client._transport.get_httpx_client(), "request", side_effect=original + ): + with pytest.raises(ConnectionError) as excinfo: + client._send("get", "/anything") + assert excinfo.value.__cause__ is original + + +def test_structure_file_raises_translated_error(sample_file): + client = _client() + with patch.object( + client._transport.get_httpx_client(), + "request", + side_effect=httpx.ConnectError("refused"), + ): + with pytest.raises(ConnectionError): + client.structure_file([sample_file]) + + +def test_check_execution_status_raises_translated_error(): + client = _client() + with patch.object( + client._transport.get_httpx_client(), + "request", + side_effect=httpx.ReadTimeout("read timed out"), + ): + with pytest.raises(Timeout): + client.check_execution_status(STATUS_ENDPOINT) + + +def test_translation_happens_inside_the_retried_call(sample_file): + """Retry counts the transport failures, which requires translation first. + + ``_request_with_retry`` retries on the ``requests`` exception types. If the + httpx exception escaped the retried callable untranslated it would never + match, and transport-error retry would quietly stop working. + """ + client = _client(api_timeout=0, max_retries=2, initial_delay=0.001, max_delay=0.002) + with patch.object( + client._transport.get_httpx_client(), + "request", + side_effect=httpx.ConnectError("refused"), + ) as mock_request: + with pytest.raises(ConnectionError): + client.structure_file([sample_file]) + assert mock_request.call_count == 3 + + +# -------------------------------------------------------------------------- +# api_timeout is an execution mode, never a transport timeout +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("api_timeout", [-1, 0, 1, 300]) +def test_api_timeout_never_configures_the_transport(api_timeout): + """``api_timeout`` selects a backend execution mode. + + ``-1``/``0`` mean async; + handing either to the transport fails inside the connection layer. + """ + client = _client(api_timeout=api_timeout) + assert client._transport.get_httpx_client().timeout == httpx.Timeout(None) + + +@pytest.mark.parametrize("api_timeout", [-1, 0, 300]) +def test_api_timeout_never_reaches_the_transport_call(sample_file, api_timeout): + client = _client(api_timeout=api_timeout) + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"message": {}}) + client.structure_file([sample_file]) + _, kwargs = mock_send.call_args + assert "timeout" not in kwargs + + +# -------------------------------------------------------------------------- +# What goes out on the wire +# -------------------------------------------------------------------------- + + +def _captured_execute_kwargs(client, file_path, **request_params): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"message": {}}) + client.structure_file([file_path], **request_params) + return mock_send.call_args + + +def _execute_parts(client, file_path, **request_params): + _, kwargs = _captured_execute_kwargs(client, file_path, **request_params) + return {name: value for name, value in kwargs["files"]} + + +def test_execute_sends_only_the_fields_the_client_sets(sample_file): + """A spec default written into the request pins a value the server would + otherwise choose, and the two diverge the moment the server's default + moves.""" + _, kwargs = _captured_execute_kwargs(_client(api_timeout=300), sample_file) + assert {name for name, _ in kwargs["files"]} == { + "files", + "include_metadata", + "timeout", + } + assert _EXECUTE_SEND_ONLY == { + "files", + "include_metadata", + "timeout", + "additional_properties", + } + + +def test_execute_multipart_values_match_the_released_client(sample_file): + _, kwargs = _captured_execute_kwargs( + _client(api_timeout=300, include_metadata=True), sample_file + ) + parts = {name: value for name, value in kwargs["files"]} + assert parts["timeout"][1] == b"300" + assert parts["include_metadata"][1] == b"True" + assert parts["files"][0] == "sample.txt" + assert parts["files"][2] == "application/octet-stream" + + +# -------------------------------------------------------------------------- +# Request parameters, added as keyword-only arguments +# -------------------------------------------------------------------------- + + +def _request_param_names(): + return [ + name + for name, p in inspect.signature( + APIDeploymentsClient.structure_file + ).parameters.items() + if p.kind is p.KEYWORD_ONLY + ] + + +def test_request_parameters_are_named_as_the_spec_names_them(): + """A rename here would need a translation table in every caller.""" + spec = json.loads(SPEC_PATH.read_text()) + declared = set(spec["components"]["schemas"]["ExecuteRequest"]["properties"]) + # ``files`` is built from ``file_paths``, not passed through. + assert set(_request_param_names()) == declared - {"files"} + + +def test_request_parameters_are_keyword_only(sample_file): + with pytest.raises(TypeError): + _client().structure_file([sample_file], 300) + + +def test_an_unset_parameter_is_not_sent(sample_file): + """Sending a default pins a value the server would otherwise choose.""" + parts = _execute_parts(_client(api_timeout=300), sample_file) + assert set(parts) == {"files", "include_metadata", "timeout"} + + +def test_a_requested_parameter_is_sent(sample_file): + parts = _execute_parts( + _client(api_timeout=300), + sample_file, + tags="a,b", + llm_profile_id="profile-1", + use_file_history=True, + ) + assert parts["tags"][1] == b"a,b" + assert parts["llm_profile_id"][1] == b"profile-1" + assert parts["use_file_history"][1] == b"True" + + +@pytest.mark.parametrize( + ("param", "value", "expected"), + [ + ("timeout", 0, b"0"), + ("include_metrics", False, b"False"), + ("include_extracted_text", False, b"False"), + ("tags", "", b""), + ], +) +def test_a_falsy_parameter_is_still_sent(sample_file, param, value, expected): + """``False``/``0``/``""`` are choices, not absences; a truthiness filter + eats them and silently hands the decision back to the server.""" + parts = _execute_parts(_client(api_timeout=300), sample_file, **{param: value}) + assert parts[param][1] == expected + + +@pytest.mark.parametrize( + "param", ["llm_profile_id", "hitl_queue_name", "hitl_packet_id"] +) +def test_an_explicit_none_is_not_sent(sample_file, param): + """``None`` is how a caller forwards "no override" from its own optional + config. A form field carries no null, so one sent at all goes out as the + literal string ``"None"`` for the service to look up.""" + parts = _execute_parts(_client(api_timeout=300), sample_file, **{param: None}) + assert param not in parts + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ({"a": 1, "b": "x"}, b'{"a": 1, "b": "x"}'), + ([1, 2], b"[1, 2]"), + (True, b"true"), + # Already a string: passed through, so a caller serialising its own + # payload does not get it JSON-encoded a second time. + ('{"a": 1}', b'{"a": 1}'), + ], +) +def test_custom_data_goes_out_as_json(sample_file, value, expected): + """The generated encoder writes ``str(value)``. For anything but a string + that is a Python repr, which the server's JSON field cannot parse.""" + parts = _execute_parts(_client(api_timeout=300), sample_file, custom_data=value) + assert parts["custom_data"][1] == expected + + +def test_custom_data_is_json_on_the_wire(tmp_path): + """Read off the socket, not out of the kwargs: the encoding step between + the two is where a repr would survive unnoticed.""" + path = tmp_path / "sample.txt" + path.write_bytes(b"hello") + + (raw,) = _wire_requests( + lambda url: _client(api_url=url, api_timeout=300).structure_file( + [str(path)], custom_data={"a": 1, "b": "x"} + ) + ) + sent = { + field: content for field, _filename, content, _headers in _multipart_parts(raw) + } + assert json.loads(sent["custom_data"]) == {"a": 1, "b": "x"} + + +def test_a_requested_parameter_overrides_the_constructor(sample_file): + parts = _execute_parts( + _client(api_timeout=300, include_metadata=False), + sample_file, + timeout=-1, + include_metadata=True, + ) + assert parts["timeout"][1] == b"-1" + assert parts["include_metadata"][1] == b"True" + + +@pytest.mark.parametrize("timeout", [0, -1, 300]) +def test_a_requested_timeout_selects_the_execution_mode(sample_file, timeout): + """``timeout`` is an execution mode: at or below zero the request only + queues, so a 5xx is safe to retry; above it the request runs the execution + and a retry would run it twice. + + ``-1`` is the API's own default for queue-only, so it belongs on the retried + side even though the released client tested for exactly ``0``. Passing the + mode per request has to move the decision with it. + """ + queues = timeout <= 0 + with patch.object(APIDeploymentsClient, "_request_with_retry") as retried: + with patch.object(APIDeploymentsClient, "_send") as sent: + retried.return_value = sent.return_value = _httpx_response( + 200, {"message": {}} + ) + # Constructor set the other way round, so only the per-request + # value can be what decided this. + _client(api_timeout=300 if queues else 0).structure_file( + [sample_file], timeout=timeout + ) + assert retried.called is queues + assert sent.called is not queues + + +@pytest.mark.parametrize("api_timeout", [0, -1]) +def test_a_queue_only_execution_is_retried(sample_file, api_timeout): + """A 5xx on a queue-only request means queuing failed; retrying cannot + duplicate work, and not retrying hands back a finished-looking result with + no execution behind it.""" + client = _client(api_timeout=api_timeout, max_retries=2, initial_delay=0, jitter=0) + with patch.object(APIDeploymentsClient, "_send") as sent: + sent.return_value = _httpx_response(503, {"message": {}}) + client.structure_file([sample_file]) + assert sent.call_count == 3 + + +def test_multipart_boundary_is_random_and_matches_the_body(sample_file): + """The generated builder pins ``boundary=+++`` in the header. A PDF + containing those bytes would corrupt the encoding, so the header is dropped + and the transport picks the boundary — as the released client did. + + Encoding happens inside the send, while the file handles are still + open. + """ + encoded = [] + + def encode(method, url, **kwargs): + assert "Content-Type" not in kwargs.get("headers", {}) + transport = httpx.Client(base_url="https://api.example.com") + request = transport.build_request(method, url, **kwargs) + encoded.append((request.headers["content-type"], request.read())) + return _httpx_response(200, {"message": {}}) + + with patch.object(APIDeploymentsClient, "_send", side_effect=encode): + _client().structure_file([sample_file]) + _client().structure_file([sample_file]) + + boundaries = [] + for content_type, body in encoded: + header_boundary = content_type.split("boundary=")[1] + assert body.split(b"\r\n")[0] == b"--" + header_boundary.encode() + assert header_boundary != "+++" + boundaries.append(header_boundary) + assert boundaries[0] != boundaries[1] + + +def _captured_status_params(client=None, endpoint=STATUS_ENDPOINT, **request_params): + client = client or _client() + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + client.check_execution_status(endpoint, **request_params) + return mock_send.call_args[1]["params"] + + +def test_status_sends_only_the_fields_the_client_sets(): + assert set(_captured_status_params()) == {"execution_id", "include_metadata"} + + +@pytest.mark.parametrize( + ("api_url", "expected"), + [ + (API_URL, "https://api.example.com/deployment/api/testorg/testapi/"), + ( + "https://api.example.com/unstract/deployment/api/testorg/testapi/", + "https://api.example.com/unstract/deployment/api/testorg/testapi/", + ), + ], +) +def test_status_is_polled_under_the_deployment_urls_own_prefix(api_url, expected): + """A poll that misses is a paid execution whose result is never collected.""" + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + _client(api_url=api_url).check_execution_status(STATUS_ENDPOINT) + assert mock_send.call_args[0][1] == expected + + +def test_the_status_endpoints_own_query_parameters_are_forwarded(): + """The endpoint is the service's instruction for reaching this execution. + + A region hint or a signature dropped from it polls somewhere the execution + is not, and the execution has already been paid for. + """ + params = _captured_status_params( + client=_client(), + endpoint=STATUS_ENDPOINT + "®ion=eu&sig=abc123", + ) + assert params["region"] == "eu" + assert params["sig"] == "abc123" + assert params["execution_id"] == "exec-123" + + +def test_a_deployment_url_without_the_spec_route_polls_the_endpoint_as_returned(): + """No prefix can be derived from a rewritten URL, and a guessed path polls + nothing.""" + client = _client(api_url="https://gw.example.com/v1/testorg/testapi/") + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + client.check_execution_status("/v1/testorg/testapi/?execution_id=exec-123") + assert mock_send.call_args[0][1] == "https://gw.example.com/v1/testorg/testapi/" + + +def test_status_request_parameters_are_named_as_the_spec_names_them(): + """A rename here would need a translation table in every caller.""" + spec = json.loads(SPEC_PATH.read_text()) + execute = "/deployment/api/{org_name}/{api_name}/" + declared = { + p["name"] + for p in spec["paths"][execute]["get"]["parameters"] + if p["in"] == "query" + } + accepted = { + name + for name, p in inspect.signature( + APIDeploymentsClient.check_execution_status + ).parameters.items() + if p.kind is p.KEYWORD_ONLY + } + # `execution_id` is read out of the endpoint URL the server handed back. + assert accepted == declared - {"execution_id"} + + +def test_status_request_parameters_are_keyword_only(): + with pytest.raises(TypeError): + _client().check_execution_status(STATUS_ENDPOINT, True) + + +def test_a_requested_status_parameter_is_sent(): + params = _captured_status_params(include_metrics=True, include_extracted_text=False) + assert params["include_metrics"] == "True" + # False is a choice; a truthiness filter would drop it and hand the decision + # back to the server. + assert params["include_extracted_text"] == "False" + + +def test_a_requested_status_parameter_overrides_the_constructor(): + params = _captured_status_params( + _client(include_metadata=False), include_metadata=True + ) + assert params["include_metadata"] == "True" + assert _STATUS_SEND_ONLY == {"execution_id", "include_metadata"} + + +def test_status_url_matches_the_released_client(): + """The status URL is rebuilt from the spec route plus the execution id + instead of concatenating the server-supplied path. + + Same request either way, which is what this pins. + """ + client = _client(include_metadata=True) + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + client.check_execution_status(STATUS_ENDPOINT) + args, kwargs = mock_send.call_args + # Encoded by the transport that will send it, rather than stringified here: + # httpx renders a bool as `true` where urlencoding one gives `True`, and + # normalising both sides is how a comparison stops seeing the difference. + sent = httpx.URL(client.base_url).join(args[1]).copy_merge_params(kwargs["params"]) + ours = urlparse(str(sent)) + ours_query = parse_qs(ours.query) + + published = urlparse(client.base_url + STATUS_ENDPOINT) + published_query = { + **parse_qs(published.query), + "include_metadata": [str(client.include_metadata)], + } + + assert args[0].lower() == "get" + assert (ours.scheme, ours.netloc, ours.path) == ( + published.scheme, + published.netloc, + published.path, + ) + assert ours_query == published_query + + +@pytest.mark.parametrize( + "endpoint", + [ + # What the server actually returns, and the only spelling the released + # client handled: it concatenated base URL and endpoint, so an absolute + # one produced `https://hosthttps://host/...` and a relative one with + # no leading slash produced `https://hostdeployment/...`. + "/deployment/api/testorg/testapi/?execution_id=exec-123", + "https://api.example.com/deployment/api/testorg/testapi/?execution_id=exec-123", + "deployment/api/testorg/testapi/?execution_id=exec-123", + ], +) +def test_the_status_endpoint_is_read_not_concatenated(endpoint): + """Only the execution id is taken from the server's endpoint; the route + comes from the spec. Every spelling therefore resolves to one request.""" + client = _client() + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + client.check_execution_status(endpoint) + args, kwargs = mock_send.call_args + assert args[0].lower() == "get" + assert str(httpx.URL(client.base_url).join(args[1])) == API_URL + assert kwargs["params"]["execution_id"] == "exec-123" + + +@pytest.mark.parametrize( + "endpoint", + [ + "https://attacker.example/deployment/api/testorg/testapi/", + # A host can be spelled without a scheme, and a path beginning `//` is + # read as one by anything that resolves a reference. + "//attacker.example/deployment/api/testorg/testapi/", + "///attacker.example/deployment/api/testorg/testapi/", + "////attacker.example/deployment/api/testorg/testapi/", + "https:////attacker.example/deployment/api/testorg/testapi/", + "https://api.example.com//attacker.example/deployment/api/testorg/testapi/", + ], +) +def test_a_status_endpoint_naming_another_host_is_not_polled(endpoint): + """The reply names the path to poll. It does not get to name the host the + deployment key is sent to. + + The deployment URL here carries a prefix the spec route cannot account for, + which is the branch that reads the endpoint rather than rebuilding it. + """ + client = _client(api_url="https://api.example.com/other/testorg/testapi/") + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + client.check_execution_status(f"{endpoint}?execution_id=exec-123") + args, _ = mock_send.call_args + sent = httpx.URL(client.base_url).join(args[1]) + + assert sent.host == "api.example.com" + + +def test_the_key_is_read_at_call_time(): + """A rotated key reaches the next request. + + The released client built its header on every call, so assigning ``api_key`` + took effect immediately; a transport that captured it once would answer with + the old one until the client was rebuilt. + """ + client = _client() + with patch.object(client._transport.get_httpx_client(), "request") as request: + request.return_value = _httpx_response(200, {}) + client._send("GET", API_URL) + before = request.call_args.kwargs["headers"]["Authorization"] + client.api_key = "rotated-key" + client._send("GET", API_URL) + after = request.call_args.kwargs["headers"]["Authorization"] + + assert before == "Bearer test-key" + assert after == "Bearer rotated-key" + + +@pytest.mark.parametrize( + "api_url", + [ + API_URL, + # Nothing normalises the deployment URL on the way in, so whatever the + # caller registered is what the released client posted to. + API_URL.rstrip("/"), + "https://api.example.com/unstract/deployment/api/testorg/testapi/", + "https://api.example.com/deployment/api/TestOrg/testapi/", + ], +) +def test_execute_url_matches_the_deployment_url(api_url): + """The deployment URL goes back out as given. + + A path prefix — an ingress route, an on-prem reverse proxy — is part of it + and no route template can carry it, so the URL cannot be rebuilt from one. + """ + client = _client(api_url=api_url) + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"message": {}}) + with patch("builtins.open", return_value=io.BytesIO(b"x")): + client.structure_file(["sample.txt"]) + args, _ = mock_send.call_args + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.post.return_value = _requests_response(200, {"message": {}}) + with patch("builtins.open", return_value=io.BytesIO(b"x")): + _baseline_client(api_url=api_url).structure_file(["sample.txt"]) + + assert args[0].lower() == "post" + assert args[1] == api_url == mock_requests.post.call_args[0][0] + + +def _wire_requests(*calls, reply=b'{"status":"COMPLETED","message":{}}'): + """Run each call against a loopback server and return the raw requests. + + Below the client, the transport adds headers of its own -- and drops none + of them into any object the client can be asked for. A socket is the only + place both clients can be compared on what they actually send. One server + serves every call, so the ``Host`` header is the same for all of them. + """ + raw = [] + server = socket.socket() + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + server.listen(len(calls)) + + def serve(): + for _ in calls: + conn, _address = server.accept() + data = b"" + while b"\r\n\r\n" not in data: + chunk = conn.recv(65536) + if not chunk: + break + data += chunk + # The body has to be drained too: a client whose upload is never + # read can block on the socket instead of returning. + head, _, rest = data.partition(b"\r\n\r\n") + declared = _header_value(head, "content-length") + while declared and len(rest) < int(declared): + chunk = conn.recv(65536) + if not chunk: + break + rest += chunk + raw.append(head + b"\r\n\r\n" + rest) + body = reply + conn.sendall( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Content-Length: %d\r\n\r\n%s" % (len(body), body) + ) + conn.close() + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.getsockname()[1]}/deployment/api/org/name/" + for call in calls: + call(url) + finally: + thread.join(timeout=10) + server.close() + + return raw + + +def _headers(head: bytes) -> dict[str, str]: + return { + name.lower(): value.strip() + for name, _, value in ( + line.partition(":") for line in head.decode().split("\r\n")[1:] + ) + } + + +def _header_value(head: bytes, name: str) -> str: + return _headers(head).get(name, "") + + +def _wire_heads(*calls): + """The request headers each call put on the wire.""" + return [_headers(raw.split(b"\r\n\r\n")[0]) for raw in _wire_requests(*calls)] + + +def _multipart_parts(raw: bytes) -> list[tuple[str, str, bytes, tuple]]: + """``(field, filename, content, headers)`` for every part of a multipart + request. + + The headers are carried too: a difference in them is invisible to a + comparison of what the parts contain, and the parts of this request do + differ from the released client's there. + + The boundary itself is deliberately not compared: it is random per request + in both clients, so only what it delimits can be. + """ + head, _, body = raw.partition(b"\r\n\r\n") + boundary = _header_value(head, "content-type").partition("boundary=")[2] + parts = [] + for chunk in body.split(b"--" + boundary.encode()): + headers, _, content = chunk.partition(b"\r\n\r\n") + disposition = headers.decode("utf-8", errors="replace") + if "content-disposition" not in disposition.lower(): + continue + field = re.search(r'name="([^"]*)"', disposition) + filename = re.search(r'filename="([^"]*)"', disposition) + parts.append( + ( + field.group(1) if field else "", + filename.group(1) if filename else "", + content.removesuffix(b"\r\n"), + tuple( + line.split(":", 1)[0].strip().lower() + for line in disposition.strip().splitlines() + if ":" in line + ), + ) + ) + return parts + + +def test_wire_headers_match_the_released_client(): + """The headers no caller sets are still on the wire. + + ``Accept-Encoding`` is the load-bearing one: it decides whether responses + come back compressed at all. + """ + ours, theirs = _wire_heads( + lambda url: _client(api_url=url).check_execution_status(STATUS_ENDPOINT), + lambda url: _baseline_client(api_url=url).check_execution_status( + STATUS_ENDPOINT + ), + ) + + assert {name: ours[name] for name in theirs if name != "user-agent"} == { + name: value for name, value in theirs.items() if name != "user-agent" + } + assert ours.keys() == theirs.keys() + # The one accepted difference: the transport names itself, and nothing on + # the wire branches on it. + assert ours["user-agent"].startswith("python-httpx/") + + +def test_a_multi_file_upload_matches_the_released_client(tmp_path): + """The method takes a list, and the second file is where a transport swap + diverges: one part written, one dropped, or two parts sharing a name the + server then reads as one.""" + paths = [] + for name, content in (("first.txt", b"one"), ("second.txt", b"two")): + path = tmp_path / name + path.write_bytes(content) + paths.append(str(path)) + + ours, theirs = _wire_requests( + lambda url: _client(api_url=url, api_timeout=300).structure_file(paths), + lambda url: _baseline_client(api_url=url, api_timeout=300).structure_file( + paths + ), + ) + + # Sorted: the two clients order the fields differently, which no multipart + # parser reads as meaning. The order of the files among themselves is the + # part that carries meaning, and it is pinned below. + assert sorted(part[:3] for part in _multipart_parts(ours)) == sorted( + part[:3] for part in _multipart_parts(theirs) + ) + uploaded = [part for part in _multipart_parts(ours) if part[0] == "files"] + assert [(part[1], part[2]) for part in uploaded] == [ + ("first.txt", b"one"), + ("second.txt", b"two"), + ] + + # The one accepted difference in the parts themselves: the released client + # sent scalars through `requests`' `data=`, which writes only a + # Content-Disposition, while the generated encoder types every scalar as + # text/plain. Django routes on the presence of `filename`, so nothing + # downstream reads the extra header -- but it is a real difference and it + # is asserted rather than left invisible. + scalar_headers = { + part[0]: part[3] for part in _multipart_parts(ours) if part[0] != "files" + } + assert scalar_headers + assert all( + headers == ("content-disposition", "content-type") + for headers in scalar_headers.values() + ) + assert all( + part[3] == ("content-disposition",) + for part in _multipart_parts(theirs) + if part[0] != "files" + ) + + +def test_redirects_are_followed(): + """The released client followed them on both verbs. + + Not following one turns a load balancer's 307 into a body the poll loop + reads as a finished execution with no status. + """ + assert _client()._transport.get_httpx_client().follow_redirects is True + + +def test_a_status_endpoint_without_an_execution_id_is_refused(): + """Polling with a blank id asks the service about an execution nobody has; + what it answers is not this execution's state.""" + with pytest.raises(APIDeploymentsClientException): + _client().check_execution_status("/deployment/api/testorg/testapi/") + + +def test_the_transport_is_untimed_by_default(): + """A connection that stalls forever is what the released client did. + + Bounding it by default would turn a hang into an exception callers have + never had to handle, and ``api_timeout`` cannot serve: it is an execution + mode the backend reads, not a socket timeout. + """ + assert _client()._transport.get_httpx_client().timeout == httpx.Timeout(None) + + +def test_transport_timeout_is_what_the_transport_uses(): + assert _client(transport_timeout=5)._transport.get_httpx_client().timeout == ( + httpx.Timeout(5) + ) + + +def test_transport_timeout_bounds_a_stalled_connection(): + """The call is made off the test thread, so the failure this pins -- a + request that never returns -- fails the test instead of hanging the run.""" + server = socket.socket() + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + server.listen(1) + accepted, outcome = [], [] + + def stall(): + conn, _address = server.accept() + accepted.append(conn) # held open, and answered by nobody + + def call(): + url = f"http://127.0.0.1:{server.getsockname()[1]}/deployment/api/org/name/" + client = _client(api_url=url, transport_timeout=0.2) + try: + client.check_execution_status(STATUS_ENDPOINT) + outcome.append(None) + except BaseException as e: # noqa: BLE001 - reported, not handled + outcome.append(e) + + stalling = threading.Thread(target=stall, daemon=True) + calling = threading.Thread(target=call, daemon=True) + stalling.start() + calling.start() + try: + calling.join(timeout=10) + assert outcome, "the request never returned" + assert isinstance(outcome[0], ReadTimeout) + finally: + stalling.join(timeout=5) + for conn in accepted: + conn.close() + server.close() + + +@pytest.mark.parametrize( + "api_url", ["https://gw.example.com/extract/", "https://api.example.com/onlyone"] +) +def test_a_deployment_url_with_no_route_in_it_is_still_polled(api_url): + """Deriving the route is only load-bearing where the deployment URL carries + the spec route as a suffix. A URL too short to have an organisation and API + name in it -- an ingress rewrite -- takes the branch that reads the endpoint + the service returned, which is the one the released client polled. + + Submitting on these already works, so refusing to poll them strands an + execution that has been paid for. + """ + client = _client(api_url=api_url) + with pytest.raises(APIDeploymentsClientException): + _ = client._deployment_route + + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + ours = client.check_execution_status(STATUS_ENDPOINT) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.request.return_value = _requests_response( + 200, {"status": "COMPLETED"} + ) + theirs = _baseline_client(api_url=api_url).check_execution_status( + STATUS_ENDPOINT + ) + + assert ours == theirs + + args, kwargs = mock_send.call_args + sent = httpx.URL(client.base_url).join(args[1]).copy_merge_params(kwargs["params"]) + published = urlparse(client.base_url + STATUS_ENDPOINT) + assert args[0].lower() == "get" + assert (sent.scheme, sent.host, sent.path) == ( + published.scheme, + published.hostname, + published.path, + ) + assert parse_qs(str(sent.params)) == { + **parse_qs(published.query), + "include_metadata": [str(client.include_metadata)], + } + + +@pytest.mark.parametrize( + "api_url", ["https://gw.example.com/extract/", "https://api.example.com/myapi/"] +) +def test_execute_does_not_need_a_derivable_route(sample_file, api_url): + """The route is only ever spent on a URL execute then discards, so requiring + one would reject deployment URLs the released client posted to -- an ingress + or reverse-proxy rewrite short enough to have no org/API pair in it. + + Only ``check_execution_status``, which rebuilds the poll path, needs it. + """ + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"message": {}}) + _client(api_url=api_url).structure_file([sample_file]) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.post.return_value = _requests_response(200, {"message": {}}) + _baseline_client(api_url=api_url).structure_file([sample_file]) + + assert mock_send.call_args[0][1] == mock_requests.post.call_args[0][0] == api_url + + +def test_the_client_releases_its_connections(): + """The transport is pooled and kept between calls; nothing else closes it, + so a client built per job would accumulate pools until collected.""" + client = _client() + httpx_client = client._transport.get_httpx_client() + client.close() + assert httpx_client.is_closed + assert client._transport_client is None + # Idempotent, and the next call builds a fresh transport rather than + # reaching into a closed pool. + client.close() + assert client._transport.get_httpx_client() is not httpx_client + + +def test_the_client_is_a_context_manager(): + with _client() as client: + httpx_client = client._transport.get_httpx_client() + assert httpx_client.is_closed + + +def test_the_transport_is_built_once_under_contention(): + """Two threads racing the first call would otherwise each build a pool, and + the one that lost would be dropped still holding its sockets.""" + client = _client() + built = [] + barrier = threading.Barrier(8) + + def race(): + barrier.wait() + built.append(client._transport) + + threads = [threading.Thread(target=race) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert len(set(map(id, built))) == 1 + + +# -------------------------------------------------------------------------- +# Return shape, compared against the released client over the same responses +# -------------------------------------------------------------------------- + + +def _message(**fields): + return {"message": fields} + + +EXECUTE_CASES = [ + ("pending", 200, _message(execution_status="PENDING", status_api=STATUS_ENDPOINT)), + ( + "executing", + 200, + _message(execution_status="EXECUTING", status_api=STATUS_ENDPOINT), + ), + ("success", 200, _message(execution_status="SUCCESS", result=[{"file": "a"}])), + ( + "success_without_result", + 200, + _message(execution_status="SUCCESS", status_api=STATUS_ENDPOINT), + ), + ("error", 200, _message(execution_status="ERROR", error="boom")), + ("unauthorized", 401, {"errors": [{"detail": "Invalid token"}]}), + ("unprocessable", 422, _message(execution_status="ERROR", error="bad input")), + ("server_error", 500, _message(execution_status="ERROR", error="oops")), +] + + +@pytest.mark.parametrize( + ("name", "status_code", "body"), EXECUTE_CASES, ids=[c[0] for c in EXECUTE_CASES] +) +def test_structure_file_returns_what_the_released_client_returned( + sample_file, name, status_code, body +): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(status_code, body) + ours = _client(api_timeout=300).structure_file([sample_file]) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.post.return_value = _requests_response(status_code, body) + theirs = _baseline_client(api_timeout=300).structure_file([sample_file]) + + assert ours == theirs + + +def test_structure_file_matches_on_a_non_json_body(sample_file): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(502, text="gateway") + ours = _client(api_timeout=300).structure_file([sample_file]) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.post.return_value = _requests_response( + 502, text="gateway" + ) + theirs = _baseline_client(api_timeout=300).structure_file([sample_file]) + + assert ours == theirs + + +def test_structure_file_missing_file_still_raises(sample_file): + with pytest.raises(APIDeploymentsClientException) as caught: + _client().structure_file(["/nonexistent/file.txt"]) + assert "File not found" in str(caught.value) + with pytest.raises(baseline.APIDeploymentsClientException): + _baseline_client().structure_file(["/nonexistent/file.txt"]) + + +def test_an_unreadable_file_raises_the_documented_exception(tmp_path, sample_file): + """Not only a missing one: a directory or an unreadable path reached the + caller as a raw builtin, past the handles already opened for the files + before it.""" + opened = [] + real_open = open + + def tracking_open(*args, **kwargs): + handle = real_open(*args, **kwargs) + opened.append(handle) + return handle + + directory = tmp_path / "adir" + directory.mkdir() + with patch("builtins.open", side_effect=tracking_open): + with pytest.raises(APIDeploymentsClientException) as caught: + _client().structure_file([sample_file, str(directory)]) + + assert "File not found" not in str(caught.value) + assert opened and all(handle.closed for handle in opened) + + +# Error shapes the API answers with, each one a body the service actually +# builds. Two families coexist on these endpoints and the spec models both: the +# statuses routed through the project's exception handler answer with +# ``{"type", "errors": [{"code", "detail", "attr"}]}``, while the ones the views +# build by hand answer in the endpoint's own envelope -- ``{"message": {...}}`` +# on execute, ``{"status", "message"}`` on the status endpoint. The baseline read +# the reason out of neither, so a caller saw an empty error next to a status +# code -- an accepted difference from parity, and the reason for these tests. +# +# Which family a status answers with is a property of the status, not of the +# endpoint, so a case belongs to whichever operation reaches it. +ERROR_BODY_CASES = [ + ( + "validation", + "execute", + 400, + { + "type": "validation_error", + "errors": [ + { + "code": "invalid", + "detail": "Queue 'nope' does not exist", + "attr": "hitl_queue_name", + } + ], + }, + "Queue 'nope' does not exist", + ), + ( + "validation_multiple", + "execute", + 400, + { + "type": "validation_error", + "errors": [ + {"code": "invalid", "detail": "first", "attr": "a"}, + {"code": "invalid", "detail": "second", "attr": "b"}, + ], + }, + "first; second", + ), + ( + "unauthorized", + "execute", + 401, + { + "type": "client_error", + "errors": [{"code": "error", "detail": "Unauthorized", "attr": None}], + }, + "Unauthorized", + ), + ( + "not_found", + "status", + 404, + { + "type": "client_error", + "errors": [ + { + "code": "error", + "detail": "Execution with ID 'exec-123' does not exist.", + "attr": None, + } + ], + }, + "Execution with ID 'exec-123' does not exist.", + ), + ( + "hand_built_acknowledged", + "status", + 406, + {"status": "COMPLETED", "message": "Result already acknowledged"}, + "Result already acknowledged", + ), + ( + "hand_built_execution_failed", + "execute", + 422, + { + "message": { + "execution_id": "exec-123", + "execution_status": "ERROR", + "error": "Tool run failed", + "result": None, + } + }, + "Tool run failed", + ), +] + +#: Bodies carrying nothing readable, which is what a proxy or gateway in front +#: of the service answers with -- the application's own two families always +#: carry one. Neither may crash, and the body itself is more use to a caller +#: than an empty string. +UNREADABLE_ERROR_BODIES = [ + ("no_detail", 500, {"type": "server_error", "errors": []}), + ("a_list", 502, [1, 2]), + ("a_string", 403, "forbidden"), +] + + +def _report(operation, status_code, body, sample_file=None): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(status_code, body) + if operation == "execute": + return _client(api_timeout=300).structure_file([sample_file]) + return _client().check_execution_status(STATUS_ENDPOINT) + + +@pytest.mark.parametrize( + ("name", "operation", "status_code", "body", "expected"), + ERROR_BODY_CASES, + ids=[c[0] for c in ERROR_BODY_CASES], +) +def test_the_reason_an_error_carries_is_reported( + sample_file, name, operation, status_code, body, expected +): + """The facade reads these as raw dicts on purpose. The generated models are + built per declared status and per declared shape, and the two families above + put the reason in different places, so bridging them is the facade's job and + a model can only ever cover one side of it.""" + result = _report(operation, status_code, body, sample_file) + + assert result["error"] == expected + assert result["status_code"] == status_code + assert result["pending"] is False + # The reason is not also handed back as an extraction: one of these shapes + # carries it under the key a success puts the result under. + assert result["extraction_result"] in ("", None) + + +def test_both_calls_are_covered_by_the_error_bodies_above(): + """A shape belongs to whichever operation reaches it, so the table only + exercises both readers while it names both.""" + assert {case[1] for case in ERROR_BODY_CASES} == WRAPPED_OPERATIONS + + +def test_an_execution_that_failed_without_a_reason_is_not_given_one(sample_file): + """Both endpoints can answer a non-2xx with their own envelope and no reason + in it -- a failed execution whose per-file entries carry the detail. The + envelope has already been read, so nothing is invented out of the raw body + the way a refused request needs.""" + body = { + "message": { + "execution_id": "exec-123", + "execution_status": "ERROR", + "error": "", + "result": None, + } + } + ours = _report("execute", 422, body, sample_file) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.post.return_value = _requests_response(422, body) + theirs = _baseline_client(api_timeout=300).structure_file([sample_file]) + + assert ours == theirs + assert ours["error"] == "" + + +@pytest.mark.parametrize( + ("name", "status_code", "body"), + UNREADABLE_ERROR_BODIES, + ids=[c[0] for c in UNREADABLE_ERROR_BODIES], +) +def test_an_unreadable_error_body_is_reported_not_raised( + sample_file, name, status_code, body +): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(status_code, body) + executed = _client(api_timeout=300).structure_file([sample_file]) + + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(status_code, body) + polled = _client().check_execution_status(STATUS_ENDPOINT) + + for result in (executed, polled): + assert result["status_code"] == status_code + assert json.loads(result["error"]) == body + + +def _declared_responses(operation_id: str) -> dict[int, str]: + """``{status: schema name}`` for one operation, read out of the spec.""" + spec = json.loads(SPEC_PATH.read_text()) + for path in spec["paths"].values(): + for method, operation in path.items(): + if method not in {"get", "post"}: + continue + if operation["operationId"] != operation_id: + continue + return { + int(code): body["content"]["application/json"]["schema"]["$ref"].split( + "/" + )[-1] + for code, body in operation["responses"].items() + } + raise AssertionError(f"{operation_id} not declared in the spec") + + +def _body_for(schema: str) -> tuple[dict, str]: + """A body of the shape the spec declares, and the reason a caller gets back. + + Built from the schema name rather than hardcoded per status, so a spec that + re-points a status at another shape is exercised as the new shape without + this table being touched. + """ + if schema == "ErrorResponse": + return { + "type": "client_error", + # `code` is free-form here, deliberately not an enum: the server + # emits "error" for statuses the handler routes without a subtype. + "errors": [{"code": "error", "detail": "the reason", "attr": None}], + }, "the reason" + if schema == "ExecuteResponse": + return { + "message": { + "execution_id": "exec-123", + "execution_status": "ERROR", + "error": "the reason", + "result": None, + } + }, "the reason" + if schema == "AcknowledgedResponse": + return {"status": "COMPLETED", "message": "the reason"}, "the reason" + # StatusResponse. The status endpoint's own envelope carries per-file + # results, never a reason: on these statuses the execution's state is the + # answer, and any reason is inside a file's own entry. + return {"status": "ERROR", "message": [{"file": "a.pdf", "error": "boom"}]}, "" + + +#: Every error status each operation declares, pinned. Read straight off the +#: spec, the coverage below shrinks in silence when the spec stops declaring one +#: -- which is the direction that costs a caller an unreported status. +DECLARED_ERROR_STATUSES = { + "execute": {400, 401, 403, 404, 413, 422, 429, 500, 502, 504}, + "status": {400, 401, 403, 404, 406, 422, 500}, +} + +#: Statuses the client does something bespoke for -- ``Retry-After`` on 429, a +#: retry on each of the 5xx the deployment answers with. +SPECIAL_CASED_STATUSES = {429, 500, 502, 504} + + +@pytest.mark.parametrize("operation", sorted(WRAPPED_OPERATIONS)) +def test_every_error_status_the_spec_declares_is_reported(sample_file, operation): + """The spec and this client have to agree on the error body, and they + disagreed once: the facade read one shape while the spec declared another. + + Every declared shape is read here, and a status the spec adds arrives as a + case rather than as an empty ``error`` in production. A status it drops + fails the manifest above instead of quietly leaving this loop. + """ + declared = _declared_responses(operation) + errors = {code: schema for code, schema in declared.items() if code != 200} + assert set(errors) == DECLARED_ERROR_STATUSES[operation], operation + # Both families are actually in play; a regression to one of them is a + # narrowing this would otherwise not notice. + assert set(errors.values()) > {"ErrorResponse"} + + for status_code, schema in errors.items(): + body, expected = _body_for(schema) + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(status_code, body) + if operation == "execute": + result = _client(api_timeout=300).structure_file([sample_file]) + else: + result = _client().check_execution_status(STATUS_ENDPOINT) + + assert result["status_code"] == status_code, (operation, status_code) + assert result["error"] == expected, (operation, status_code, schema) + + +def test_the_statuses_the_client_handles_specially_are_declared(): + """A status carrying dedicated handling that no operation declares is either + dead code here or a gap in the spec, and nothing else would say which.""" + client = _client() + assert all(client._is_retryable_status(code) for code in SPECIAL_CASED_STATUSES) + declared = set().union(*DECLARED_ERROR_STATUSES.values()) + assert SPECIAL_CASED_STATUSES <= declared + + +def test_the_generated_models_read_the_bodies_the_server_sends(): + """The generated models are an implementation detail, but a wrong one is a + trap for anyone who imports them: the facade would keep working while the + models silently lost the payload.""" + from unstract.api_deployments._sdk_docstudio.models import ( + ErrorResponse, + ExecutionMessage, + FileResult, + ) + + error = ErrorResponse.from_dict(_body_for("ErrorResponse")[0]) + assert [detail.detail for detail in error.errors] == ["the reason"] + assert error.type_ == "client_error" + assert not error.additional_properties + + # `error` and `status_api` are absent on a success and null on a failure; + # declared required, either one raised a bare KeyError. + message = ExecutionMessage.from_dict( + {"execution_id": "exec-123", "execution_status": "SUCCESS"} + ) + assert message.error is UNSET and message.status_api is UNSET + + # `metrics` is not a field here and should not be: `include_metrics` turns + # it on inside the untyped `result` payload, which the spec declares as + # opaque, so it never reached the top level of a file result. + result = FileResult.from_dict( + { + "file": "a.pdf", + "status": "SUCCESS", + "extracted_text": "hello", + "result": {"output": {}, "metrics": {"elapsed": 1}}, + } + ) + assert result.extracted_text == "hello" + assert result.result["metrics"] == {"elapsed": 1} + assert not result.additional_properties + assert not hasattr(result, "metrics") + + +def test_the_custom_data_round_trip_is_documented_as_undeclared(): + """``structure_file`` documents ``custom_data`` coming back under each result + item's ``metadata.custom_data``. That is what the service does, and it is + worth documenting -- but the spec carries the field on the request only, so + nothing here pins the round trip and the docstring has to say so.""" + schemas = json.loads(SPEC_PATH.read_text())["components"]["schemas"] + assert "custom_data" in schemas["ExecuteRequest"]["properties"] + assert not any( + "custom_data" in schema.get("properties", {}) + for name, schema in schemas.items() + if name != "ExecuteRequest" + ) + + doc = inspect.getdoc(APIDeploymentsClient.structure_file) + claim = doc[doc.index("custom_data (Any)") :] + assert "metadata.custom_data" in claim + assert "not declared" in claim + + +def test_a_success_envelope_is_never_read_as_an_error(sample_file): + """The fallback only fires where the envelope had nothing: a non-2xx that + still carries one is read out of it, the way the released client did.""" + body = {"message": {"execution_status": "ERROR", "error": "bad input"}} + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(422, body) + ours = _client(api_timeout=300).structure_file([sample_file]) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.post.return_value = _requests_response(422, body) + theirs = _baseline_client(api_timeout=300).structure_file([sample_file]) + + assert ours == theirs + assert ours["error"] == "bad input" + + +def test_a_status_endpoint_without_an_execution_id_does_not_report_its_query(): + """The documented usage prints this exception straight to a log, and the + query is the service's to shape.""" + with pytest.raises(APIDeploymentsClientException) as caught: + _client().check_execution_status( + "/deployment/api/testorg/testapi/?token=s3cret-signature" + ) + assert "s3cret-signature" not in str(caught.value) + assert "/deployment/api/testorg/testapi/" in str(caught.value) + + +def test_structure_file_closes_its_handles(sample_file): + """The released client leaked these; closing them is invisible to + callers.""" + opened = [] + real_open = open + + def tracking_open(*args, **kwargs): + handle = real_open(*args, **kwargs) + opened.append(handle) + return handle + + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"message": {}}) + with patch("builtins.open", side_effect=tracking_open): + _client().structure_file([sample_file]) + + assert opened and all(handle.closed for handle in opened) + + +#: What the status endpoint answers with, in the shape it builds them: the +#: execution's state plus its per-file results, and 200 only once the execution +#: has completed -- an in-progress poll is a 422 carrying the same envelope. The +#: reason a failed execution carries lives in a file's own entry, not beside the +#: envelope, so none of these is read as a refused request. +STATUS_CASES = [ + ("completed", 200, {"status": "COMPLETED", "message": [{"file": "a.pdf"}]}), + ("executing", 422, {"status": "EXECUTING", "message": None}), + ("queued", 422, {"status": "QUEUED", "message": None}), + ( + "error", + 422, + {"status": "ERROR", "message": [{"file": "a.pdf", "error": "boom"}]}, + ), + ( + "tool_not_found", + 500, + {"status": "ERROR", "message": [{"file": "a.pdf", "error": "no such tool"}]}, + ), +] + + +@pytest.mark.parametrize( + ("name", "status_code", "body"), STATUS_CASES, ids=[c[0] for c in STATUS_CASES] +) +def test_check_execution_status_returns_what_the_released_client_returned( + name, status_code, body +): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(status_code, body) + ours = _client().check_execution_status(STATUS_ENDPOINT) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.request.return_value = _requests_response(status_code, body) + theirs = _baseline_client().check_execution_status(STATUS_ENDPOINT) + + assert ours == theirs + + +def test_check_execution_status_matches_on_a_non_json_body(): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(502, text="gateway") + ours = _client().check_execution_status(STATUS_ENDPOINT) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.request.return_value = _requests_response( + 502, text="gateway" + ) + theirs = _baseline_client().check_execution_status(STATUS_ENDPOINT) + + assert ours == theirs + + +# -------------------------------------------------------------------------- +# Construction and surface +# -------------------------------------------------------------------------- + + +def _baseline_class_node(): + tree = ast.parse(BASELINE_PATH.read_text()) + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == "APIDeploymentsClient": + return node + raise AssertionError("APIDeploymentsClient not found in the baseline") + + +def _baseline_init_params(): + """Constructor parameters and defaults, read out of the baseline source. + + Parsed rather than imported so the comparison is against the + released text, not against whatever a shared import happened to + bind. + """ + for node in _baseline_class_node().body: + if isinstance(node, ast.FunctionDef) and node.name == "__init__": + args = node.args.args[1:] + defaults = [None] * (len(args) - len(node.args.defaults)) + [ + ast.literal_eval(d) for d in node.args.defaults + ] + return list(zip([a.arg for a in args], defaults)) + raise AssertionError("__init__ not found in the baseline") + + +def test_constructor_parameters_are_unchanged(): + """Names, order and defaults all matter: callers pass some positionally.""" + live = inspect.signature(APIDeploymentsClient.__init__).parameters + live_params = [ + (name, None if p.default is inspect.Parameter.empty else p.default) + for name, p in live.items() + # Keyword-only parameters are excluded: they cannot be reached by any + # existing call, so adding one leaves every released call shape intact. + if name != "self" and p.kind is not p.KEYWORD_ONLY + ] + assert live_params == _baseline_init_params() + + +def test_public_methods_are_unchanged(): + baseline_methods = { + node.name: node + for node in _baseline_class_node().body + if isinstance(node, ast.FunctionDef) and not node.name.startswith("_") + } + assert baseline_methods + + for name, node in baseline_methods.items(): + live = getattr(APIDeploymentsClient, name, None) + assert live is not None, f"{name} disappeared from the client" + # Keyword-only parameters are excluded: they cannot be reached by any + # existing call, so adding one leaves every released call shape intact. + live_args = [ + arg + for arg, p in inspect.signature(live).parameters.items() + if arg not in ("self", "cls") and p.kind is not p.KEYWORD_ONLY + ] + assert live_args == [a.arg for a in node.args.args[1:]], name + + +def test_class_attributes_are_unchanged(): + compared = 0 + for node in _baseline_class_node().body: + if not isinstance(node, ast.Assign): + continue + try: + value = ast.literal_eval(node.value) + except ValueError: + continue # logger and friends: identity, not value + for target in node.targets: + assert getattr(APIDeploymentsClient, target.id) == value, target.id + compared += 1 + # A parse that matches no node asserts nothing and still reports green. + assert compared + + +def test_module_level_names_are_unchanged(): + import unstract.api_deployments.client as live + + tree = ast.parse(BASELINE_PATH.read_text()) + names = [ + node.name + for node in tree.body + if isinstance(node, ast.ClassDef) and not node.name.startswith("_") + ] + assert names + for name in names: + assert hasattr(live, name), name + + +#: What an install of this package puts on the path. A removal here is a +#: breaking change for anyone whose deploy runs the command or imports the +#: module, so it has to be made in the same diff that moves this list. +INSTALLED_CONSOLE_SCRIPTS: dict[str, str] = {} +INSTALLED_TOP_LEVEL_MODULES = {"unstract.api_deployments", "unstract.clone"} + +#: What the released baseline's install put on the path. Everything the wheel +#: ships comes from ``src/unstract``, and this is smaller now. +BASELINE_CONSOLE_SCRIPTS = {"unstract": "unstract.cli:main"} + + +def test_the_packaging_surface_is_what_it_claims(): + """The wire behaviour above is all pinned against the baseline and stays + green through a console script being deleted, which is the one published + contract this change does break. + + Every entry under ``src/unstract`` counts, not just the packages: the build + ships the directory whole, so a top-level module dropped beside them is + installed as ``unstract.`` without appearing as a package at all. + """ + pyproject = tomllib.loads( + (Path(__file__).parents[1] / "pyproject.toml").read_text() + ) + assert pyproject["project"].get("scripts", {}) == INSTALLED_CONSOLE_SCRIPTS + + src = Path(__file__).parents[1] / "src" / "unstract" + packaged = { + f"unstract.{path.stem}" + for path in src.iterdir() + if path.name != "__pycache__" + and (path.is_dir() or path.suffix in {".py", ".pyi"}) + } + assert packaged == INSTALLED_TOP_LEVEL_MODULES + + for module in INSTALLED_TOP_LEVEL_MODULES: + assert importlib.util.find_spec(module) is not None, module + + +def test_a_shrinking_packaging_surface_cannot_ship_as_a_patch(): + """The release workflow does not publish the version recorded here: it reads + it as the last released one and applies the bump chosen at dispatch. So this + still reads the baseline's version until a release runs, and once one does, + a surface smaller than that baseline's cannot have been published as a patch + over it. + """ + assert set(BASELINE_CONSOLE_SCRIPTS) - set(INSTALLED_CONSOLE_SCRIPTS) + + ours = tuple(int(part) for part in api_deployments.__version__.split(".")) + theirs = tuple(int(part) for part in BASELINE_VERSION.split(".")) + assert ours >= theirs + assert ours == theirs or ours[:2] > theirs[:2], api_deployments.__version__ + + +def test_the_vendored_spec_is_the_revision_the_generator_pins(): + """The provenance pin is a comment, and a comment is enforced by nothing: a + spec edited in place goes on claiming the upstream revision it is no longer + a copy of, and the drift gate reports clean either way.""" + pinned = re.search(r"sha256 ([0-9a-f]{64})", GENERATOR_PATH.read_text()) + assert pinned, GENERATOR_PATH + assert hashlib.sha256(SPEC_PATH.read_bytes()).hexdigest() == pinned.group(1) + + +def test_every_declared_operation_is_wrapped(): + """A new spec operation shows up here as a failure, not as silence. + + Compared whole rather than after subtracting an exception list: an entry + excusing an operation the spec no longer declares keeps passing forever, and + nothing about a green run says the list is still describing anything. + """ + spec = json.loads(SPEC_PATH.read_text()) + declared = { + operation["operationId"] + for path in spec["paths"].values() + for method, operation in path.items() + if method in {"get", "post", "put", "patch", "delete"} + } + assert declared == WRAPPED_OPERATIONS + + +def test_the_baseline_is_the_released_client_unmodified(): + # A digest, not a version string in a comment: an edited baseline can claim + # any provenance it likes, and every parity test here would still pass. + assert BASELINE_PATH.name == f"client_{BASELINE_VERSION.replace('.', '_')}.py" + assert hashlib.sha256(BASELINE_PATH.read_bytes()).hexdigest() == BASELINE_SHA256 diff --git a/tests/test_retry.py b/tests/test_retry.py index d0acb81..04676a3 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -1,6 +1,9 @@ """Tests for the exponential backoff retry logic in APIDeploymentsClient.""" import io +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from unittest.mock import MagicMock, patch import pytest @@ -212,7 +215,7 @@ def test_exception_outcome_uses_exponential(self): class TestRequestWithRetrySuccess: - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_success_on_first_try(self, mock_request, client): mock_request.return_value = _mock_response(200) resp = client._request_with_retry("GET", "https://api.example.com/test") @@ -220,7 +223,7 @@ def test_success_on_first_try(self, mock_request, client): assert mock_request.call_count == 1 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_503_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(503), @@ -232,7 +235,7 @@ def test_retry_on_503_then_success(self, mock_request, mock_sleep, client): assert mock_sleep.call_count == 1 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_500_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(500), @@ -245,7 +248,7 @@ def test_retry_on_500_then_success(self, mock_request, mock_sleep, client): assert mock_sleep.call_count == 2 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_429_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(429), @@ -256,7 +259,7 @@ def test_retry_on_429_then_success(self, mock_request, mock_sleep, client): assert mock_request.call_count == 2 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_502_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(502), @@ -267,7 +270,7 @@ def test_retry_on_502_then_success(self, mock_request, mock_sleep, client): assert mock_request.call_count == 2 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_504_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(504), @@ -283,7 +286,7 @@ def test_retry_on_504_then_success(self, mock_request, mock_sleep, client): class TestRequestWithRetryConnectionErrors: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_connection_error_then_success( self, mock_request, mock_sleep, client ): @@ -296,7 +299,7 @@ def test_retry_on_connection_error_then_success( assert mock_request.call_count == 2 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_timeout_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ Timeout("Request timed out"), @@ -307,7 +310,7 @@ def test_retry_on_timeout_then_success(self, mock_request, mock_sleep, client): assert mock_request.call_count == 2 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_connection_error_exhausted_raises(self, mock_request, mock_sleep, client): mock_request.side_effect = ConnectionError("Connection refused") with pytest.raises(ConnectionError): @@ -315,7 +318,7 @@ def test_connection_error_exhausted_raises(self, mock_request, mock_sleep, clien assert mock_request.call_count == client.max_retries + 1 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_timeout_exhausted_raises(self, mock_request, mock_sleep, client): mock_request.side_effect = Timeout("Request timed out") with pytest.raises(Timeout): @@ -328,7 +331,7 @@ def test_timeout_exhausted_raises(self, mock_request, mock_sleep, client): class TestRequestWithRetryExhaustion: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_all_retries_exhausted_returns_last_response( self, mock_request, mock_sleep, client ): @@ -342,35 +345,35 @@ def test_all_retries_exhausted_returns_last_response( class TestNoRetryOnNonRetryable: - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_on_200(self, mock_request, client): mock_request.return_value = _mock_response(200) resp = client._request_with_retry("GET", "https://api.example.com/test") assert resp.status_code == 200 assert mock_request.call_count == 1 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_on_400(self, mock_request, client): mock_request.return_value = _mock_response(400) resp = client._request_with_retry("GET", "https://api.example.com/test") assert resp.status_code == 400 assert mock_request.call_count == 1 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_on_401(self, mock_request, client): mock_request.return_value = _mock_response(401) resp = client._request_with_retry("GET", "https://api.example.com/test") assert resp.status_code == 401 assert mock_request.call_count == 1 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_on_404(self, mock_request, client): mock_request.return_value = _mock_response(404) resp = client._request_with_retry("GET", "https://api.example.com/test") assert resp.status_code == 404 assert mock_request.call_count == 1 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_on_422(self, mock_request, client): mock_request.return_value = _mock_response(422) resp = client._request_with_retry("GET", "https://api.example.com/test") @@ -382,7 +385,7 @@ def test_no_retry_on_422(self, mock_request, client): class TestTimeoutPassed: - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_default_timeout_set(self, mock_request, client): """api_timeout is a server-side parameter, not an HTTP socket timeout. @@ -393,7 +396,7 @@ def test_no_default_timeout_set(self, mock_request, client): _, kwargs = mock_request.call_args assert "timeout" not in kwargs - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_explicit_timeout_not_overridden(self, mock_request, client): """Callers can still pass an explicit HTTP socket timeout.""" mock_request.return_value = _mock_response(200) @@ -407,7 +410,7 @@ def test_explicit_timeout_not_overridden(self, mock_request, client): class TestRetryAfterHeader: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_429_respects_retry_after_header(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(429, headers={"Retry-After": "5"}), @@ -418,7 +421,7 @@ def test_429_respects_retry_after_header(self, mock_request, mock_sleep, client) mock_sleep.assert_called_once_with(5.0) @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_429_invalid_retry_after_falls_back(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(429, headers={"Retry-After": "not-a-number"}), @@ -437,7 +440,7 @@ def test_429_invalid_retry_after_falls_back(self, mock_request, mock_sleep, clie class TestFileSeekOnRetry: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_file_objects_rewound_on_retry(self, mock_request, mock_sleep, client): file_obj = io.BytesIO(b"test data") files = [("files", ("test.pdf", file_obj, "application/octet-stream"))] @@ -457,7 +460,7 @@ def test_file_objects_rewound_on_retry(self, mock_request, mock_sleep, client): class TestDisabledRetry: - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_when_max_retries_zero(self, mock_request, client_no_retry): mock_request.return_value = _mock_response(503) resp = client_no_retry._request_with_retry( @@ -466,7 +469,7 @@ def test_no_retry_when_max_retries_zero(self, mock_request, client_no_retry): assert resp.status_code == 503 assert mock_request.call_count == 1 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_connection_error_raises_immediately_when_disabled( self, mock_request, client_no_retry ): @@ -481,27 +484,27 @@ def test_connection_error_raises_immediately_when_disabled( class TestCheckExecutionStatusPendingFix: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_503_after_exhaustion_sets_pending_true( self, mock_request, mock_sleep, client ): mock_request.return_value = _mock_response( 503, json_data={"status": "", "error": "Service Unavailable", "message": ""} ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is True assert result["status_code"] == 503 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_200_with_pending_status_sets_pending_true(self, mock_request, client): mock_request.return_value = _mock_response( 200, json_data={"status": "EXECUTING", "error": "", "message": ""} ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is True assert result["status_code"] == 200 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_200_with_completed_status_sets_pending_false(self, mock_request, client): mock_request.return_value = _mock_response( 200, @@ -511,10 +514,10 @@ def test_200_with_completed_status_sets_pending_false(self, mock_request, client "message": '{"result": "data"}', }, ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is False - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_422_with_executing_status_sets_pending_true(self, mock_request, client): """HTTP 422 is currently returned by Unstract for in-progress statuses. @@ -526,27 +529,27 @@ def test_422_with_executing_status_sets_pending_true(self, mock_request, client) mock_request.return_value = _mock_response( 422, json_data={"status": "EXECUTING", "error": "", "message": ""} ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is True assert result["status_code"] == 422 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_422_with_pending_status_sets_pending_true(self, mock_request, client): """HTTP 422 with PENDING body status — still detected via body check.""" mock_request.return_value = _mock_response( 422, json_data={"status": "PENDING", "error": "", "message": ""} ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is True assert result["status_code"] == 422 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_400_does_not_set_pending(self, mock_request, client): mock_request.return_value = _mock_response( 400, json_data={"status": "", "error": "Bad request", "message": ""} ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is False @@ -555,7 +558,7 @@ def test_400_does_not_set_pending(self, mock_request, client): class TestStructureFileUsesRetry: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_structure_file_retries_on_503_async_mode( self, mock_request, mock_sleep, tmp_path ): @@ -591,7 +594,7 @@ def test_structure_file_retries_on_503_async_mode( class TestStructureFileNoRetryInSyncMode: - @patch("unstract.api_deployments.client.requests.post") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_structure_file_no_retry_on_503_sync_mode(self, mock_post, tmp_path): """In sync mode (api_timeout>0), POST is NOT retried on 5xx.""" test_file = tmp_path / "test.pdf" @@ -620,7 +623,7 @@ def test_structure_file_no_retry_on_503_sync_mode(self, mock_post, tmp_path): # Only one call — no retries in sync mode assert mock_post.call_count == 1 - @patch("unstract.api_deployments.client.requests.post") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_structure_file_sync_mode_default_timeout(self, mock_post, tmp_path): """Default api_timeout=300 means sync mode — no POST retry.""" test_file = tmp_path / "test.pdf" @@ -654,7 +657,7 @@ def test_structure_file_sync_mode_default_timeout(self, mock_post, tmp_path): class TestStructureFile422DoesNotSetPending: - @patch("unstract.api_deployments.client.requests.post") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_422_pending_does_not_set_pending(self, mock_post, tmp_path): """POST 422 with PENDING status should NOT set pending=True. @@ -686,7 +689,7 @@ def test_422_pending_does_not_set_pending(self, mock_post, tmp_path): assert result["pending"] is False assert result["status_code"] == 422 - @patch("unstract.api_deployments.client.requests.post") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_422_executing_does_not_set_pending(self, mock_post, tmp_path): """POST 422 with EXECUTING status should NOT set pending=True. @@ -717,7 +720,7 @@ def test_422_executing_does_not_set_pending(self, mock_post, tmp_path): assert result["pending"] is False assert result["status_code"] == 422 - @patch("unstract.api_deployments.client.requests.post") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_200_pending_sets_pending_true(self, mock_post, tmp_path): """POST 200 + PENDING correctly sets pending=True for polling.""" test_file = tmp_path / "test.pdf" @@ -743,3 +746,63 @@ def test_200_pending_sets_pending_true(self, mock_post, tmp_path): result = c.structure_file([str(test_file)]) assert result["pending"] is True assert result["status_code"] == 200 + + +def test_a_retried_upload_replays_the_whole_body(tmp_path): + """A retry has to put the same bytes on the wire as the first attempt. + + The multipart body is built from open file handles, which the first + attempt reads to EOF. Without a rewind between attempts the retry sends a + body missing the file it is uploading -- and no test that stubs the + transport can see it, because the encoding is what drops the bytes. + """ + content = b"PDFBYTES" * 512 + upload = tmp_path / "sample.pdf" + upload.write_bytes(content) + attempts = [] + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def do_POST(self): # noqa: N802 -- the name BaseHTTPRequestHandler dispatches to + declared = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(declared) + attempts.append((declared, len(body), content in body)) + first = len(attempts) == 1 + payload = json.dumps( + {"message": "upstream unavailable"} + if first + else {"message": {"execution_status": "COMPLETED", "result": "ok"}} + ).encode() + self.send_response(502 if first else 200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + c = APIDeploymentsClient( + api_url=f"http://127.0.0.1:{server.server_address[1]}/deploy", + api_key="test-key", + logging_level="ERROR", + max_retries=2, + initial_delay=0.01, + max_delay=0.02, + ) + try: + # Queuing mode: the synchronous mode deliberately does not retry an + # upload, so this is the only path a second attempt can be reached on. + c.structure_file([str(upload)], timeout=-1) + finally: + server.shutdown() + server.server_close() + + assert len(attempts) == 2, "the 502 was not retried, so nothing was replayed" + declared, received, carried_file = attempts[0] + assert carried_file + assert attempts == [(declared, declared, True)] * 2 diff --git a/tools/gen_sdk.sh b/tools/gen_sdk.sh new file mode 100755 index 0000000..ac1c4cf --- /dev/null +++ b/tools/gen_sdk.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Regenerate the transport layer from the committed OpenAPI spec. +# +# The generated tree is committed but NEVER hand-edited: regeneration overwrites +# it wholesale, so a fix applied there is lost on the next run. Fixes belong in +# the facade (client.py) or upstream in the spec. +# +# The spec itself is produced by the backend that serves these endpoints +# (`manage.py generate_docstudio_spec`); refresh it from there rather than +# editing it here, and move SPEC_SOURCE below in the same commit. +# +# `git add -N` first: a diff alone cannot see a file the generator has newly +# created, which is exactly what a spec growing an endpoint does. +# +# ./tools/gen_sdk.sh \ +# && git add -N -- src/unstract/api_deployments/_sdk_docstudio \ +# && git diff --stat -- src/unstract/api_deployments/_sdk_docstudio +# +# Where specs/docstudio-oss.json was copied from. Without a revision recorded, +# nothing distinguishes a current copy from one the backend has moved past, and +# both this script and the drift gate report clean either way. +# +# SPEC_SOURCE: Zipstack/unstract specs/docstudio-oss.json +# @ eddd4b746765c77a3d6f64b428fd35d2261e60e7 +# sha256 e453d4f7444d3757a24a1da73373b11c3d362ceb2d7e13e8658a5b3c068b86f5 +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV="$REPO/.gen-venv" +# The generator shells out to ruff for its own post-processing. Without this it +# finds whatever ruff the caller happens to have, or none, and reports the miss +# as a warning -- which the gate below reads as an unparsable spec. +export PATH="$VENV/bin:$PATH" +OUT="src/unstract/api_deployments/_sdk_docstudio" +# Pinned: unpinned, a generator upgrade and a spec change produce the same diff, +# and the drift gate can no longer tell them apart. +GENERATOR="openapi-python-client==0.29.0" + +if [ ! -x "$VENV/bin/openapi-python-client" ]; then + uv venv "$VENV" + uv pip install --python "$VENV/bin/python" "$GENERATOR" +fi + +want="${GENERATOR#*==}" +have="$("$VENV/bin/openapi-python-client" --version | awk '{print $NF}')" +if [ "$have" != "$want" ]; then + echo "generator is $have, expected $want — reinstalling" >&2 + uv pip install --python "$VENV/bin/python" "$GENERATOR" +fi + +rm -rf "${REPO:?}/$OUT" +log="$(mktemp)" +trap 'rm -f "$log"' EXIT +(cd "$REPO" && "$VENV/bin/openapi-python-client" generate \ + --path "$REPO/specs/docstudio-oss.json" --output-path "$REPO/$OUT" \ + --config "$REPO/tools/openapi-client.yaml" --overwrite --meta none) 2>&1 | tee "$log" + +# The generator downgrades a schema it cannot parse to a warning, drops +# the endpoint or model it belongs to, writes the rest and exits 0. The +# result is a client missing an operation and a spec that still looks fine. +if grep -qi warning "$log"; then + echo "the generator reported a problem above and still exited 0; whatever it could not parse is missing from the output" >&2 + exit 1 +fi + +# Stamp every file, so the rule survives contact with a reader who arrived via +# grep rather than via this script. +header='# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT.' +find "$REPO/$OUT" -name '*.py' -print0 | while IFS= read -r -d '' f; do + printf '%s\n%s\n' "$header" "$(cat "$f")" > "$f.tmp" && mv "$f.tmp" "$f" +done + +echo "generated $OUT ($(find "$REPO/$OUT" -name '*.py' | wc -l) files)" diff --git a/tools/openapi-client.yaml b/tools/openapi-client.yaml new file mode 100644 index 0000000..d2196b9 --- /dev/null +++ b/tools/openapi-client.yaml @@ -0,0 +1,3 @@ +# openapi-python-client config. Kept minimal on purpose: every knob here is +# maintenance surface, and post-processing the generated code is a kill criterion. +literal_enums: true diff --git a/tools/refresh_baseline.sh b/tools/refresh_baseline.sh new file mode 100755 index 0000000..065cfb8 --- /dev/null +++ b/tools/refresh_baseline.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Refresh the vendored parity baseline in tests/baseline/. +# +# The compat suite compares this client against the last RELEASED one, not +# against the working tree — a baseline that moves with local edits measures +# nothing. It is vendored rather than downloaded at test time so the suite stays +# offline, and refreshing it is a deliberate act with a reviewable diff. +# +# ./tools/refresh_baseline.sh 1.5.3 +set -euo pipefail + +VERSION="${1:?usage: refresh_baseline.sh }" +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SLUG="${VERSION//./_}" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +(cd "$WORK" && pip download "unstract-client==$VERSION" --no-deps -q && unzip -o -q ./*.whl -d x) + +OUT="$REPO/tests/baseline/client_$SLUG.py" +{ + echo "# Vendored from the released unstract-client $VERSION wheel on PyPI. DO NOT EDIT." + echo "# Refresh with tools/refresh_baseline.sh when the parity baseline is intentionally moved." + cat "$WORK/x/unstract/api_deployments/client.py" +} > "$OUT" + +echo "wrote $OUT" +echo "in tests/test_compat.py set:" +echo " BASELINE_VERSION = \"$VERSION\"" +echo " BASELINE_SHA256 = \"$(sha256sum "$OUT" | cut -d' ' -f1)\"" diff --git a/uv.lock b/uv.lock index 99dffef..fe306ba 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,28 @@ version = 1 revision = 3 requires-python = ">=3.11" +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "certifi" version = "2026.1.4" @@ -267,6 +289,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/04/a94ebfb4eaaa08db56725a40de2887e95de4e8641b9e902c311bfa00aa39/filelock-3.24.2-py3-none-any.whl", hash = "sha256:667d7dc0b7d1e1064dd5f8f8e80bdac157a6482e8d2e02cd16fd3b6b33bd6556", size = 24152, upload-time = "2026-02-16T02:50:44Z" }, ] +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + [[package]] name = "identify" version = "2.6.16" @@ -799,7 +858,9 @@ wheels = [ name = "unstract-client" source = { editable = "." } dependencies = [ + { name = "attrs" }, { name = "click" }, + { name = "httpx" }, { name = "requests" }, { name = "rich" }, { name = "tenacity" }, @@ -837,7 +898,9 @@ test = [ [package.metadata] requires-dist = [ + { name = "attrs", specifier = ">=23.2" }, { name = "click", specifier = ">=8.1" }, + { name = "httpx", specifier = ">=0.27,<0.29" }, { name = "requests", specifier = ">=2.32.3" }, { name = "rich", specifier = ">=13.7" }, { name = "tenacity", specifier = ">=8.2.0" },