diff --git a/apps/sim/lib/core/utils/fetch-deadline.test.ts b/apps/sim/lib/core/utils/fetch-deadline.test.ts index 750616ce1f7..667e6f0062b 100644 --- a/apps/sim/lib/core/utils/fetch-deadline.test.ts +++ b/apps/sim/lib/core/utils/fetch-deadline.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { Agent } from 'undici' import { describe, expect, it } from 'vitest' import { isTransportTimeoutError, withCallerOwnedDeadline } from '@/lib/core/utils/fetch-deadline' @@ -14,6 +15,20 @@ describe('withCallerOwnedDeadline', () => { expect(withCallerOwnedDeadline({}).timeout).toBe(false) }) + /* + * Tests run under Node, where `timeout: false` is ignored and undici's + * 300s `headersTimeout` default is what killed async (Trigger.dev) sandbox + * runs. The dispatcher is what disarms it there; dropping it regresses every + * worker-side internal route call longer than five minutes. + */ + it('attaches a dispatcher on runtimes whose fetch is undici', () => { + expect(withCallerOwnedDeadline({}).dispatcher).toBeInstanceOf(Agent) + }) + + it('reuses one dispatcher across calls so pooled connections are shared', () => { + expect(withCallerOwnedDeadline({}).dispatcher).toBe(withCallerOwnedDeadline({}).dispatcher) + }) + it('preserves the init the caller already built', () => { const signal = new AbortController().signal const init = withCallerOwnedDeadline({ method: 'POST', body: 'x', signal }) @@ -26,6 +41,7 @@ describe('withCallerOwnedDeadline', () => { const original: RequestInit = { method: 'POST' } withCallerOwnedDeadline(original) expect('timeout' in original).toBe(false) + expect('dispatcher' in original).toBe(false) }) }) diff --git a/apps/sim/lib/core/utils/fetch-deadline.ts b/apps/sim/lib/core/utils/fetch-deadline.ts index 9673fdfac11..cd04b09c4e1 100644 --- a/apps/sim/lib/core/utils/fetch-deadline.ts +++ b/apps/sim/lib/core/utils/fetch-deadline.ts @@ -1,25 +1,40 @@ +import { Agent, type Dispatcher } from 'undici' + /** * Keeps the transport deadline from undercutting the application deadline. * - * Bun's HTTP client arms an idle timer defaulting to 300s. It is not raised by - * an `AbortSignal`, and it does not re-arm while awaiting response headers, so - * it acts as an absolute deadline for the peer to begin answering. Any request - * whose peer legitimately works before it replies dies at five minutes no - * matter what deadline the caller computed for it. + * Both runtimes this code executes under ship a five-minute transport default + * that is not raised by an `AbortSignal`, so it acts as an absolute deadline + * for the peer to begin answering. Any request whose peer legitimately works + * before it replies dies at five minutes no matter what deadline the caller + * computed for it: + * + * - Bun's HTTP client arms an idle timer defaulting to 300s. It does not + * re-arm while awaiting response headers. `timeout: false` disarms it. + * - Node's fetch is undici, whose default dispatcher arms `headersTimeout` + * and `bodyTimeout`, both defaulting to 300e3. An expiry surfaces as + * `TypeError: fetch failed` with cause `HeadersTimeoutError` + * (`UND_ERR_HEADERS_TIMEOUT`). A request-scoped `dispatcher` that arms + * neither timer disarms it. * - * This bit production. Workflow function blocks are bounded by a plan deadline - * (50 minutes on enterprise), but the executor's call into the internal - * function route inherited Bun's default instead, so every sandbox run longer - * than five minutes failed with a bare `fetch failed` that read as user-code - * failure rather than a transport cap. + * This bit production twice, once per runtime. Workflow function blocks are + * bounded by a plan deadline (50 minutes on enterprise, 7 days async), but the + * executor's call into the internal function route inherited the transport + * default instead, so every sandbox run longer than five minutes failed with a + * bare `fetch failed` that read as user-code failure rather than a transport + * cap. The first fix (`timeout: false`) covered the app server, which runs + * Bun; async executions run in Trigger.dev workers (`runtime: 'node-24'` in + * `trigger.config.ts`), where that option is silently ignored and the same + * five-minute death reappeared as + * `Transport failure calling function_execute after 300401ms`. * - * The timer is therefore disarmed rather than re-negotiated: callers on this + * The timers are therefore disarmed rather than re-negotiated: callers on this * path already own an in-process deadline (an `AbortController` armed with the * plan timeout), and a second, shorter, invisible deadline underneath it is * exactly the bug. Disarming leaves one enforcement point instead of two that * disagree. * - * The pinned runtime accepts only the boolean/zero form. Measured on Bun 1.3.14 + * Bun accepts only the boolean/zero form of `timeout`. Measured on Bun 1.3.14 * against a server that withholds response headers, so the numbers below are * the real deadline rather than an inferred one: * @@ -32,24 +47,50 @@ * on Bun's `main`. Do not "improve" this into a numeric pass-through until the * pinned version supports it, and re-measure with the probe above if you do. * - * `bun-types@1.3.14` does not declare `timeout` on `BunFetchRequestInit` even - * though the runtime honors the boolean form — the types lag the runtime, which - * is why the interface below is declared locally rather than imported. + * Measured on Node 23.11 (built-in fetch, bundled undici 6.21.2) driving an + * npm `undici@7.29.0` `Agent` — a wider version split than the node-24 workers + * run, so the cross-copy `dispatcher` handoff is proven, not assumed: * - * Node's undici has no equivalent default and ignores the option, so this is - * safe on both runtimes. + * timeout: false only -> THREW 300996ms (fetch failed, + * HeadersTimeoutError) <- ignored + * dispatcher armed at 200ms -> THREW 1011ms <- honored + * dispatcher with 0/0 vs a 310s server -> RESOLVED 310016ms <- disarmed + * + * `bun-types@1.3.14` does not declare `timeout` on `BunFetchRequestInit`, and + * the DOM lib does not declare undici's `dispatcher`, even though each runtime + * honors its respective option — the types lag the runtimes, which is why the + * interface below is declared locally rather than imported. */ /** - * `RequestInit` plus Bun's idle-timeout control, which the DOM lib does not - * declare. `false` disarms the timer; `true` or omitted keeps the default. + * `RequestInit` plus each runtime's transport-timer control, which the DOM lib + * does not declare. Bun reads `timeout` (`false` disarms its idle timer) and + * ignores `dispatcher`; Node's undici fetch reads `dispatcher` and ignores + * `timeout`. */ export interface DeadlineRequestInit extends RequestInit { timeout?: number | boolean + dispatcher?: Dispatcher +} + +let callerOwnedDeadlineDispatcher: Dispatcher | undefined + +/** + * The shared dispatcher whose header/body timers are disarmed, for runtimes + * whose fetch is undici. Constructed lazily so Bun — where `dispatcher` is + * ignored and `timeout: false` does the disarming — never pays for it, and + * shared so repeated internal-route calls reuse its keep-alive connections. + */ +function getCallerOwnedDeadlineDispatcher(): Dispatcher | undefined { + if (typeof process !== 'undefined' && process.versions?.bun) { + return undefined + } + callerOwnedDeadlineDispatcher ??= new Agent({ headersTimeout: 0, bodyTimeout: 0 }) + return callerOwnedDeadlineDispatcher } /** - * Disarms the transport idle timer so the caller's own deadline is the only one + * Disarms the transport timers so the caller's own deadline is the only one * in force. * * Only use this where the caller genuinely enforces a deadline in-process — @@ -57,7 +98,8 @@ export interface DeadlineRequestInit extends RequestInit { * request to a peer that never answers would hang until the socket dies. */ export function withCallerOwnedDeadline(init: RequestInit): DeadlineRequestInit { - return { ...init, timeout: false } + const dispatcher = getCallerOwnedDeadlineDispatcher() + return { ...init, timeout: false, ...(dispatcher ? { dispatcher } : {}) } } /** @@ -66,9 +108,12 @@ export function withCallerOwnedDeadline(init: RequestInit): DeadlineRequestInit * * Bun reports both an unanswered request and a truncated body as * `TimeoutError: The operation timed out.`, and surfaces a severed connection - * as a bare `fetch failed` — none of which name the hop, the elapsed time, or - * the fact that a cap was hit. Callers use this to annotate before rethrowing - * so a transport cap cannot masquerade as a failure of the work itself. + * as a bare `fetch failed`. Node's undici reports its expired header/body + * timers and severed connections alike as `TypeError: fetch failed`, with the + * distinguishing `HeadersTimeoutError`/`BodyTimeoutError` only on `cause` — + * none of which name the hop, the elapsed time, or the fact that a cap was + * hit. Callers use this to annotate before rethrowing so a transport cap + * cannot masquerade as a failure of the work itself. */ export function isTransportTimeoutError(error: unknown): error is Error { if (!(error instanceof Error)) return false diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index c210e589d68..edaedd1ea80 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -2599,8 +2599,9 @@ async function executeToolRequest( try { /* * `controller` above is armed with `timeout`, so the plan deadline is - * already enforced in-process; the transport timer is disarmed so its - * 300s default cannot undercut it. + * already enforced in-process; the transport timers (Bun's idle timer, + * undici's header/body timers in the Node workers) are disarmed so + * their 300s defaults cannot undercut it. */ const internalResponse = await fetch( fullUrl,