From 4fdd3e913e0faefd7180a0710b9497cfdcf4225f Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Sat, 22 Aug 2026 13:13:47 -0700 Subject: [PATCH 1/3] fix(uploads): require an explicit byte ceiling on workspace-file downloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workspace files are admitted at 5 GB because they stream straight to object storage, but a tool that pulls one back to hand it to a third party buffers the whole thing in the shared app process. maxBytes was optional on every download helper, so 51 call sites had silently inherited "unbounded". Make maxBytes required on all five entry points so a new call site cannot inherit it again, and give each existing site a ceiling: the destination's own documented limit where the route already declared one, otherwise the 100 MB this codebase already uses for buffered work. Multi-attachment routes were the worse case — Gmail, Outlook, SendGrid and SMTP downloaded every attachment via Promise.all and only summed the sizes once they were all resident, so their pre-check on declared sizes protected nothing. Add downloadServableFilesWithinBudget, which walks the list against a shrinking budget, and use the same running budget in Slack, Jira, Discord and Quiver. --- .../sim/app/api/tools/agiloft/attach/route.ts | 11 +- apps/sim/app/api/tools/box/upload/route.ts | 8 +- .../tools/brex/upload-receipt/route.test.ts | 23 +++- .../api/tools/brex/upload-receipt/route.ts | 17 +-- .../confluence/upload-attachment/route.ts | 13 +- .../sim/app/api/tools/daytona/upload/route.ts | 11 +- .../api/tools/discord/send-message/route.ts | 16 +-- .../sim/app/api/tools/dropbox/upload/route.ts | 8 +- .../app/api/tools/elevenlabs/audio/route.ts | 5 +- .../app/api/tools/firecrawl/parse/route.ts | 6 +- apps/sim/app/api/tools/gmail/draft/route.ts | 37 +++--- .../app/api/tools/gmail/edit-draft/route.ts | 37 +++--- apps/sim/app/api/tools/gmail/send/route.ts | 37 +++--- .../api/tools/google_drive/upload/route.ts | 8 +- .../api/tools/jira/add-attachment/route.ts | 9 +- .../sim/app/api/tools/jupyter/upload/route.ts | 8 +- apps/sim/app/api/tools/linq/upload/route.ts | 25 ++-- .../microsoft-dataverse/upload-file/route.ts | 26 ++-- apps/sim/app/api/tools/mistral/parse/route.ts | 6 +- .../app/api/tools/onedrive/upload/route.ts | 36 ++++-- apps/sim/app/api/tools/outlook/draft/route.ts | 37 +++--- apps/sim/app/api/tools/outlook/send/route.ts | 37 +++--- .../tools/persona/import-accounts/route.ts | 8 +- .../api/tools/quiver/image-to-svg/route.ts | 9 +- .../app/api/tools/quiver/text-to-svg/route.ts | 14 ++- apps/sim/app/api/tools/s3/put-object/route.ts | 8 +- .../app/api/tools/sendgrid/send-mail/route.ts | 44 +++---- .../servicenow/upload-attachment/route.ts | 8 +- apps/sim/app/api/tools/sftp/upload/route.ts | 4 +- .../app/api/tools/sharepoint/upload/route.ts | 42 ++++--- apps/sim/app/api/tools/slack/utils.ts | 8 +- apps/sim/app/api/tools/smtp/send/route.ts | 37 +++--- .../api/tools/square/catalog-image/route.ts | 5 +- apps/sim/app/api/tools/stt/route.ts | 8 +- .../tools/supabase/storage-upload/route.ts | 8 +- .../api/tools/telegram/send-document/route.ts | 15 ++- apps/sim/app/api/tools/textract/shared.ts | 6 +- .../app/api/tools/uptimerobot/server-utils.ts | 10 +- apps/sim/app/api/tools/vanta/upload/route.ts | 8 +- .../sim/app/api/tools/vision/analyze/route.ts | 10 +- .../app/api/tools/wordpress/upload/route.ts | 8 +- apps/sim/lib/collab-doc/persist.ts | 11 +- .../tools/handlers/materialize-file.ts | 7 +- .../workspace/workspace-file-manager.ts | 4 +- apps/sim/lib/uploads/shared/types.ts | 19 +++ .../uploads/utils/file-utils.server.test.ts | 117 +++++++++++++++++- .../lib/uploads/utils/file-utils.server.ts | 103 ++++++++++++--- .../fetch-servable-workspace-file-buffer.ts | 2 +- ...read-workspace-file-content-by-key.test.ts | 5 +- .../read-workspace-file-content-by-key.ts | 6 +- .../read-workspace-file-content.ts | 5 +- .../sim/tools/microsoft_teams/server-utils.ts | 19 ++- 52 files changed, 693 insertions(+), 286 deletions(-) diff --git a/apps/sim/app/api/tools/agiloft/attach/route.ts b/apps/sim/app/api/tools/agiloft/attach/route.ts index 014e34eb271..a9f0ce22ac6 100644 --- a/apps/sim/app/api/tools/agiloft/attach/route.ts +++ b/apps/sim/app/api/tools/agiloft/attach/route.ts @@ -6,7 +6,9 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { secureFetchWithPinnedIP } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -75,13 +77,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let fileBuffer: Buffer try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = servable.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady logger.error(`[${requestId}] Failed to download file from storage:`, error) - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) + return NextResponse.json( + { success: false, error: toError(error).message }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) } const resolvedFileName = data.fileName || userFile.name || 'attachment' diff --git a/apps/sim/app/api/tools/box/upload/route.ts b/apps/sim/app/api/tools/box/upload/route.ts index 57ec5f4223c..f490fdc02a9 100644 --- a/apps/sim/app/api/tools/box/upload/route.ts +++ b/apps/sim/app/api/tools/box/upload/route.ts @@ -5,7 +5,9 @@ import { boxUploadContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -55,14 +57,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = result.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } fileName = validatedData.fileName || userFile.name diff --git a/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts b/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts index eae6759b6cb..1702e54cc86 100644 --- a/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts +++ b/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts @@ -27,6 +27,7 @@ vi.mock('@/app/api/files/authorization', () => ({ assertToolFileAccess: mockAssertToolFileAccess, })) +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { POST } from '@/app/api/tools/brex/upload-receipt/route' const mockFetch = vi.fn() @@ -194,11 +195,25 @@ describe('POST /api/tools/brex/upload-receipt', () => { expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() }) + it('asks the downloader for at most 50 MB', async () => { + await POST(createMockRequest('POST', baseBody)) + + expect(mockDownloadFileFromStorage).toHaveBeenCalledWith( + expect.anything(), + expect.any(String), + expect.anything(), + { maxBytes: 50 * 1024 * 1024 } + ) + }) + it('rejects files over the 50 MB limit', async () => { - mockDownloadFileFromStorage.mockResolvedValueOnce({ - buffer: Buffer.alloc(50 * 1024 * 1024 + 1), - contentType: 'application/pdf', - }) + mockDownloadFileFromStorage.mockRejectedValueOnce( + new PayloadSizeLimitError({ + label: 'storage file download', + maxBytes: 50 * 1024 * 1024, + observedBytes: 50 * 1024 * 1024 + 1, + }) + ) const response = await POST(createMockRequest('POST', baseBody)) expect(response.status).toBe(400) diff --git a/apps/sim/app/api/tools/brex/upload-receipt/route.ts b/apps/sim/app/api/tools/brex/upload-receipt/route.ts index 6fb4ccfecf8..c63bb4374b7 100644 --- a/apps/sim/app/api/tools/brex/upload-receipt/route.ts +++ b/apps/sim/app/api/tools/brex/upload-receipt/route.ts @@ -9,6 +9,7 @@ import { validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -51,23 +52,25 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let fileBuffer: Buffer try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_RECEIPT_SIZE_BYTES, + }) fileBuffer = resolved.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + return NextResponse.json( + { success: false, error: 'Receipt file exceeds the 50 MB limit' }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download receipt file:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Unknown error') }, { status: 500 } ) } - if (fileBuffer.length > MAX_RECEIPT_SIZE_BYTES) { - return NextResponse.json( - { success: false, error: 'Receipt file exceeds the 50 MB limit' }, - { status: 400 } - ) - } const effectiveReceiptName = receiptName || userFile.name const endpoint = expenseId diff --git a/apps/sim/app/api/tools/confluence/upload-attachment/route.ts b/apps/sim/app/api/tools/confluence/upload-attachment/route.ts index 00057e4d10c..8e0b14f5d4f 100644 --- a/apps/sim/app/api/tools/confluence/upload-attachment/route.ts +++ b/apps/sim/app/api/tools/confluence/upload-attachment/route.ts @@ -5,7 +5,9 @@ import { confluenceUploadAttachmentContract } from '@/lib/api/contracts/selector import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processSingleFileToUserFile, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -94,7 +96,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let fileBuffer: Buffer let resolvedContentType: string try { - const servable = await downloadServableFileFromStorage(userFile, 'confluence-upload', logger) + const servable = await downloadServableFileFromStorage( + userFile, + 'confluence-upload', + logger, + { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + } + ) fileBuffer = servable.buffer resolvedContentType = servable.contentType } catch (error) { @@ -105,7 +114,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/daytona/upload/route.ts b/apps/sim/app/api/tools/daytona/upload/route.ts index 5c52509bb75..800649f0f4d 100644 --- a/apps/sim/app/api/tools/daytona/upload/route.ts +++ b/apps/sim/app/api/tools/daytona/upload/route.ts @@ -5,6 +5,7 @@ import { daytonaUploadFileContract } from '@/lib/api/contracts/tools/daytona' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -62,11 +63,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.info(`[${requestId}] Downloading file: ${userFile.name} (${userFile.size} bytes)`) try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_UPLOAD_SIZE_BYTES, + }) fileBuffer = servable.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + return NextResponse.json( + { success: false, error: 'File exceeds upload limit of 100MB' }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download file from storage:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, diff --git a/apps/sim/app/api/tools/discord/send-message/route.ts b/apps/sim/app/api/tools/discord/send-message/route.ts index aa644457654..bc9d30d526a 100644 --- a/apps/sim/app/api/tools/discord/send-message/route.ts +++ b/apps/sim/app/api/tools/discord/send-message/route.ts @@ -6,9 +6,11 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateNumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -146,12 +148,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - userFiles.map(async (file, i) => { - logger.info(`[${requestId}] Downloading file ${i}: ${file.name}`) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(userFiles, requestId, logger, { + totalMaxBytes: MAX_BUFFERED_TRANSFER_BYTES, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady @@ -161,7 +161,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { success: false, error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/dropbox/upload/route.ts b/apps/sim/app/api/tools/dropbox/upload/route.ts index 58873c334bc..af04b69db16 100644 --- a/apps/sim/app/api/tools/dropbox/upload/route.ts +++ b/apps/sim/app/api/tools/dropbox/upload/route.ts @@ -5,8 +5,10 @@ import { dropboxUploadContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { httpHeaderSafeJson } from '@/lib/core/utils/validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -58,14 +60,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = result.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } fileName = userFile.name diff --git a/apps/sim/app/api/tools/elevenlabs/audio/route.ts b/apps/sim/app/api/tools/elevenlabs/audio/route.ts index 969f6f4a231..871868037ad 100644 --- a/apps/sim/app/api/tools/elevenlabs/audio/route.ts +++ b/apps/sim/app/api/tools/elevenlabs/audio/route.ts @@ -20,6 +20,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -153,7 +154,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(file, requestId, logger) + const buffer = await downloadFileFromStorage(file, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) const ext = file.name.split('.').pop()?.toLowerCase() || '' source = { buffer, diff --git a/apps/sim/app/api/tools/firecrawl/parse/route.ts b/apps/sim/app/api/tools/firecrawl/parse/route.ts index 2fb4c7d1cd0..a1b1fe8888f 100644 --- a/apps/sim/app/api/tools/firecrawl/parse/route.ts +++ b/apps/sim/app/api/tools/firecrawl/parse/route.ts @@ -11,6 +11,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -85,7 +86,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const { buffer, contentType } = await downloadServableFileFromStorage( userFile, requestId, - logger + logger, + { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + } ) const formData = new FormData() diff --git a/apps/sim/app/api/tools/gmail/draft/route.ts b/apps/sim/app/api/tools/gmail/draft/route.ts index e376be1f01e..255c5242967 100644 --- a/apps/sim/app/api/tools/gmail/draft/route.ts +++ b/apps/sim/app/api/tools/gmail/draft/route.ts @@ -5,9 +5,10 @@ import { gmailDraftContract } from '@/lib/api/contracts/tools/google' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { @@ -97,17 +98,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -118,18 +125,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, mimeType: resolved[i].contentType || file.type || 'application/octet-stream', diff --git a/apps/sim/app/api/tools/gmail/edit-draft/route.ts b/apps/sim/app/api/tools/gmail/edit-draft/route.ts index f986a610f11..9e88ce6cdbd 100644 --- a/apps/sim/app/api/tools/gmail/edit-draft/route.ts +++ b/apps/sim/app/api/tools/gmail/edit-draft/route.ts @@ -5,9 +5,10 @@ import { gmailEditDraftContract } from '@/lib/api/contracts/tools/google' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { @@ -93,17 +94,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -114,18 +121,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, mimeType: resolved[i].contentType || file.type || 'application/octet-stream', diff --git a/apps/sim/app/api/tools/gmail/send/route.ts b/apps/sim/app/api/tools/gmail/send/route.ts index 59df7377b34..62b10377e88 100644 --- a/apps/sim/app/api/tools/gmail/send/route.ts +++ b/apps/sim/app/api/tools/gmail/send/route.ts @@ -5,9 +5,10 @@ import { gmailSendContract } from '@/lib/api/contracts/tools/google' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { @@ -97,17 +98,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -118,18 +125,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, mimeType: resolved[i].contentType || file.type || 'application/octet-stream', diff --git a/apps/sim/app/api/tools/google_drive/upload/route.ts b/apps/sim/app/api/tools/google_drive/upload/route.ts index 0600386324e..ca134da06cc 100644 --- a/apps/sim/app/api/tools/google_drive/upload/route.ts +++ b/apps/sim/app/api/tools/google_drive/upload/route.ts @@ -6,7 +6,9 @@ import { googleDriveUploadContract } from '@/lib/api/contracts/tools/google' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -124,7 +126,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let downloadedContentType = '' try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = result.buffer downloadedContentType = result.contentType } catch (error) { @@ -136,7 +140,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { success: false, error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/jira/add-attachment/route.ts b/apps/sim/app/api/tools/jira/add-attachment/route.ts index b23abf6656b..70218256ffc 100644 --- a/apps/sim/app/api/tools/jira/add-attachment/route.ts +++ b/apps/sim/app/api/tools/jira/add-attachment/route.ts @@ -5,6 +5,7 @@ import { jiraAddAttachmentContract } from '@/lib/api/contracts/selectors/jira' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -44,6 +45,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { (await getJiraCloudId(validatedData.domain, validatedData.accessToken)) const formData = new FormData() + // Every attachment lands in the same multipart body, so the ceiling covers the + // set rather than each file on its own. + let remainingBytes = MAX_BUFFERED_TRANSFER_BYTES for (const file of userFiles) { const denied = await assertToolFileAccess(file.key, authResult.userId, requestId, logger) @@ -51,7 +55,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let buffer: Buffer let downloadedContentType = '' try { - const result = await downloadServableFileFromStorage(file, requestId, logger) + const result = await downloadServableFileFromStorage(file, requestId, logger, { + maxBytes: remainingBytes, + }) buffer = result.buffer downloadedContentType = result.contentType } catch (error) { @@ -59,6 +65,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (notReady) return notReady throw error } + remainingBytes -= buffer.length const blob = new Blob([new Uint8Array(buffer)], { type: downloadedContentType || file.type || 'application/octet-stream', }) diff --git a/apps/sim/app/api/tools/jupyter/upload/route.ts b/apps/sim/app/api/tools/jupyter/upload/route.ts index ff3656cce44..0c4efff228a 100644 --- a/apps/sim/app/api/tools/jupyter/upload/route.ts +++ b/apps/sim/app/api/tools/jupyter/upload/route.ts @@ -10,7 +10,9 @@ import { validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -58,14 +60,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = result.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } fileName = data.fileName || userFile.name diff --git a/apps/sim/app/api/tools/linq/upload/route.ts b/apps/sim/app/api/tools/linq/upload/route.ts index 6b0004df4e2..deb8ffb79d6 100644 --- a/apps/sim/app/api/tools/linq/upload/route.ts +++ b/apps/sim/app/api/tools/linq/upload/route.ts @@ -5,6 +5,7 @@ import { linqUploadAttachmentContract } from '@/lib/api/contracts/tools/communic import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -19,6 +20,16 @@ const logger = createLogger('LinqUploadAttachmentAPI') /** Linq pre-upload caps attachments at 100MB. */ const MAX_SIZE_BYTES = 100 * 1024 * 1024 +function fileTooLargeError(sizeBytes: number): NextResponse { + return NextResponse.json( + { + success: false, + error: `File exceeds Linq's 100MB attachment limit (${(sizeBytes / (1024 * 1024)).toFixed(2)}MB)`, + }, + { status: 400 } + ) +} + /** * Upload a file to Linq as a reusable attachment. * @@ -61,12 +72,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied let resolvedContentTypeFromStorage: string try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_SIZE_BYTES, + }) buffer = resolved.buffer resolvedContentTypeFromStorage = resolved.contentType } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) + return fileTooLargeError(error.observedBytes ?? userFile.size) logger.error(`[${requestId}] Failed to download Linq attachment file:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, @@ -93,13 +108,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ success: false, error: 'File is empty' }, { status: 400 }) } if (sizeBytes > MAX_SIZE_BYTES) { - return NextResponse.json( - { - success: false, - error: `File exceeds Linq's 100MB attachment limit (${(sizeBytes / (1024 * 1024)).toFixed(2)}MB)`, - }, - { status: 400 } - ) + return fileTooLargeError(sizeBytes) } logger.info(`[${requestId}] Registering Linq attachment`, { diff --git a/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts b/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts index eb78c96325f..097c94a1e7a 100644 --- a/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts +++ b/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts @@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -20,6 +21,18 @@ const logger = createLogger('DataverseUploadFileAPI') /** Dataverse Web API's absolute ceiling for a single-request (non-chunked) file column upload. */ const DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES = 128 * 1024 * 1024 +function uploadTooLargeError(observedBytes?: number): NextResponse { + const sizeLabel = + observedBytes === undefined ? '' : `(${(observedBytes / (1024 * 1024)).toFixed(2)}MB) ` + return NextResponse.json( + { + success: false, + error: `File size ${sizeLabel}exceeds Dataverse's 128MB limit for single-request file column uploads. Split the file and use chunked upload instead.`, + }, + { status: 400 } + ) +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() @@ -77,11 +90,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES, + }) fileBuffer = servable.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) return uploadTooLargeError(error.observedBytes) logger.error(`[${requestId}] Failed to download file from storage:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, @@ -100,13 +116,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (fileBuffer.length > DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES) { const sizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2) logger.warn(`[${requestId}] File too large for single-request upload: ${sizeMB}MB`) - return NextResponse.json( - { - success: false, - error: `File size (${sizeMB}MB) exceeds Dataverse's 128MB limit for single-request file column uploads. Split the file and use chunked upload instead.`, - }, - { status: 400 } - ) + return uploadTooLargeError(fileBuffer.length) } const baseUrl = getDataverseBaseUrl(validatedData.environmentUrl) diff --git a/apps/sim/app/api/tools/mistral/parse/route.ts b/apps/sim/app/api/tools/mistral/parse/route.ts index 5805142915d..e994e718ac3 100644 --- a/apps/sim/app/api/tools/mistral/parse/route.ts +++ b/apps/sim/app/api/tools/mistral/parse/route.ts @@ -15,6 +15,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { extractStorageKey, isInternalFileUrl, @@ -157,7 +158,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const { buffer, contentType } = await downloadServableFileFromStorage( userFile, requestId, - logger + logger, + { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + } ) base64 = buffer.toString('base64') if (contentType && contentType !== 'application/octet-stream') { diff --git a/apps/sim/app/api/tools/onedrive/upload/route.ts b/apps/sim/app/api/tools/onedrive/upload/route.ts index 2c94986de9f..2e7ff49837d 100644 --- a/apps/sim/app/api/tools/onedrive/upload/route.ts +++ b/apps/sim/app/api/tools/onedrive/upload/route.ts @@ -8,6 +8,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getExtensionFromMimeType, @@ -24,6 +25,20 @@ const logger = createLogger('OneDriveUploadAPI') const MICROSOFT_GRAPH_BASE = 'https://graph.microsoft.com/v1.0' +/** Microsoft Graph's ceiling for a simple (non-chunked) drive-item upload. */ +const MAX_SIMPLE_UPLOAD_BYTES = 250 * 1024 * 1024 + +function fileTooLargeError(observedBytes: number): NextResponse { + const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `File size (${sizeMB}MB) exceeds OneDrive's limit of 250MB for simple uploads. Use chunked upload for larger files.`, + }, + { status: 400 } + ) +} + /** Microsoft Graph DriveItem response */ interface OneDriveFileData { id: string @@ -115,12 +130,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_SIMPLE_UPLOAD_BYTES, + }) fileBuffer = result.buffer mimeType = result.contentType || userFile.type || 'application/octet-stream' } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + return fileTooLargeError(error.observedBytes ?? userFile.size) + } logger.error(`[${requestId}] Failed to download file from storage:`, error) return NextResponse.json( { @@ -132,17 +152,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } } - const maxSize = 250 * 1024 * 1024 - if (fileBuffer.length > maxSize) { - const sizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2) - logger.warn(`[${requestId}] File too large: ${sizeMB}MB`) - return NextResponse.json( - { - success: false, - error: `File size (${sizeMB}MB) exceeds OneDrive's limit of 250MB for simple uploads. Use chunked upload for larger files.`, - }, - { status: 400 } + if (fileBuffer.length > MAX_SIMPLE_UPLOAD_BYTES) { + logger.warn( + `[${requestId}] File too large: ${(fileBuffer.length / (1024 * 1024)).toFixed(2)}MB` ) + return fileTooLargeError(fileBuffer.length) } let fileName = validatedData.fileName diff --git a/apps/sim/app/api/tools/outlook/draft/route.ts b/apps/sim/app/api/tools/outlook/draft/route.ts index 979634ffcf3..52b114bed2a 100644 --- a/apps/sim/app/api/tools/outlook/draft/route.ts +++ b/apps/sim/app/api/tools/outlook/draft/route.ts @@ -5,9 +5,10 @@ import { outlookDraftContract } from '@/lib/api/contracts/tools/microsoft' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -110,17 +111,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -131,18 +138,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request`, - }, - { status: 400 } - ) - } - const attachmentObjects = attachments.map((file, i) => ({ '@odata.type': '#microsoft.graph.fileAttachment', name: file.name, diff --git a/apps/sim/app/api/tools/outlook/send/route.ts b/apps/sim/app/api/tools/outlook/send/route.ts index 7a59f44410a..82dd3e4474a 100644 --- a/apps/sim/app/api/tools/outlook/send/route.ts +++ b/apps/sim/app/api/tools/outlook/send/route.ts @@ -5,9 +5,10 @@ import { outlookSendContract } from '@/lib/api/contracts/tools/microsoft' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -110,17 +111,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -131,18 +138,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request`, - }, - { status: 400 } - ) - } - const attachmentObjects = attachments.map((file, i) => ({ '@odata.type': '#microsoft.graph.fileAttachment', name: file.name, diff --git a/apps/sim/app/api/tools/persona/import-accounts/route.ts b/apps/sim/app/api/tools/persona/import-accounts/route.ts index 0844b11c16a..ca59a41b053 100644 --- a/apps/sim/app/api/tools/persona/import-accounts/route.ts +++ b/apps/sim/app/api/tools/persona/import-accounts/route.ts @@ -5,7 +5,9 @@ import { personaImportAccountsContract } from '@/lib/api/contracts/tools/persona import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -58,7 +60,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let buffer: Buffer try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) buffer = resolved.buffer } catch (error) { const notReady = docNotReadyResponse(error) @@ -66,7 +70,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.error(`[${requestId}] Failed to download Persona import file:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/quiver/image-to-svg/route.ts b/apps/sim/app/api/tools/quiver/image-to-svg/route.ts index 8149a8b5dd0..faf79554834 100644 --- a/apps/sim/app/api/tools/quiver/image-to-svg/route.ts +++ b/apps/sim/app/api/tools/quiver/image-to-svg/route.ts @@ -11,6 +11,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -81,7 +82,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger) + const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) apiImage = { base64: buffer.toString('base64') } } else { return NextResponse.json( @@ -114,7 +117,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger) + const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) apiImage = { base64: buffer.toString('base64') } } else { return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) diff --git a/apps/sim/app/api/tools/quiver/text-to-svg/route.ts b/apps/sim/app/api/tools/quiver/text-to-svg/route.ts index fb6df66b7e5..eb40c76997d 100644 --- a/apps/sim/app/api/tools/quiver/text-to-svg/route.ts +++ b/apps/sim/app/api/tools/quiver/text-to-svg/route.ts @@ -11,6 +11,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -59,6 +60,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const apiReferences: Array<{ url: string } | { base64: string }> = [] + // Every reference is buffered and base64'd before the list is sliced to 4, so the + // budget has to span the whole loop rather than bound each file on its own. + let referenceBudget = MAX_BUFFERED_TRANSFER_BYTES if (data.references) { const rawRefs = Array.isArray(data.references) ? data.references : [data.references] @@ -86,7 +90,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger) + const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { + maxBytes: referenceBudget, + }) + referenceBudget -= buffer.length apiReferences.push({ base64: buffer.toString('base64') }) } } @@ -107,7 +114,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger) + const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { + maxBytes: referenceBudget, + }) + referenceBudget -= buffer.length apiReferences.push({ base64: buffer.toString('base64') }) } } diff --git a/apps/sim/app/api/tools/s3/put-object/route.ts b/apps/sim/app/api/tools/s3/put-object/route.ts index 2019de2b4d4..026713914fb 100644 --- a/apps/sim/app/api/tools/s3/put-object/route.ts +++ b/apps/sim/app/api/tools/s3/put-object/route.ts @@ -6,7 +6,9 @@ import { awsS3PutObjectContract } from '@/lib/api/contracts/tools/aws/s3-put-obj import { parseToolRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -84,7 +86,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let downloadedContentType = '' try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) uploadBody = result.buffer downloadedContentType = result.contentType } catch (error) { @@ -92,7 +96,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (notReady) return notReady return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/sendgrid/send-mail/route.ts b/apps/sim/app/api/tools/sendgrid/send-mail/route.ts index d9b2b97876a..f0bc1b3e660 100644 --- a/apps/sim/app/api/tools/sendgrid/send-mail/route.ts +++ b/apps/sim/app/api/tools/sendgrid/send-mail/route.ts @@ -5,9 +5,10 @@ import { sendGridSendMailContract } from '@/lib/api/contracts/tools/communicatio import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -15,6 +16,9 @@ export const dynamic = 'force-dynamic' const logger = createLogger('SendGridSendMailAPI') +/** SendGrid rejects a message whose total attachment payload exceeds 30MB. */ +const MAX_ATTACHMENT_TOTAL_BYTES = 30 * 1024 * 1024 + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() @@ -109,17 +113,26 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - userFiles.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(userFiles, requestId, logger, { + totalMaxBytes: MAX_ATTACHMENT_TOTAL_BYTES, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ( + (error.observedBytes ?? MAX_ATTACHMENT_TOTAL_BYTES) / + (1024 * 1024) + ).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds SendGrid's limit of 30MB`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -130,19 +143,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - const maxSize = 30 * 1024 * 1024 - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds SendGrid's limit of 30MB`, - }, - { status: 400 } - ) - } - const sendGridAttachments = userFiles.map((file, i) => ({ content: resolved[i].buffer.toString('base64'), filename: file.name, diff --git a/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts b/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts index 78dcdb0883b..532b3712572 100644 --- a/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts +++ b/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts @@ -6,7 +6,9 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -56,7 +58,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let fileBuffer: Buffer let resolvedContentType: string try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = servable.buffer resolvedContentType = servable.contentType } catch (error) { @@ -65,7 +69,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.error(`[${requestId}] Failed to download file from storage:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/sftp/upload/route.ts b/apps/sim/app/api/tools/sftp/upload/route.ts index 09029d5cd81..cd842fedbb8 100644 --- a/apps/sim/app/api/tools/sftp/upload/route.ts +++ b/apps/sim/app/api/tools/sftp/upload/route.ts @@ -109,7 +109,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.info( `[${requestId}] Downloading file for upload: ${file.name} (${file.size} bytes)` ) - const { buffer } = await downloadServableFileFromStorage(file, requestId, logger) + const { buffer } = await downloadServableFileFromStorage(file, requestId, logger, { + maxBytes: maxSize - resolvedTotal, + }) resolvedTotal += buffer.length if (resolvedTotal > maxSize) { diff --git a/apps/sim/app/api/tools/sharepoint/upload/route.ts b/apps/sim/app/api/tools/sharepoint/upload/route.ts index 55bb4eec935..b29a51fd387 100644 --- a/apps/sim/app/api/tools/sharepoint/upload/route.ts +++ b/apps/sim/app/api/tools/sharepoint/upload/route.ts @@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -88,34 +89,39 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied logger.info(`[${requestId}] Uploading file: ${userFile.name}`) - let buffer: Buffer - let downloadedContentType = '' - try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) - buffer = result.buffer - downloadedContentType = result.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - throw error - } - const fileName = validatedData.fileName || userFile.name const folderPath = validatedData.folderPath?.trim() || '' - const fileSizeMB = buffer.length / (1024 * 1024) - - if (buffer.length > MAX_SHAREPOINT_UPLOAD_BYTES) { + const skipOversized = (size: number) => { logger.warn( - `[${requestId}] File ${fileName} is ${fileSizeMB.toFixed(2)}MB, exceeds 250MB limit` + `[${requestId}] File ${fileName} is ${(size / (1024 * 1024)).toFixed(2)}MB, exceeds 250MB limit` ) skippedFiles.push({ name: fileName, - size: buffer.length, + size, limit: MAX_SHAREPOINT_UPLOAD_BYTES, reason: 'File exceeds the 250 MB Microsoft Graph small upload limit', }) - continue + } + + let buffer: Buffer + let downloadedContentType = '' + try { + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_SHAREPOINT_UPLOAD_BYTES, + }) + buffer = result.buffer + downloadedContentType = result.contentType + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + // An oversized file is skipped rather than failing the whole batch, exactly as + // it was when the size was only discovered after the download. + if (isPayloadSizeLimitError(error)) { + skipOversized(error.observedBytes ?? userFile.size) + continue + } + throw error } let uploadPath = '' diff --git a/apps/sim/app/api/tools/slack/utils.ts b/apps/sim/app/api/tools/slack/utils.ts index b3ff2205806..d40b3a81a66 100644 --- a/apps/sim/app/api/tools/slack/utils.ts +++ b/apps/sim/app/api/tools/slack/utils.ts @@ -1,5 +1,6 @@ import type { Logger } from '@sim/logger' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { FileAccessDeniedError, verifyFileAccess } from '@/app/api/files/authorization' @@ -80,6 +81,9 @@ async function uploadFilesToSlack( const userFiles = processFilesToUserFiles(files, requestId, logger) const uploadedFileIds: string[] = [] const uploadedFiles: ToolFileData[] = [] + // One share can carry several files, so the ceiling spans the set: each file may + // only use what its predecessors left. + let remainingBytes = MAX_BUFFERED_TRANSFER_BYTES for (const userFile of userFiles) { logger.info(`[${requestId}] Uploading file: ${userFile.name}`) @@ -92,8 +96,10 @@ async function uploadFilesToSlack( const { buffer, contentType } = await downloadServableFileFromStorage( userFile, requestId, - logger + logger, + { maxBytes: remainingBytes } ) + remainingBytes -= buffer.length const getUrlResponse = await fetch('https://slack.com/api/files.getUploadURLExternal', { method: 'POST', diff --git a/apps/sim/app/api/tools/smtp/send/route.ts b/apps/sim/app/api/tools/smtp/send/route.ts index 920f6925784..83a48f3ce2b 100644 --- a/apps/sim/app/api/tools/smtp/send/route.ts +++ b/apps/sim/app/api/tools/smtp/send/route.ts @@ -7,10 +7,11 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getSmtpEhloName } from '@/lib/messaging/email/ehlo' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -132,17 +133,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds SMTP limit of 25MB`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -153,18 +160,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds SMTP limit of 25MB`, - }, - { status: 400 } - ) - } - const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, content: resolved[i].buffer, diff --git a/apps/sim/app/api/tools/square/catalog-image/route.ts b/apps/sim/app/api/tools/square/catalog-image/route.ts index 622c5c1c7a2..229e70deff3 100644 --- a/apps/sim/app/api/tools/square/catalog-image/route.ts +++ b/apps/sim/app/api/tools/square/catalog-image/route.ts @@ -7,6 +7,7 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -52,7 +53,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) const fileName = validatedData.fileName || userFile.name const mimeType = userFile.type || 'application/octet-stream' diff --git a/apps/sim/app/api/tools/stt/route.ts b/apps/sim/app/api/tools/stt/route.ts index f8f468ea11f..3ff3cb3eaaa 100644 --- a/apps/sim/app/api/tools/stt/route.ts +++ b/apps/sim/app/api/tools/stt/route.ts @@ -117,7 +117,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - audioBuffer = await downloadFileFromStorage(file, requestId, logger) + audioBuffer = await downloadFileFromStorage(file, requestId, logger, { + maxBytes: MAX_FILE_SIZE, + }) audioFileName = file.name // file.type may be missing if the file came from a block that doesn't preserve it // Infer from filename extension as fallback @@ -143,7 +145,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - audioBuffer = await downloadFileFromStorage(file, requestId, logger) + audioBuffer = await downloadFileFromStorage(file, requestId, logger, { + maxBytes: MAX_FILE_SIZE, + }) audioFileName = file.name const ext = file.name.split('.').pop()?.toLowerCase() || '' diff --git a/apps/sim/app/api/tools/supabase/storage-upload/route.ts b/apps/sim/app/api/tools/supabase/storage-upload/route.ts index 7f02f7791ea..ac9d6bda605 100644 --- a/apps/sim/app/api/tools/supabase/storage-upload/route.ts +++ b/apps/sim/app/api/tools/supabase/storage-upload/route.ts @@ -6,7 +6,9 @@ import { parseToolRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateSupabaseProjectId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -152,7 +154,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let buffer: Buffer let resolvedContentType: string try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) buffer = resolved.buffer resolvedContentType = resolved.contentType } catch (error) { @@ -161,7 +165,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.error(`[${requestId}] Failed to download file for Supabase upload:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/telegram/send-document/route.ts b/apps/sim/app/api/tools/telegram/send-document/route.ts index f454717e290..6d4cf533d64 100644 --- a/apps/sim/app/api/tools/telegram/send-document/route.ts +++ b/apps/sim/app/api/tools/telegram/send-document/route.ts @@ -5,6 +5,7 @@ import { telegramSendDocumentContract } from '@/lib/api/contracts/tools/communic import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -97,12 +98,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let buffer: Buffer let contentType: string try { - const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger) + const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: maxSize, + }) buffer = downloaded.buffer contentType = downloaded.contentType } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? userFile.size) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `The following files exceed Telegram's 50MB limit: ${userFile.name} (${sizeMB}MB)`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download document ${userFile.name}:`, error) return NextResponse.json( { diff --git a/apps/sim/app/api/tools/textract/shared.ts b/apps/sim/app/api/tools/textract/shared.ts index b6b570cb506..0150c8f0c99 100644 --- a/apps/sim/app/api/tools/textract/shared.ts +++ b/apps/sim/app/api/tools/textract/shared.ts @@ -12,6 +12,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import type { RawFileInput } from '@/lib/uploads/utils/file-utils' import { extractStorageKey, @@ -160,7 +161,10 @@ export async function resolveDocumentInput( const { buffer, contentType } = await downloadServableFileFromStorage( userFile, requestId, - logger + logger, + { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + } ) const resolvedContentType = contentType || userFile.type || 'application/octet-stream' diff --git a/apps/sim/app/api/tools/uptimerobot/server-utils.ts b/apps/sim/app/api/tools/uptimerobot/server-utils.ts index d7a1ef9ea8c..9c3c7d2f316 100644 --- a/apps/sim/app/api/tools/uptimerobot/server-utils.ts +++ b/apps/sim/app/api/tools/uptimerobot/server-utils.ts @@ -1,5 +1,6 @@ import type { Logger } from '@sim/logger' import { NextResponse } from 'next/server' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -47,7 +48,14 @@ async function appendPspImage( const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) if (denied) return denied - const { buffer, contentType } = await downloadServableFileFromStorage(userFile, requestId, logger) + const { buffer, contentType } = await downloadServableFileFromStorage( + userFile, + requestId, + logger, + { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + } + ) const mimeType = contentType || userFile.type || 'application/octet-stream' form.append(field, new Blob([new Uint8Array(buffer)], { type: mimeType }), userFile.name) return null diff --git a/apps/sim/app/api/tools/vanta/upload/route.ts b/apps/sim/app/api/tools/vanta/upload/route.ts index 81785b515c8..932b68658c7 100644 --- a/apps/sim/app/api/tools/vanta/upload/route.ts +++ b/apps/sim/app/api/tools/vanta/upload/route.ts @@ -5,6 +5,7 @@ import { vantaUploadContract } from '@/lib/api/contracts/tools/vanta' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -72,7 +73,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_UPLOAD_SIZE_BYTES, + }) fileBuffer = resolved.buffer fileName = params.fileName || userFile.name mimeType = @@ -80,6 +83,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + return uploadSizeError(error.observedBytes ?? userFile.size) + } logger.error(`[${requestId}] Failed to download Vanta upload file`, { error: getErrorMessage(error), }) diff --git a/apps/sim/app/api/tools/vision/analyze/route.ts b/apps/sim/app/api/tools/vision/analyze/route.ts index fb74630943f..74a0cb0560f 100644 --- a/apps/sim/app/api/tools/vision/analyze/route.ts +++ b/apps/sim/app/api/tools/vision/analyze/route.ts @@ -16,6 +16,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { extractStorageKey, isInternalFileUrl, @@ -125,7 +126,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(userFile, requestId, logger) + // The three providers this route serves disagree too much for a single + // route-wide image limit to be right (Anthropic: 10MB base64 per image; + // OpenAI: 512MB total request payload), and picking the lowest would reject + // images the others accept. So bound the buffer we hold and let each provider + // reject what it will not take, with its own message. + const buffer = await downloadFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) base64 = buffer.toString('base64') bufferLength = buffer.length } diff --git a/apps/sim/app/api/tools/wordpress/upload/route.ts b/apps/sim/app/api/tools/wordpress/upload/route.ts index aac9e9e3354..b7b9ba8c2a1 100644 --- a/apps/sim/app/api/tools/wordpress/upload/route.ts +++ b/apps/sim/app/api/tools/wordpress/upload/route.ts @@ -5,7 +5,9 @@ import { wordpressUploadContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { getFileExtension, getMimeTypeFromExtension, @@ -93,7 +95,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolvedContentType: string try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = servable.buffer resolvedContentType = servable.contentType } catch (error) { @@ -105,7 +109,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { success: false, error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/lib/collab-doc/persist.ts b/apps/sim/lib/collab-doc/persist.ts index 2e8e9692ff4..6d8f387ce6d 100644 --- a/apps/sim/lib/collab-doc/persist.ts +++ b/apps/sim/lib/collab-doc/persist.ts @@ -7,6 +7,7 @@ import { getWorkspaceFile, updateWorkspaceFileContent, } from '@/lib/uploads/contexts/workspace' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { collabDocStateSourceHash, hashMarkdown, saveCollabDocState } from './collab-state' import { canonicalizeYDoc, yDocToFileMarkdown } from './converter' @@ -94,7 +95,11 @@ export async function persistFileDoc( // nothing to clobber, so this reports the file's CURRENT durable version rather than conflicting on a // stale `expectedVersion`: it resynchronizes the relay's If-Match token instead of stranding it. if (record.size === markdownBuffer.length) { - const current = await fetchWorkspaceFileBuffer(record).catch(() => null) + // A byte-for-byte equality check: anything longer than what we are comparing against + // cannot match, so the buffer we are about to compare is itself the ceiling. + const current = await fetchWorkspaceFileBuffer(record, { + maxBytes: markdownBuffer.length, + }).catch(() => null) if (current?.equals(markdownBuffer)) { // Still refresh the cached snapshot: the markdown is unchanged (so its `sourceHash` tag stays // valid) but the doc state may have just been canonicalized, and a cold open should seed from @@ -191,7 +196,9 @@ async function recoverFromVersionConflict( try { const current = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) if (!current) return { status: 'missing' } - const durable = await fetchWorkspaceFileBuffer(current) + const durable = await fetchWorkspaceFileBuffer(current, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) if (hashMarkdown(durable) !== (await collabDocStateSourceHash(fileId))) return conflict() logger.info( `Persist token for file ${fileId} was stale, not the file; re-syncing and writing the projection` diff --git a/apps/sim/lib/copilot/tools/handlers/materialize-file.ts b/apps/sim/lib/copilot/tools/handlers/materialize-file.ts index 26f87478e51..8c13a9bbef8 100644 --- a/apps/sim/lib/copilot/tools/handlers/materialize-file.ts +++ b/apps/sim/lib/copilot/tools/handlers/materialize-file.ts @@ -39,6 +39,7 @@ import { hasCloudStorage, headObject } from '@/lib/uploads/core/storage-service' import { toLegacyWorkspaceFileSize } from '@/lib/uploads/shared/types' import { isArchiveFileName } from '@/lib/uploads/utils/file-utils' import { parseWorkflowJson } from '@/lib/workflows/operations/import-export' +import { MAX_IMPORT_BODY_BYTES } from '@/lib/workflows/operations/import-workflow' import { saveWorkflowToNormalizedTables } from '@/lib/workflows/persistence/utils' import { deduplicateWorkflowName } from '@/lib/workflows/utils' import { admitCreateWorkspaceFile } from '@/lib/workspace-files/application/create-workspace-file' @@ -260,7 +261,11 @@ async function executeImport( } } - const buffer = await fetchWorkspaceFileBuffer(toFileRecord(row)) + // The bytes are headed straight for `parseWorkflowJson`, so the import body ceiling is + // the real limit here — a larger file could not be imported even if it were read. + const buffer = await fetchWorkspaceFileBuffer(toFileRecord(row), { + maxBytes: MAX_IMPORT_BODY_BYTES, + }) const content = buffer.toString('utf-8') let parsed: unknown diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index b4a4756ff65..84a06683677 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -1659,7 +1659,7 @@ export async function getWorkspaceFile( */ export async function fetchServableWorkspaceFileBuffer( fileRecord: WorkspaceFileRecord, - options: { maxBytes?: number; signal?: AbortSignal; requestId?: string } = {} + options: { maxBytes: number; signal?: AbortSignal; requestId?: string } ): Promise<{ buffer: Buffer; contentType: string }> { const { downloadServableFileFromStorage } = await import('@/lib/uploads/utils/file-utils.server') @@ -1685,7 +1685,7 @@ export async function fetchServableWorkspaceFileBuffer( */ export async function fetchWorkspaceFileBuffer( fileRecord: WorkspaceFileRecord, - options: { maxBytes?: number } = {} + options: { maxBytes: number } ): Promise { logger.info(`Downloading workspace file: ${fileRecord.name}`) diff --git a/apps/sim/lib/uploads/shared/types.ts b/apps/sim/lib/uploads/shared/types.ts index e0fb1ffcb94..7d9ebc3adca 100644 --- a/apps/sim/lib/uploads/shared/types.ts +++ b/apps/sim/lib/uploads/shared/types.ts @@ -25,6 +25,25 @@ export const MAX_WORKSPACE_FORMDATA_FILE_SIZE = 100 * 1024 * 1024 /** Maximum size accepted by the knowledge-document parsing pipeline. */ export const MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE = 100 * 1024 * 1024 +/** + * Default ceiling for a read that holds the whole file resident as one `Buffer`. + * + * Workspace files are admitted at {@link MAX_WORKSPACE_FILE_SIZE} (5 GB) because they + * are streamed straight to object storage and never sit in the app process. A tool + * that pulls one back to hand it to a third party does not stream — it buffers, then + * usually copies again (base64, `Blob`, multipart), so peak resident memory is a + * multiple of the file. Sharing one ceiling keeps that multiple bounded no matter how + * many blocks run concurrently. + * + * 100 MB is the value this codebase already converged on for buffered work + * ({@link MAX_WORKSPACE_FORMDATA_FILE_SIZE}, `MAX_ARCHIVE_BYTES`, the 100 MB + * `maxResponseBytes` on the STT URL branch, and the ClickUp/Vanta/Daytona/Linq/SFTP + * upload routes). Use a destination's own documented limit instead whenever it is + * lower — failing here beats a slow round trip to a provider that will reject it. + * Genuinely large transfers belong on `downloadFileStream`, not on a bigger ceiling. + */ +export const MAX_BUFFERED_TRANSFER_BYTES = 100 * 1024 * 1024 + /** * Rejection wording shared by every surface that admits a knowledge document. * diff --git a/apps/sim/lib/uploads/utils/file-utils.server.test.ts b/apps/sim/lib/uploads/utils/file-utils.server.test.ts index 6cae5e88cfa..f93ce28d490 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.test.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.test.ts @@ -33,9 +33,12 @@ vi.mock('@/app/api/files/authorization', () => ({ })) import { createLogger } from '@sim/logger' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { downloadFileFromStorage, downloadServableFileFromStorage, + downloadServableFilesWithinBudget, } from '@/lib/uploads/utils/file-utils.server' import type { UserFile } from '@/executor/types' @@ -61,7 +64,9 @@ describe('downloadFileFromStorage context derivation', () => { context: 'og-images', } - await downloadFileFromStorage(userFile, 'req-1', createLogger('test')) + await downloadFileFromStorage(userFile, 'req-1', createLogger('test'), { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) expect(mockDownloadFile).toHaveBeenCalledTimes(1) expect(mockDownloadFile).toHaveBeenCalledWith( @@ -83,6 +88,7 @@ describe('downloadFileFromStorage context derivation', () => { const filePrincipal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } await downloadServableFileFromStorage(userFile, 'req-1', createLogger('test'), { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, filePrincipal, }) @@ -91,3 +97,112 @@ describe('downloadFileFromStorage context derivation', () => { ) }) }) + +describe('downloadFileFromStorage size ceiling', () => { + const logger = createLogger('test') + const fileOfSize = (size: number): UserFile => ({ + id: 'f1', + name: 'clip.wav', + url: '', + size, + type: 'audio/wav', + key: 'workspace/ws-1/1700000000000-abc1234-clip.wav', + }) + + beforeEach(() => { + vi.clearAllMocks() + mockParseWorkspaceFileKey.mockReturnValue(null) + }) + + it('rejects on the declared size before moving any bytes', async () => { + await expect( + downloadFileFromStorage(fileOfSize(2048), 'req-1', logger, { maxBytes: 1024 }) + ).rejects.toThrow(PayloadSizeLimitError) + + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('rejects on the delivered bytes when the declared size understated them', async () => { + mockDownloadFile.mockResolvedValue(Buffer.alloc(2048)) + + await expect( + downloadFileFromStorage(fileOfSize(1), 'req-1', logger, { maxBytes: 1024 }) + ).rejects.toThrow(PayloadSizeLimitError) + }) + + it('forwards the ceiling to the storage layer so a provider can stop mid-stream', async () => { + mockDownloadFile.mockResolvedValue(Buffer.alloc(512)) + + await downloadFileFromStorage(fileOfSize(512), 'req-1', logger, { maxBytes: 1024 }) + + expect(mockDownloadFile).toHaveBeenCalledWith(expect.objectContaining({ maxBytes: 1024 })) + }) +}) + +describe('downloadServableFilesWithinBudget', () => { + const logger = createLogger('test') + const fileOfSize = (name: string, size: number): UserFile => ({ + id: name, + name, + url: '', + size, + type: 'application/octet-stream', + key: `workspace/ws-1/1700000000000-abc1234-${name}`, + }) + + beforeEach(() => { + vi.clearAllMocks() + mockParseWorkspaceFileKey.mockReturnValue(null) + mockDownloadFile.mockImplementation(async ({ key }) => + Buffer.alloc(key.endsWith('big.bin') ? 900 : 400) + ) + }) + + it('spends the budget across the list rather than per file', async () => { + const resolved = await downloadServableFilesWithinBudget( + [fileOfSize('a.bin', 400), fileOfSize('b.bin', 400)], + 'req-1', + logger, + { totalMaxBytes: 1000, label: 'Total attachment size' } + ) + + expect(resolved.map((r) => r.buffer.length)).toEqual([400, 400]) + // The second file was only offered what the first left behind. + expect(mockDownloadFile).toHaveBeenNthCalledWith(2, expect.objectContaining({ maxBytes: 600 })) + }) + + it('rejects the combined size even when every file is individually under the limit', async () => { + const failure = await downloadServableFilesWithinBudget( + [fileOfSize('a.bin', 400), fileOfSize('b.bin', 400), fileOfSize('c.bin', 400)], + 'req-1', + logger, + { totalMaxBytes: 1000, label: 'Total attachment size' } + ).catch((error) => error) + + // Restated in the caller's terms: the whole set against the whole budget, not the + // third file against the 200 bytes the first two happened to leave. + expect(failure).toBeInstanceOf(PayloadSizeLimitError) + expect(failure).toMatchObject({ + label: 'Total attachment size', + maxBytes: 1000, + observedBytes: 1200, + }) + + // The third file's declared size already exceeds what the first two left, so it is + // refused without fetching its bytes — the whole set is never resident at once. + expect(mockDownloadFile).toHaveBeenCalledTimes(2) + }) + + it('refuses the next file on its declared size once the budget is spent', async () => { + await expect( + downloadServableFilesWithinBudget( + [fileOfSize('big.bin', 900), fileOfSize('a.bin', 400)], + 'req-1', + logger, + { totalMaxBytes: 1000, label: 'Total attachment size' } + ) + ).rejects.toThrow(PayloadSizeLimitError) + + expect(mockDownloadFile).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index e20b1900b49..579ad67f65d 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -9,6 +9,8 @@ import { import { assertKnownSizeWithinLimit, consumeOrCancelBody, + isPayloadSizeLimitError, + PayloadSizeLimitError, readResponseToBufferWithLimit, } from '@/lib/core/utils/stream-limits' import { StorageService } from '@/lib/uploads' @@ -335,29 +337,38 @@ export async function resolveInternalFileUrl( } /** - * Downloads a file from storage (execution or regular) + * Downloads a file from storage (execution or regular) into a single resident buffer. + * + * `maxBytes` is required, not optional. Workspace files are admitted at 5 GB, so a + * caller that forgets a ceiling inherits "unbounded" and can allocate gigabytes inside + * the shared app process. Making the parameter mandatory means a new call site has to + * name its limit — see {@link MAX_BUFFERED_TRANSFER_BYTES} for the default, and prefer + * the destination's own documented limit whenever it is lower. The size is checked + * twice: against the declared size before any bytes move, and against the delivered + * buffer, because the declared size comes from the caller and may be a lie. + * * @param userFile - UserFile object * @param requestId - Request ID for logging * @param logger - Logger instance + * @param options.maxBytes - Hard ceiling; throws `PayloadSizeLimitError` when exceeded * @returns Buffer containing file data */ export async function downloadFileFromStorage( userFile: UserFile, requestId: string, logger: Logger, - options: { maxBytes?: number } = {} + options: { maxBytes: number } ): Promise { + const { maxBytes } = options let buffer: Buffer - if (options.maxBytes !== undefined && userFile.size > options.maxBytes) { - assertKnownSizeWithinLimit(userFile.size, options.maxBytes, 'storage file download') - } + assertKnownSizeWithinLimit(userFile.size, maxBytes, 'storage file download') if (isExecutionFile(userFile)) { logger.info(`[${requestId}] Downloading from execution storage: ${userFile.key}`) const { downloadExecutionFile } = await import( '@/lib/uploads/contexts/execution/execution-file-manager' ) - buffer = await downloadExecutionFile(userFile, { maxBytes: options.maxBytes }) + buffer = await downloadExecutionFile(userFile, { maxBytes }) } else if (userFile.key) { const context = resolveTrustedFileContext(userFile.key, userFile.context) logger.info(`[${requestId}] Downloading from ${context} storage: ${userFile.key}`) @@ -366,15 +377,13 @@ export async function downloadFileFromStorage( buffer = await downloadFile({ key: userFile.key, context, - maxBytes: options.maxBytes, + maxBytes, }) } else { throw new Error('File has no key - cannot download') } - if (options.maxBytes !== undefined) { - assertKnownSizeWithinLimit(buffer.length, options.maxBytes, 'storage file download') - } + assertKnownSizeWithinLimit(buffer.length, maxBytes, 'storage file download') return buffer } @@ -404,17 +413,21 @@ export interface ServableFile { * * Throws `DocCompileUserError` when a generated doc's artifact is not ready (still * compiling) — callers should surface a retryable error rather than attach source. + * + * `maxBytes` is required for the reason given on {@link downloadFileFromStorage}: it + * bounds the source read. A compiled artifact is re-checked against the same ceiling + * below, since rendering can grow a small source into a large document. */ export async function downloadServableFileFromStorage( userFile: UserFile, requestId: string, logger: Logger, options: { - maxBytes?: number + maxBytes: number signal?: AbortSignal ownerKey?: string filePrincipal?: Principal - } = {} + } ): Promise { const buffer = await downloadFileFromStorage(userFile, requestId, logger, { maxBytes: options.maxBytes, @@ -430,10 +443,14 @@ export async function downloadServableFileFromStorage( const workspaceId = userFile.key ? (parseWorkspaceFileKey(userFile.key) ?? undefined) : undefined - return { - buffer: Buffer.from(await renderSimPageDocumentWithAssets(text, { workspaceId }), 'utf8'), - contentType: 'text/html', - } + const rendered = Buffer.from( + await renderSimPageDocumentWithAssets(text, { workspaceId }), + 'utf8' + ) + // Rendering inlines referenced assets, so a source well under the ceiling can + // resolve to a document well over it. + assertKnownSizeWithinLimit(rendered.length, options.maxBytes, 'servable page render') + return { buffer: rendered, contentType: 'text/html' } } } @@ -462,8 +479,58 @@ export async function downloadServableFileFromStorage( // Re-check: the raw download enforced maxBytes on the source, but a generated doc // resolves to a larger artifact. - if (options.maxBytes !== undefined && resolved.buffer.length > options.maxBytes) { - assertKnownSizeWithinLimit(resolved.buffer.length, options.maxBytes, 'servable file download') + assertKnownSizeWithinLimit(resolved.buffer.length, options.maxBytes, 'servable file download') + + return resolved +} + +/** + * Resolve every file of a multi-attachment request while bounding their COMBINED + * resident size. + * + * A per-file ceiling is not enough when a request carries an array: N attachments + * each just under the limit still cost N times the limit, and downloading them with + * `Promise.all` makes that the peak. Routes that pre-checked `sum(file.size)` were + * not protected either — the declared sizes come from the caller, which is why the + * downloader re-checks the delivered bytes. + * + * So this walks the list in order against a shrinking budget: each file may only use + * what the previous ones left. Sequential is the point — it is what keeps the peak at + * `totalMaxBytes` instead of the sum, and attachment lists are short enough that the + * lost parallelism does not register next to the provider round trip that follows. + * + * The overrun surfaces as a `PayloadSizeLimitError` restated in the caller's terms: + * the per-file failure underneath reports one file against whatever budget was left, + * which would read as a nonsense limit in a "total attachment size" message. `label` + * and `totalMaxBytes` are the caller's, and `observedBytes` is what the set needed. + */ +export async function downloadServableFilesWithinBudget( + userFiles: readonly UserFile[], + requestId: string, + logger: Logger, + options: { totalMaxBytes: number; label: string; signal?: AbortSignal } +): Promise { + const resolved: ServableFile[] = [] + let spent = 0 + + for (const userFile of userFiles) { + logger.info(`[${requestId}] Downloading ${userFile.name} (${userFile.size} bytes)`) + let servable: ServableFile + try { + servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: options.totalMaxBytes - spent, + signal: options.signal, + }) + } catch (error) { + if (!isPayloadSizeLimitError(error)) throw error + throw new PayloadSizeLimitError({ + label: options.label, + maxBytes: options.totalMaxBytes, + observedBytes: spent + (error.observedBytes ?? userFile.size), + }) + } + spent += servable.buffer.length + resolved.push(servable) } return resolved diff --git a/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts b/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts index f7456f44d73..725740ed411 100644 --- a/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts +++ b/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts @@ -13,7 +13,7 @@ const logger = createLogger('FetchServableWorkspaceFileBuffer') export async function fetchAuthorizedServableWorkspaceFileBuffer( fileRecord: WorkspaceFileRecord, filePrincipal: Principal, - options: { maxBytes?: number; signal?: AbortSignal; requestId?: string } = {} + options: { maxBytes: number; signal?: AbortSignal; requestId?: string } ): Promise<{ buffer: Buffer; contentType: string }> { return downloadServableFileFromStorage( { diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts index 39943c138e4..ca1ca6b784d 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts @@ -26,6 +26,7 @@ vi.mock('@sim/platform-authz/workspace', () => ({ resolveEffectiveWorkspacePermission: mocks.resolvePermission, })) +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { readWorkspaceFileContentByKey } from '@/lib/workspace-files/application/read-workspace-file-content-by-key' const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } @@ -75,7 +76,9 @@ describe('readWorkspaceFileContentByKey', () => { expect(mocks.getFile).toHaveBeenCalledWith(file.workspaceId, file.id, { throwOnError: true, }) - expect(mocks.fetchContent).toHaveBeenCalledWith(file) + expect(mocks.fetchContent).toHaveBeenCalledWith(file, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) }) it('rejects a stale key instead of serving the file current at the same ID', async () => { diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts index d5629ed275b..4a669fd940f 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts @@ -8,6 +8,7 @@ import { type WorkspaceFileRecord, } from '@/lib/uploads/contexts/workspace' import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { fileOperations } from '@/lib/workspace-files/application/operations' @@ -33,7 +34,10 @@ async function executeReadWorkspaceFileContentByKey({ throwOnError: true, }) if (!file || file.key !== input.key) throw new OrchestrationError('not_found', 'File not found') - return { file, content: await fetchWorkspaceFileBuffer(file) } + return { + file, + content: await fetchWorkspaceFileBuffer(file, { maxBytes: MAX_BUFFERED_TRANSFER_BYTES }), + } } export const readWorkspaceFileContentByKey = defineAuthorizedWorkspaceFileUseCase({ diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-content.ts index f35a14df784..bfd1020bd50 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-content.ts @@ -10,6 +10,7 @@ import { getBoundWorkspaceFileSecretProvenance, type WorkspaceFileSecretProvenance, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { fileOperations } from '@/lib/workspace-files/application/operations' import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/application/workspace-file-context' @@ -43,7 +44,9 @@ async function executeReadWorkspaceFileContent({ throwOnError: true, }) if (!file) throw new OrchestrationError('not_found', 'File not found') - const content = await fetchWorkspaceFileBuffer(file, { maxBytes: input.maxBytes }) + const content = await fetchWorkspaceFileBuffer(file, { + maxBytes: input.maxBytes ?? MAX_BUFFERED_TRANSFER_BYTES, + }) const secretProvenance = input.includeSecretProvenance ? await getBoundWorkspaceFileSecretProvenance(context.workspaceId, { fileId: file.id, diff --git a/apps/sim/tools/microsoft_teams/server-utils.ts b/apps/sim/tools/microsoft_teams/server-utils.ts index c5558ad8512..75df5c83fe6 100644 --- a/apps/sim/tools/microsoft_teams/server-utils.ts +++ b/apps/sim/tools/microsoft_teams/server-utils.ts @@ -5,6 +5,7 @@ */ import type { Logger } from '@sim/logger' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { FileAccessDeniedError, verifyFileAccess } from '@/app/api/files/authorization' @@ -79,11 +80,19 @@ export async function uploadFilesForTeamsMessage(params: { throw new FileAccessDeniedError() } - // Download file from storage - const { buffer, contentType } = await downloadServableFileFromStorage(file, requestId, log) - - if (buffer.length > MAX_TEAMS_FILE_SIZE) { - const sizeMB = (buffer.length / (1024 * 1024)).toFixed(2) + // Download file from storage, bounded by the same 4MB limit checked above — the + // declared size is the caller's claim, so the ceiling has to reach the read itself. + let buffer: Buffer + let contentType: string + try { + const servable = await downloadServableFileFromStorage(file, requestId, log, { + maxBytes: MAX_TEAMS_FILE_SIZE, + }) + buffer = servable.buffer + contentType = servable.contentType + } catch (error) { + if (!isPayloadSizeLimitError(error)) throw error + const sizeMB = ((error.observedBytes ?? file.size) / (1024 * 1024)).toFixed(2) throw new Error( `File "${file.name}" (${sizeMB}MB) exceeds the 4MB limit for Teams attachments. Use smaller files or upload to SharePoint/OneDrive first.` ) From 6e9406cd93e8b4456f197e717510ba2f272c9719 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Sat, 22 Aug 2026 13:23:41 -0700 Subject: [PATCH 2/3] fix(uploads): bound Sim-page asset inlining before the bytes are resident MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ceiling on the rendered page checked the finished document, by which point renderSimPageDocumentWithAssets had already downloaded every referenced image concurrently with no per-download limit and base64-inlined them — so the allocation the check exists to prevent had already happened. Pick the inline set from recorded sizes before fetching anything, against a per-document budget as well as the existing per-image one, and give each download its own ceiling in case a row understates its object. An image that does not fit keeps its URL reference, exactly as an oversized one already did. --- .../page-document.server.test.ts | 112 ++++++++++++++++++ .../workspace-files/page-document.server.ts | 46 +++++-- 2 files changed, 151 insertions(+), 7 deletions(-) create mode 100644 apps/sim/lib/workspace-files/page-document.server.test.ts diff --git a/apps/sim/lib/workspace-files/page-document.server.test.ts b/apps/sim/lib/workspace-files/page-document.server.test.ts new file mode 100644 index 00000000000..a2ee13149f3 --- /dev/null +++ b/apps/sim/lib/workspace-files/page-document.server.test.ts @@ -0,0 +1,112 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockDownloadFile, mockGetFileMetadataById, mockRenderSimPageDocument } = vi.hoisted(() => ({ + mockDownloadFile: vi.fn(), + mockGetFileMetadataById: vi.fn(), + mockRenderSimPageDocument: vi.fn(), +})) + +vi.mock('@/lib/uploads/core/storage-service', () => ({ + downloadFile: mockDownloadFile, +})) + +vi.mock('@/lib/uploads/server/metadata', () => ({ + getFileMetadataById: mockGetFileMetadataById, +})) + +vi.mock('@/lib/workspace-files/page-document', () => ({ + renderSimPageDocument: mockRenderSimPageDocument, +})) + +import { renderSimPageDocumentWithAssets } from '@/lib/workspace-files/page-document.server' + +const WORKSPACE_ID = 'ws-1' +const MB = 1024 * 1024 + +function imageRecord(id: string, size: number) { + return { + id, + key: `workspace/${WORKSPACE_ID}/${id}.png`, + context: 'workspace', + workspaceId: WORKSPACE_ID, + contentType: 'image/png', + size, + sizeBytes: size, + } +} + +function documentReferencing(ids: string[]) { + return ids.map((id) => ``).join('') +} + +describe('renderSimPageDocumentWithAssets memory bounds', () => { + beforeEach(() => { + vi.clearAllMocks() + mockDownloadFile.mockImplementation(async ({ maxBytes }) => Buffer.alloc(maxBytes ?? 4 * MB)) + }) + + it('stops inlining once the per-document budget is spent, without fetching the rest', async () => { + // Five 8MB images against a 32MB document budget: four fit, the fifth must not + // even be downloaded — discovering its size after the fact is the bug. + const ids = ['a', 'b', 'c', 'd', 'e'] + mockRenderSimPageDocument.mockReturnValue(documentReferencing(ids)) + mockGetFileMetadataById.mockImplementation(async (id: string) => imageRecord(id, 8 * MB)) + mockDownloadFile.mockImplementation(async () => Buffer.alloc(8 * MB)) + + const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) + + expect(mockDownloadFile).toHaveBeenCalledTimes(4) + // The image that did not fit keeps its URL reference rather than failing the render. + expect(html).toContain('src="/api/files/view/e"') + }) + + it('never fetches an image whose recorded size already exceeds the per-image limit', async () => { + mockRenderSimPageDocument.mockReturnValue(documentReferencing(['big'])) + mockGetFileMetadataById.mockResolvedValue(imageRecord('big', 9 * MB)) + + const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(html).toContain('src="/api/files/view/big"') + }) + + it('caps each download so a row understating its object cannot be inlined', async () => { + mockRenderSimPageDocument.mockReturnValue(documentReferencing(['liar'])) + mockGetFileMetadataById.mockResolvedValue(imageRecord('liar', 1024)) + + await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) + + expect(mockDownloadFile).toHaveBeenCalledWith( + expect.objectContaining({ maxBytes: 8 * MB, context: 'workspace' }) + ) + }) + + it('keeps the URL reference when a capped download rejects', async () => { + mockRenderSimPageDocument.mockReturnValue(documentReferencing(['liar'])) + mockGetFileMetadataById.mockResolvedValue(imageRecord('liar', 1024)) + mockDownloadFile.mockRejectedValue(new Error('storage download exceeds maximum size')) + + const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) + + expect(html).toContain('src="/api/files/view/liar"') + }) + + it('inlines images that fit and leaves cross-workspace references alone', async () => { + mockRenderSimPageDocument.mockReturnValue(documentReferencing(['mine', 'theirs'])) + mockGetFileMetadataById.mockImplementation(async (id: string) => + id === 'mine' + ? imageRecord('mine', 1024) + : { ...imageRecord('theirs', 1024), workspaceId: 'ws-2' } + ) + mockDownloadFile.mockResolvedValue(Buffer.from('png-bytes')) + + const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) + + expect(mockDownloadFile).toHaveBeenCalledTimes(1) + expect(html).toContain(`data:image/png;base64,${Buffer.from('png-bytes').toString('base64')}`) + expect(html).toContain('src="/api/files/view/theirs"') + }) +}) diff --git a/apps/sim/lib/workspace-files/page-document.server.ts b/apps/sim/lib/workspace-files/page-document.server.ts index d0e2d5a7c22..b608cb63bfd 100644 --- a/apps/sim/lib/workspace-files/page-document.server.ts +++ b/apps/sim/lib/workspace-files/page-document.server.ts @@ -5,6 +5,14 @@ import { renderSimPageDocument } from '@/lib/workspace-files/page-document' /** Images past this size stay as URL references rather than bloating the document. */ const MAX_INLINE_IMAGE_BYTES = 8 * 1024 * 1024 +/** + * Ceiling on everything a single document inlines. A per-image limit does not bound + * the page on its own: N images each just under it still cost N times it, and they + * are fetched concurrently, so that product is also the peak. Images that do not fit + * the remaining budget keep their URL reference, exactly like an oversized one. + */ +const MAX_INLINE_TOTAL_BYTES = 32 * 1024 * 1024 + const IMAGE_SRC = /src="[^"]*\/api\/files\/view\/([^"]+)"/g /** @@ -24,21 +32,45 @@ export async function renderSimPageDocumentWithAssets( const ids = [...new Set([...documentHtml.matchAll(IMAGE_SRC)].map((match) => match[1]))] if (ids.length === 0 || !options.workspaceId) return documentHtml + const candidates = await Promise.all( + ids.map(async (id) => { + const record = await getFileMetadataById(id).catch(() => null) + if (!record || record.context !== 'workspace' || record.workspaceId !== options.workspaceId) + return null + return { id, record } + }) + ) + + // Pick the inline set from recorded sizes BEFORE fetching anything, so the concurrent + // downloads below are bounded in count and in total bytes rather than discovering the + // size of each image only once it is already resident. These sizes are written by the + // upload pipeline, not supplied by the caller, so they are sound to plan against — + // each download still carries its own ceiling in case a row understates its object. + let remaining = MAX_INLINE_TOTAL_BYTES + const eligible: NonNullable<(typeof candidates)[number]>[] = [] + for (const candidate of candidates) { + if (!candidate) continue + const size = candidate.record.sizeBytes ?? candidate.record.size + if (size > MAX_INLINE_IMAGE_BYTES || size > remaining) continue + remaining -= size + eligible.push(candidate) + } + const inlined = new Map() await Promise.all( - ids.map(async (id) => { + eligible.map(async ({ id, record }) => { try { - const record = await getFileMetadataById(id) - if (!record || record.context !== 'workspace' || record.workspaceId !== options.workspaceId) - return - const bytes = await downloadFile({ key: record.key, context: 'workspace' }) - if (bytes.length > MAX_INLINE_IMAGE_BYTES) return + const bytes = await downloadFile({ + key: record.key, + context: 'workspace', + maxBytes: MAX_INLINE_IMAGE_BYTES, + }) const mime = record.contentType?.startsWith('image/') ? record.contentType : 'application/octet-stream' inlined.set(id, `data:${mime};base64,${bytes.toString('base64')}`) } catch { - // A missing or unreadable image keeps its URL reference. + // A missing, unreadable or oversized image keeps its URL reference. } }) ) From 733c6fa8cfd5907058e08052d1ecbfb79cc8ce35 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Sat, 22 Aug 2026 13:48:08 -0700 Subject: [PATCH 3/3] improvement(uploads): charge the page-render budget by delivered bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Planning the inline set from recorded sizes left the aggregate ceiling resting on metadata being accurate, and needed a paragraph explaining why that was safe. Downloading one image at a time and subtracting what each download actually returned needs no such argument: the budget cannot be exceeded whatever a row says, and the peak is one image rather than the sum of them. Also drop two rough edges the first pass left behind — an SFTP total-size check that became unreachable once the download carried the remaining budget, and a Dataverse error helper whose optional size argument existed only to paper over one caller that had not passed it. --- .../microsoft-dataverse/upload-file/route.ts | 10 ++-- apps/sim/app/api/tools/sftp/upload/route.ts | 16 +++--- .../page-document.server.test.ts | 47 +++++++++------- .../workspace-files/page-document.server.ts | 56 ++++++++----------- 4 files changed, 66 insertions(+), 63 deletions(-) diff --git a/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts b/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts index 097c94a1e7a..826c547df96 100644 --- a/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts +++ b/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts @@ -21,13 +21,12 @@ const logger = createLogger('DataverseUploadFileAPI') /** Dataverse Web API's absolute ceiling for a single-request (non-chunked) file column upload. */ const DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES = 128 * 1024 * 1024 -function uploadTooLargeError(observedBytes?: number): NextResponse { - const sizeLabel = - observedBytes === undefined ? '' : `(${(observedBytes / (1024 * 1024)).toFixed(2)}MB) ` +function uploadTooLargeError(observedBytes: number): NextResponse { + const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) return NextResponse.json( { success: false, - error: `File size ${sizeLabel}exceeds Dataverse's 128MB limit for single-request file column uploads. Split the file and use chunked upload instead.`, + error: `File size (${sizeMB}MB) exceeds Dataverse's 128MB limit for single-request file column uploads. Split the file and use chunked upload instead.`, }, { status: 400 } ) @@ -97,7 +96,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady - if (isPayloadSizeLimitError(error)) return uploadTooLargeError(error.observedBytes) + if (isPayloadSizeLimitError(error)) + return uploadTooLargeError(error.observedBytes ?? userFile.size) logger.error(`[${requestId}] Failed to download file from storage:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, diff --git a/apps/sim/app/api/tools/sftp/upload/route.ts b/apps/sim/app/api/tools/sftp/upload/route.ts index cd842fedbb8..bca922e7d20 100644 --- a/apps/sim/app/api/tools/sftp/upload/route.ts +++ b/apps/sim/app/api/tools/sftp/upload/route.ts @@ -5,6 +5,7 @@ import { sftpUploadContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -114,13 +115,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { }) resolvedTotal += buffer.length - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { success: false, error: `Total file size (${sizeMB}MB) exceeds limit of 100MB` }, - { status: 400 } - ) - } const safeFileName = sanitizeFileName(file.name) const fullRemotePath = remotePath.endsWith('/') @@ -157,6 +151,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const observed = resolvedTotal + (error.observedBytes ?? file.size) + const sizeMB = (observed / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { success: false, error: `Total file size (${sizeMB}MB) exceeds limit of 100MB` }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to upload file ${file.name}:`, error) throw new Error( `Failed to upload file "${file.name}": ${getErrorMessage(error, 'Unknown error')}` diff --git a/apps/sim/lib/workspace-files/page-document.server.test.ts b/apps/sim/lib/workspace-files/page-document.server.test.ts index a2ee13149f3..14375b70a16 100644 --- a/apps/sim/lib/workspace-files/page-document.server.test.ts +++ b/apps/sim/lib/workspace-files/page-document.server.test.ts @@ -45,37 +45,46 @@ function documentReferencing(ids: string[]) { describe('renderSimPageDocumentWithAssets memory bounds', () => { beforeEach(() => { vi.clearAllMocks() - mockDownloadFile.mockImplementation(async ({ maxBytes }) => Buffer.alloc(maxBytes ?? 4 * MB)) }) - it('stops inlining once the per-document budget is spent, without fetching the rest', async () => { - // Five 8MB images against a 32MB document budget: four fit, the fifth must not - // even be downloaded — discovering its size after the fact is the bug. - const ids = ['a', 'b', 'c', 'd', 'e'] + it('charges the budget by delivered bytes, not by what the metadata claimed', async () => { + // Every row claims to be tiny; the objects are 8MB each. The budget must still + // stop at 32MB — planning off the recorded size would admit all six. + const ids = ['a', 'b', 'c', 'd', 'e', 'f'] mockRenderSimPageDocument.mockReturnValue(documentReferencing(ids)) - mockGetFileMetadataById.mockImplementation(async (id: string) => imageRecord(id, 8 * MB)) + mockGetFileMetadataById.mockImplementation(async (id: string) => imageRecord(id, 1024)) mockDownloadFile.mockImplementation(async () => Buffer.alloc(8 * MB)) const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) expect(mockDownloadFile).toHaveBeenCalledTimes(4) - // The image that did not fit keeps its URL reference rather than failing the render. + // The images past the budget keep their URL reference rather than failing the render. expect(html).toContain('src="/api/files/view/e"') + expect(html).toContain('src="/api/files/view/f"') }) - it('never fetches an image whose recorded size already exceeds the per-image limit', async () => { - mockRenderSimPageDocument.mockReturnValue(documentReferencing(['big'])) - mockGetFileMetadataById.mockResolvedValue(imageRecord('big', 9 * MB)) + it('offers each download only what the budget has left', async () => { + mockRenderSimPageDocument.mockReturnValue(documentReferencing(['a', 'b'])) + mockGetFileMetadataById.mockImplementation(async (id: string) => imageRecord(id, 1024)) + mockDownloadFile.mockImplementation(async () => Buffer.alloc(30 * MB)) - const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) + await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) - expect(mockDownloadFile).not.toHaveBeenCalled() - expect(html).toContain('src="/api/files/view/big"') + expect(mockDownloadFile).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ maxBytes: 8 * MB }) + ) + // 30MB delivered leaves 2MB, which is below the per-image limit and becomes the cap. + expect(mockDownloadFile).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ maxBytes: 2 * MB }) + ) }) - it('caps each download so a row understating its object cannot be inlined', async () => { - mockRenderSimPageDocument.mockReturnValue(documentReferencing(['liar'])) - mockGetFileMetadataById.mockResolvedValue(imageRecord('liar', 1024)) + it('never offers a download more than the per-image limit', async () => { + mockRenderSimPageDocument.mockReturnValue(documentReferencing(['solo'])) + mockGetFileMetadataById.mockResolvedValue(imageRecord('solo', 1024)) + mockDownloadFile.mockResolvedValue(Buffer.from('png-bytes')) await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) @@ -85,13 +94,13 @@ describe('renderSimPageDocumentWithAssets memory bounds', () => { }) it('keeps the URL reference when a capped download rejects', async () => { - mockRenderSimPageDocument.mockReturnValue(documentReferencing(['liar'])) - mockGetFileMetadataById.mockResolvedValue(imageRecord('liar', 1024)) + mockRenderSimPageDocument.mockReturnValue(documentReferencing(['big'])) + mockGetFileMetadataById.mockResolvedValue(imageRecord('big', 1024)) mockDownloadFile.mockRejectedValue(new Error('storage download exceeds maximum size')) const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) - expect(html).toContain('src="/api/files/view/liar"') + expect(html).toContain('src="/api/files/view/big"') }) it('inlines images that fit and leaves cross-workspace references alone', async () => { diff --git a/apps/sim/lib/workspace-files/page-document.server.ts b/apps/sim/lib/workspace-files/page-document.server.ts index b608cb63bfd..4c4954e33dc 100644 --- a/apps/sim/lib/workspace-files/page-document.server.ts +++ b/apps/sim/lib/workspace-files/page-document.server.ts @@ -7,9 +7,8 @@ const MAX_INLINE_IMAGE_BYTES = 8 * 1024 * 1024 /** * Ceiling on everything a single document inlines. A per-image limit does not bound - * the page on its own: N images each just under it still cost N times it, and they - * are fetched concurrently, so that product is also the peak. Images that do not fit - * the remaining budget keep their URL reference, exactly like an oversized one. + * the page on its own — N images each just under it still cost N times it. Images + * that do not fit what is left keep their URL reference, exactly like an oversized one. */ const MAX_INLINE_TOTAL_BYTES = 32 * 1024 * 1024 @@ -41,39 +40,32 @@ export async function renderSimPageDocumentWithAssets( }) ) - // Pick the inline set from recorded sizes BEFORE fetching anything, so the concurrent - // downloads below are bounded in count and in total bytes rather than discovering the - // size of each image only once it is already resident. These sizes are written by the - // upload pipeline, not supplied by the caller, so they are sound to plan against — - // each download still carries its own ceiling in case a row understates its object. + // One image at a time, charged against the budget by what each download actually + // delivered. Fetching them concurrently made the peak the sum of every image rather + // than the largest one, and the ceiling on the finished document could only observe + // that after the fact. Each download is given whatever the budget has left, so an + // image that does not fit is refused by the read itself instead of after it lands. + const inlined = new Map() let remaining = MAX_INLINE_TOTAL_BYTES - const eligible: NonNullable<(typeof candidates)[number]>[] = [] for (const candidate of candidates) { if (!candidate) continue - const size = candidate.record.sizeBytes ?? candidate.record.size - if (size > MAX_INLINE_IMAGE_BYTES || size > remaining) continue - remaining -= size - eligible.push(candidate) + if (remaining === 0) break + const { id, record } = candidate + try { + const bytes = await downloadFile({ + key: record.key, + context: 'workspace', + maxBytes: Math.min(MAX_INLINE_IMAGE_BYTES, remaining), + }) + remaining -= bytes.length + const mime = record.contentType?.startsWith('image/') + ? record.contentType + : 'application/octet-stream' + inlined.set(id, `data:${mime};base64,${bytes.toString('base64')}`) + } catch { + // A missing, unreadable or too-large image keeps its URL reference. + } } - - const inlined = new Map() - await Promise.all( - eligible.map(async ({ id, record }) => { - try { - const bytes = await downloadFile({ - key: record.key, - context: 'workspace', - maxBytes: MAX_INLINE_IMAGE_BYTES, - }) - const mime = record.contentType?.startsWith('image/') - ? record.contentType - : 'application/octet-stream' - inlined.set(id, `data:${mime};base64,${bytes.toString('base64')}`) - } catch { - // A missing, unreadable or oversized image keeps its URL reference. - } - }) - ) if (inlined.size === 0) return documentHtml return documentHtml.replace(IMAGE_SRC, (match, id: string) => { const dataUri = inlined.get(id)