diff --git a/apps/sim/app/api/tools/agiloft/attach/route.ts b/apps/sim/app/api/tools/agiloft/attach/route.ts index 014e34eb271..a9f0ce22ac6 100644 --- a/apps/sim/app/api/tools/agiloft/attach/route.ts +++ b/apps/sim/app/api/tools/agiloft/attach/route.ts @@ -6,7 +6,9 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { secureFetchWithPinnedIP } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -75,13 +77,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let fileBuffer: Buffer try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = servable.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady logger.error(`[${requestId}] Failed to download file from storage:`, error) - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) + return NextResponse.json( + { success: false, error: toError(error).message }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) } const resolvedFileName = data.fileName || userFile.name || 'attachment' diff --git a/apps/sim/app/api/tools/box/upload/route.ts b/apps/sim/app/api/tools/box/upload/route.ts index 57ec5f4223c..f490fdc02a9 100644 --- a/apps/sim/app/api/tools/box/upload/route.ts +++ b/apps/sim/app/api/tools/box/upload/route.ts @@ -5,7 +5,9 @@ import { boxUploadContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -55,14 +57,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = result.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } fileName = validatedData.fileName || userFile.name diff --git a/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts b/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts index eae6759b6cb..1702e54cc86 100644 --- a/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts +++ b/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts @@ -27,6 +27,7 @@ vi.mock('@/app/api/files/authorization', () => ({ assertToolFileAccess: mockAssertToolFileAccess, })) +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { POST } from '@/app/api/tools/brex/upload-receipt/route' const mockFetch = vi.fn() @@ -194,11 +195,25 @@ describe('POST /api/tools/brex/upload-receipt', () => { expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() }) + it('asks the downloader for at most 50 MB', async () => { + await POST(createMockRequest('POST', baseBody)) + + expect(mockDownloadFileFromStorage).toHaveBeenCalledWith( + expect.anything(), + expect.any(String), + expect.anything(), + { maxBytes: 50 * 1024 * 1024 } + ) + }) + it('rejects files over the 50 MB limit', async () => { - mockDownloadFileFromStorage.mockResolvedValueOnce({ - buffer: Buffer.alloc(50 * 1024 * 1024 + 1), - contentType: 'application/pdf', - }) + mockDownloadFileFromStorage.mockRejectedValueOnce( + new PayloadSizeLimitError({ + label: 'storage file download', + maxBytes: 50 * 1024 * 1024, + observedBytes: 50 * 1024 * 1024 + 1, + }) + ) const response = await POST(createMockRequest('POST', baseBody)) expect(response.status).toBe(400) diff --git a/apps/sim/app/api/tools/brex/upload-receipt/route.ts b/apps/sim/app/api/tools/brex/upload-receipt/route.ts index 6fb4ccfecf8..c63bb4374b7 100644 --- a/apps/sim/app/api/tools/brex/upload-receipt/route.ts +++ b/apps/sim/app/api/tools/brex/upload-receipt/route.ts @@ -9,6 +9,7 @@ import { validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -51,23 +52,25 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let fileBuffer: Buffer try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_RECEIPT_SIZE_BYTES, + }) fileBuffer = resolved.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + return NextResponse.json( + { success: false, error: 'Receipt file exceeds the 50 MB limit' }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download receipt file:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Unknown error') }, { status: 500 } ) } - if (fileBuffer.length > MAX_RECEIPT_SIZE_BYTES) { - return NextResponse.json( - { success: false, error: 'Receipt file exceeds the 50 MB limit' }, - { status: 400 } - ) - } const effectiveReceiptName = receiptName || userFile.name const endpoint = expenseId diff --git a/apps/sim/app/api/tools/confluence/upload-attachment/route.ts b/apps/sim/app/api/tools/confluence/upload-attachment/route.ts index 00057e4d10c..8e0b14f5d4f 100644 --- a/apps/sim/app/api/tools/confluence/upload-attachment/route.ts +++ b/apps/sim/app/api/tools/confluence/upload-attachment/route.ts @@ -5,7 +5,9 @@ import { confluenceUploadAttachmentContract } from '@/lib/api/contracts/selector import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processSingleFileToUserFile, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -94,7 +96,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let fileBuffer: Buffer let resolvedContentType: string try { - const servable = await downloadServableFileFromStorage(userFile, 'confluence-upload', logger) + const servable = await downloadServableFileFromStorage( + userFile, + 'confluence-upload', + logger, + { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + } + ) fileBuffer = servable.buffer resolvedContentType = servable.contentType } catch (error) { @@ -105,7 +114,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/daytona/upload/route.ts b/apps/sim/app/api/tools/daytona/upload/route.ts index 5c52509bb75..800649f0f4d 100644 --- a/apps/sim/app/api/tools/daytona/upload/route.ts +++ b/apps/sim/app/api/tools/daytona/upload/route.ts @@ -5,6 +5,7 @@ import { daytonaUploadFileContract } from '@/lib/api/contracts/tools/daytona' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -62,11 +63,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.info(`[${requestId}] Downloading file: ${userFile.name} (${userFile.size} bytes)`) try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_UPLOAD_SIZE_BYTES, + }) fileBuffer = servable.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + return NextResponse.json( + { success: false, error: 'File exceeds upload limit of 100MB' }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download file from storage:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, diff --git a/apps/sim/app/api/tools/discord/send-message/route.ts b/apps/sim/app/api/tools/discord/send-message/route.ts index aa644457654..bc9d30d526a 100644 --- a/apps/sim/app/api/tools/discord/send-message/route.ts +++ b/apps/sim/app/api/tools/discord/send-message/route.ts @@ -6,9 +6,11 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateNumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -146,12 +148,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - userFiles.map(async (file, i) => { - logger.info(`[${requestId}] Downloading file ${i}: ${file.name}`) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(userFiles, requestId, logger, { + totalMaxBytes: MAX_BUFFERED_TRANSFER_BYTES, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady @@ -161,7 +161,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { success: false, error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/dropbox/upload/route.ts b/apps/sim/app/api/tools/dropbox/upload/route.ts index 58873c334bc..af04b69db16 100644 --- a/apps/sim/app/api/tools/dropbox/upload/route.ts +++ b/apps/sim/app/api/tools/dropbox/upload/route.ts @@ -5,8 +5,10 @@ import { dropboxUploadContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { httpHeaderSafeJson } from '@/lib/core/utils/validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -58,14 +60,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = result.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } fileName = userFile.name diff --git a/apps/sim/app/api/tools/elevenlabs/audio/route.ts b/apps/sim/app/api/tools/elevenlabs/audio/route.ts index 969f6f4a231..871868037ad 100644 --- a/apps/sim/app/api/tools/elevenlabs/audio/route.ts +++ b/apps/sim/app/api/tools/elevenlabs/audio/route.ts @@ -20,6 +20,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -153,7 +154,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(file, requestId, logger) + const buffer = await downloadFileFromStorage(file, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) const ext = file.name.split('.').pop()?.toLowerCase() || '' source = { buffer, diff --git a/apps/sim/app/api/tools/firecrawl/parse/route.ts b/apps/sim/app/api/tools/firecrawl/parse/route.ts index 2fb4c7d1cd0..a1b1fe8888f 100644 --- a/apps/sim/app/api/tools/firecrawl/parse/route.ts +++ b/apps/sim/app/api/tools/firecrawl/parse/route.ts @@ -11,6 +11,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -85,7 +86,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const { buffer, contentType } = await downloadServableFileFromStorage( userFile, requestId, - logger + logger, + { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + } ) const formData = new FormData() diff --git a/apps/sim/app/api/tools/gmail/draft/route.ts b/apps/sim/app/api/tools/gmail/draft/route.ts index e376be1f01e..255c5242967 100644 --- a/apps/sim/app/api/tools/gmail/draft/route.ts +++ b/apps/sim/app/api/tools/gmail/draft/route.ts @@ -5,9 +5,10 @@ import { gmailDraftContract } from '@/lib/api/contracts/tools/google' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { @@ -97,17 +98,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -118,18 +125,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, mimeType: resolved[i].contentType || file.type || 'application/octet-stream', diff --git a/apps/sim/app/api/tools/gmail/edit-draft/route.ts b/apps/sim/app/api/tools/gmail/edit-draft/route.ts index f986a610f11..9e88ce6cdbd 100644 --- a/apps/sim/app/api/tools/gmail/edit-draft/route.ts +++ b/apps/sim/app/api/tools/gmail/edit-draft/route.ts @@ -5,9 +5,10 @@ import { gmailEditDraftContract } from '@/lib/api/contracts/tools/google' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { @@ -93,17 +94,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -114,18 +121,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, mimeType: resolved[i].contentType || file.type || 'application/octet-stream', diff --git a/apps/sim/app/api/tools/gmail/send/route.ts b/apps/sim/app/api/tools/gmail/send/route.ts index 59df7377b34..62b10377e88 100644 --- a/apps/sim/app/api/tools/gmail/send/route.ts +++ b/apps/sim/app/api/tools/gmail/send/route.ts @@ -5,9 +5,10 @@ import { gmailSendContract } from '@/lib/api/contracts/tools/google' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { @@ -97,17 +98,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -118,18 +125,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, mimeType: resolved[i].contentType || file.type || 'application/octet-stream', diff --git a/apps/sim/app/api/tools/google_drive/upload/route.ts b/apps/sim/app/api/tools/google_drive/upload/route.ts index 0600386324e..ca134da06cc 100644 --- a/apps/sim/app/api/tools/google_drive/upload/route.ts +++ b/apps/sim/app/api/tools/google_drive/upload/route.ts @@ -6,7 +6,9 @@ import { googleDriveUploadContract } from '@/lib/api/contracts/tools/google' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -124,7 +126,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let downloadedContentType = '' try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = result.buffer downloadedContentType = result.contentType } catch (error) { @@ -136,7 +140,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { success: false, error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/jira/add-attachment/route.ts b/apps/sim/app/api/tools/jira/add-attachment/route.ts index b23abf6656b..70218256ffc 100644 --- a/apps/sim/app/api/tools/jira/add-attachment/route.ts +++ b/apps/sim/app/api/tools/jira/add-attachment/route.ts @@ -5,6 +5,7 @@ import { jiraAddAttachmentContract } from '@/lib/api/contracts/selectors/jira' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -44,6 +45,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { (await getJiraCloudId(validatedData.domain, validatedData.accessToken)) const formData = new FormData() + // Every attachment lands in the same multipart body, so the ceiling covers the + // set rather than each file on its own. + let remainingBytes = MAX_BUFFERED_TRANSFER_BYTES for (const file of userFiles) { const denied = await assertToolFileAccess(file.key, authResult.userId, requestId, logger) @@ -51,7 +55,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let buffer: Buffer let downloadedContentType = '' try { - const result = await downloadServableFileFromStorage(file, requestId, logger) + const result = await downloadServableFileFromStorage(file, requestId, logger, { + maxBytes: remainingBytes, + }) buffer = result.buffer downloadedContentType = result.contentType } catch (error) { @@ -59,6 +65,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (notReady) return notReady throw error } + remainingBytes -= buffer.length const blob = new Blob([new Uint8Array(buffer)], { type: downloadedContentType || file.type || 'application/octet-stream', }) diff --git a/apps/sim/app/api/tools/jupyter/upload/route.ts b/apps/sim/app/api/tools/jupyter/upload/route.ts index ff3656cce44..0c4efff228a 100644 --- a/apps/sim/app/api/tools/jupyter/upload/route.ts +++ b/apps/sim/app/api/tools/jupyter/upload/route.ts @@ -10,7 +10,9 @@ import { validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -58,14 +60,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = result.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } fileName = data.fileName || userFile.name diff --git a/apps/sim/app/api/tools/linq/upload/route.ts b/apps/sim/app/api/tools/linq/upload/route.ts index 6b0004df4e2..deb8ffb79d6 100644 --- a/apps/sim/app/api/tools/linq/upload/route.ts +++ b/apps/sim/app/api/tools/linq/upload/route.ts @@ -5,6 +5,7 @@ import { linqUploadAttachmentContract } from '@/lib/api/contracts/tools/communic import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -19,6 +20,16 @@ const logger = createLogger('LinqUploadAttachmentAPI') /** Linq pre-upload caps attachments at 100MB. */ const MAX_SIZE_BYTES = 100 * 1024 * 1024 +function fileTooLargeError(sizeBytes: number): NextResponse { + return NextResponse.json( + { + success: false, + error: `File exceeds Linq's 100MB attachment limit (${(sizeBytes / (1024 * 1024)).toFixed(2)}MB)`, + }, + { status: 400 } + ) +} + /** * Upload a file to Linq as a reusable attachment. * @@ -61,12 +72,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied let resolvedContentTypeFromStorage: string try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_SIZE_BYTES, + }) buffer = resolved.buffer resolvedContentTypeFromStorage = resolved.contentType } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) + return fileTooLargeError(error.observedBytes ?? userFile.size) logger.error(`[${requestId}] Failed to download Linq attachment file:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, @@ -93,13 +108,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ success: false, error: 'File is empty' }, { status: 400 }) } if (sizeBytes > MAX_SIZE_BYTES) { - return NextResponse.json( - { - success: false, - error: `File exceeds Linq's 100MB attachment limit (${(sizeBytes / (1024 * 1024)).toFixed(2)}MB)`, - }, - { status: 400 } - ) + return fileTooLargeError(sizeBytes) } logger.info(`[${requestId}] Registering Linq attachment`, { diff --git a/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts b/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts index eb78c96325f..826c547df96 100644 --- a/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts +++ b/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts @@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -20,6 +21,17 @@ const logger = createLogger('DataverseUploadFileAPI') /** Dataverse Web API's absolute ceiling for a single-request (non-chunked) file column upload. */ const DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES = 128 * 1024 * 1024 +function uploadTooLargeError(observedBytes: number): NextResponse { + const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `File size (${sizeMB}MB) exceeds Dataverse's 128MB limit for single-request file column uploads. Split the file and use chunked upload instead.`, + }, + { status: 400 } + ) +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() @@ -77,11 +89,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES, + }) fileBuffer = servable.buffer } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) + return uploadTooLargeError(error.observedBytes ?? userFile.size) logger.error(`[${requestId}] Failed to download file from storage:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, @@ -100,13 +116,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (fileBuffer.length > DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES) { const sizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2) logger.warn(`[${requestId}] File too large for single-request upload: ${sizeMB}MB`) - return NextResponse.json( - { - success: false, - error: `File size (${sizeMB}MB) exceeds Dataverse's 128MB limit for single-request file column uploads. Split the file and use chunked upload instead.`, - }, - { status: 400 } - ) + return uploadTooLargeError(fileBuffer.length) } const baseUrl = getDataverseBaseUrl(validatedData.environmentUrl) diff --git a/apps/sim/app/api/tools/mistral/parse/route.ts b/apps/sim/app/api/tools/mistral/parse/route.ts index 5805142915d..e994e718ac3 100644 --- a/apps/sim/app/api/tools/mistral/parse/route.ts +++ b/apps/sim/app/api/tools/mistral/parse/route.ts @@ -15,6 +15,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { extractStorageKey, isInternalFileUrl, @@ -157,7 +158,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const { buffer, contentType } = await downloadServableFileFromStorage( userFile, requestId, - logger + logger, + { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + } ) base64 = buffer.toString('base64') if (contentType && contentType !== 'application/octet-stream') { diff --git a/apps/sim/app/api/tools/onedrive/upload/route.ts b/apps/sim/app/api/tools/onedrive/upload/route.ts index 2c94986de9f..2e7ff49837d 100644 --- a/apps/sim/app/api/tools/onedrive/upload/route.ts +++ b/apps/sim/app/api/tools/onedrive/upload/route.ts @@ -8,6 +8,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getExtensionFromMimeType, @@ -24,6 +25,20 @@ const logger = createLogger('OneDriveUploadAPI') const MICROSOFT_GRAPH_BASE = 'https://graph.microsoft.com/v1.0' +/** Microsoft Graph's ceiling for a simple (non-chunked) drive-item upload. */ +const MAX_SIMPLE_UPLOAD_BYTES = 250 * 1024 * 1024 + +function fileTooLargeError(observedBytes: number): NextResponse { + const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `File size (${sizeMB}MB) exceeds OneDrive's limit of 250MB for simple uploads. Use chunked upload for larger files.`, + }, + { status: 400 } + ) +} + /** Microsoft Graph DriveItem response */ interface OneDriveFileData { id: string @@ -115,12 +130,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_SIMPLE_UPLOAD_BYTES, + }) fileBuffer = result.buffer mimeType = result.contentType || userFile.type || 'application/octet-stream' } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + return fileTooLargeError(error.observedBytes ?? userFile.size) + } logger.error(`[${requestId}] Failed to download file from storage:`, error) return NextResponse.json( { @@ -132,17 +152,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } } - const maxSize = 250 * 1024 * 1024 - if (fileBuffer.length > maxSize) { - const sizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2) - logger.warn(`[${requestId}] File too large: ${sizeMB}MB`) - return NextResponse.json( - { - success: false, - error: `File size (${sizeMB}MB) exceeds OneDrive's limit of 250MB for simple uploads. Use chunked upload for larger files.`, - }, - { status: 400 } + if (fileBuffer.length > MAX_SIMPLE_UPLOAD_BYTES) { + logger.warn( + `[${requestId}] File too large: ${(fileBuffer.length / (1024 * 1024)).toFixed(2)}MB` ) + return fileTooLargeError(fileBuffer.length) } let fileName = validatedData.fileName diff --git a/apps/sim/app/api/tools/outlook/draft/route.ts b/apps/sim/app/api/tools/outlook/draft/route.ts index 979634ffcf3..52b114bed2a 100644 --- a/apps/sim/app/api/tools/outlook/draft/route.ts +++ b/apps/sim/app/api/tools/outlook/draft/route.ts @@ -5,9 +5,10 @@ import { outlookDraftContract } from '@/lib/api/contracts/tools/microsoft' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -110,17 +111,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -131,18 +138,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request`, - }, - { status: 400 } - ) - } - const attachmentObjects = attachments.map((file, i) => ({ '@odata.type': '#microsoft.graph.fileAttachment', name: file.name, diff --git a/apps/sim/app/api/tools/outlook/send/route.ts b/apps/sim/app/api/tools/outlook/send/route.ts index 7a59f44410a..82dd3e4474a 100644 --- a/apps/sim/app/api/tools/outlook/send/route.ts +++ b/apps/sim/app/api/tools/outlook/send/route.ts @@ -5,9 +5,10 @@ import { outlookSendContract } from '@/lib/api/contracts/tools/microsoft' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -110,17 +111,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -131,18 +138,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request`, - }, - { status: 400 } - ) - } - const attachmentObjects = attachments.map((file, i) => ({ '@odata.type': '#microsoft.graph.fileAttachment', name: file.name, diff --git a/apps/sim/app/api/tools/persona/import-accounts/route.ts b/apps/sim/app/api/tools/persona/import-accounts/route.ts index 0844b11c16a..ca59a41b053 100644 --- a/apps/sim/app/api/tools/persona/import-accounts/route.ts +++ b/apps/sim/app/api/tools/persona/import-accounts/route.ts @@ -5,7 +5,9 @@ import { personaImportAccountsContract } from '@/lib/api/contracts/tools/persona import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -58,7 +60,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let buffer: Buffer try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) buffer = resolved.buffer } catch (error) { const notReady = docNotReadyResponse(error) @@ -66,7 +70,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.error(`[${requestId}] Failed to download Persona import file:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/quiver/image-to-svg/route.ts b/apps/sim/app/api/tools/quiver/image-to-svg/route.ts index 8149a8b5dd0..faf79554834 100644 --- a/apps/sim/app/api/tools/quiver/image-to-svg/route.ts +++ b/apps/sim/app/api/tools/quiver/image-to-svg/route.ts @@ -11,6 +11,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -81,7 +82,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger) + const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) apiImage = { base64: buffer.toString('base64') } } else { return NextResponse.json( @@ -114,7 +117,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger) + const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) apiImage = { base64: buffer.toString('base64') } } else { return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) diff --git a/apps/sim/app/api/tools/quiver/text-to-svg/route.ts b/apps/sim/app/api/tools/quiver/text-to-svg/route.ts index fb6df66b7e5..eb40c76997d 100644 --- a/apps/sim/app/api/tools/quiver/text-to-svg/route.ts +++ b/apps/sim/app/api/tools/quiver/text-to-svg/route.ts @@ -11,6 +11,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -59,6 +60,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const apiReferences: Array<{ url: string } | { base64: string }> = [] + // Every reference is buffered and base64'd before the list is sliced to 4, so the + // budget has to span the whole loop rather than bound each file on its own. + let referenceBudget = MAX_BUFFERED_TRANSFER_BYTES if (data.references) { const rawRefs = Array.isArray(data.references) ? data.references : [data.references] @@ -86,7 +90,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger) + const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { + maxBytes: referenceBudget, + }) + referenceBudget -= buffer.length apiReferences.push({ base64: buffer.toString('base64') }) } } @@ -107,7 +114,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger) + const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { + maxBytes: referenceBudget, + }) + referenceBudget -= buffer.length apiReferences.push({ base64: buffer.toString('base64') }) } } diff --git a/apps/sim/app/api/tools/s3/put-object/route.ts b/apps/sim/app/api/tools/s3/put-object/route.ts index 2019de2b4d4..026713914fb 100644 --- a/apps/sim/app/api/tools/s3/put-object/route.ts +++ b/apps/sim/app/api/tools/s3/put-object/route.ts @@ -6,7 +6,9 @@ import { awsS3PutObjectContract } from '@/lib/api/contracts/tools/aws/s3-put-obj import { parseToolRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -84,7 +86,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let downloadedContentType = '' try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) uploadBody = result.buffer downloadedContentType = result.contentType } catch (error) { @@ -92,7 +96,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (notReady) return notReady return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/sendgrid/send-mail/route.ts b/apps/sim/app/api/tools/sendgrid/send-mail/route.ts index d9b2b97876a..f0bc1b3e660 100644 --- a/apps/sim/app/api/tools/sendgrid/send-mail/route.ts +++ b/apps/sim/app/api/tools/sendgrid/send-mail/route.ts @@ -5,9 +5,10 @@ import { sendGridSendMailContract } from '@/lib/api/contracts/tools/communicatio import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -15,6 +16,9 @@ export const dynamic = 'force-dynamic' const logger = createLogger('SendGridSendMailAPI') +/** SendGrid rejects a message whose total attachment payload exceeds 30MB. */ +const MAX_ATTACHMENT_TOTAL_BYTES = 30 * 1024 * 1024 + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() @@ -109,17 +113,26 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - userFiles.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(userFiles, requestId, logger, { + totalMaxBytes: MAX_ATTACHMENT_TOTAL_BYTES, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ( + (error.observedBytes ?? MAX_ATTACHMENT_TOTAL_BYTES) / + (1024 * 1024) + ).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds SendGrid's limit of 30MB`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -130,19 +143,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - const maxSize = 30 * 1024 * 1024 - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds SendGrid's limit of 30MB`, - }, - { status: 400 } - ) - } - const sendGridAttachments = userFiles.map((file, i) => ({ content: resolved[i].buffer.toString('base64'), filename: file.name, diff --git a/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts b/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts index 78dcdb0883b..532b3712572 100644 --- a/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts +++ b/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts @@ -6,7 +6,9 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -56,7 +58,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let fileBuffer: Buffer let resolvedContentType: string try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = servable.buffer resolvedContentType = servable.contentType } catch (error) { @@ -65,7 +69,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.error(`[${requestId}] Failed to download file from storage:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/sftp/upload/route.ts b/apps/sim/app/api/tools/sftp/upload/route.ts index 09029d5cd81..bca922e7d20 100644 --- a/apps/sim/app/api/tools/sftp/upload/route.ts +++ b/apps/sim/app/api/tools/sftp/upload/route.ts @@ -5,6 +5,7 @@ import { sftpUploadContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -109,16 +110,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.info( `[${requestId}] Downloading file for upload: ${file.name} (${file.size} bytes)` ) - const { buffer } = await downloadServableFileFromStorage(file, requestId, logger) + const { buffer } = await downloadServableFileFromStorage(file, requestId, logger, { + maxBytes: maxSize - resolvedTotal, + }) resolvedTotal += buffer.length - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { success: false, error: `Total file size (${sizeMB}MB) exceeds limit of 100MB` }, - { status: 400 } - ) - } const safeFileName = sanitizeFileName(file.name) const fullRemotePath = remotePath.endsWith('/') @@ -155,6 +151,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const observed = resolvedTotal + (error.observedBytes ?? file.size) + const sizeMB = (observed / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { success: false, error: `Total file size (${sizeMB}MB) exceeds limit of 100MB` }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to upload file ${file.name}:`, error) throw new Error( `Failed to upload file "${file.name}": ${getErrorMessage(error, 'Unknown error')}` diff --git a/apps/sim/app/api/tools/sharepoint/upload/route.ts b/apps/sim/app/api/tools/sharepoint/upload/route.ts index 55bb4eec935..b29a51fd387 100644 --- a/apps/sim/app/api/tools/sharepoint/upload/route.ts +++ b/apps/sim/app/api/tools/sharepoint/upload/route.ts @@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -88,34 +89,39 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied logger.info(`[${requestId}] Uploading file: ${userFile.name}`) - let buffer: Buffer - let downloadedContentType = '' - try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger) - buffer = result.buffer - downloadedContentType = result.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - throw error - } - const fileName = validatedData.fileName || userFile.name const folderPath = validatedData.folderPath?.trim() || '' - const fileSizeMB = buffer.length / (1024 * 1024) - - if (buffer.length > MAX_SHAREPOINT_UPLOAD_BYTES) { + const skipOversized = (size: number) => { logger.warn( - `[${requestId}] File ${fileName} is ${fileSizeMB.toFixed(2)}MB, exceeds 250MB limit` + `[${requestId}] File ${fileName} is ${(size / (1024 * 1024)).toFixed(2)}MB, exceeds 250MB limit` ) skippedFiles.push({ name: fileName, - size: buffer.length, + size, limit: MAX_SHAREPOINT_UPLOAD_BYTES, reason: 'File exceeds the 250 MB Microsoft Graph small upload limit', }) - continue + } + + let buffer: Buffer + let downloadedContentType = '' + try { + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_SHAREPOINT_UPLOAD_BYTES, + }) + buffer = result.buffer + downloadedContentType = result.contentType + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + // An oversized file is skipped rather than failing the whole batch, exactly as + // it was when the size was only discovered after the download. + if (isPayloadSizeLimitError(error)) { + skipOversized(error.observedBytes ?? userFile.size) + continue + } + throw error } let uploadPath = '' diff --git a/apps/sim/app/api/tools/slack/utils.ts b/apps/sim/app/api/tools/slack/utils.ts index b3ff2205806..d40b3a81a66 100644 --- a/apps/sim/app/api/tools/slack/utils.ts +++ b/apps/sim/app/api/tools/slack/utils.ts @@ -1,5 +1,6 @@ import type { Logger } from '@sim/logger' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { FileAccessDeniedError, verifyFileAccess } from '@/app/api/files/authorization' @@ -80,6 +81,9 @@ async function uploadFilesToSlack( const userFiles = processFilesToUserFiles(files, requestId, logger) const uploadedFileIds: string[] = [] const uploadedFiles: ToolFileData[] = [] + // One share can carry several files, so the ceiling spans the set: each file may + // only use what its predecessors left. + let remainingBytes = MAX_BUFFERED_TRANSFER_BYTES for (const userFile of userFiles) { logger.info(`[${requestId}] Uploading file: ${userFile.name}`) @@ -92,8 +96,10 @@ async function uploadFilesToSlack( const { buffer, contentType } = await downloadServableFileFromStorage( userFile, requestId, - logger + logger, + { maxBytes: remainingBytes } ) + remainingBytes -= buffer.length const getUrlResponse = await fetch('https://slack.com/api/files.getUploadURLExternal', { method: 'POST', diff --git a/apps/sim/app/api/tools/smtp/send/route.ts b/apps/sim/app/api/tools/smtp/send/route.ts index 920f6925784..83a48f3ce2b 100644 --- a/apps/sim/app/api/tools/smtp/send/route.ts +++ b/apps/sim/app/api/tools/smtp/send/route.ts @@ -7,10 +7,11 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getSmtpEhloName } from '@/lib/messaging/email/ehlo' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -132,17 +133,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolved: Array<{ buffer: Buffer; contentType: string }> try { - resolved = await Promise.all( - attachments.map(async (file) => { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - return await downloadServableFileFromStorage(file, requestId, logger) - }) - ) + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: maxSize, + label: 'Total attachment size', + }) } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds SMTP limit of 25MB`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download an attachment:`, error) return NextResponse.json( { @@ -153,18 +160,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) - if (resolvedTotal > maxSize) { - const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds SMTP limit of 25MB`, - }, - { status: 400 } - ) - } - const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, content: resolved[i].buffer, diff --git a/apps/sim/app/api/tools/square/catalog-image/route.ts b/apps/sim/app/api/tools/square/catalog-image/route.ts index 622c5c1c7a2..229e70deff3 100644 --- a/apps/sim/app/api/tools/square/catalog-image/route.ts +++ b/apps/sim/app/api/tools/square/catalog-image/route.ts @@ -7,6 +7,7 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -52,7 +53,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) const fileName = validatedData.fileName || userFile.name const mimeType = userFile.type || 'application/octet-stream' diff --git a/apps/sim/app/api/tools/stt/route.ts b/apps/sim/app/api/tools/stt/route.ts index f8f468ea11f..3ff3cb3eaaa 100644 --- a/apps/sim/app/api/tools/stt/route.ts +++ b/apps/sim/app/api/tools/stt/route.ts @@ -117,7 +117,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - audioBuffer = await downloadFileFromStorage(file, requestId, logger) + audioBuffer = await downloadFileFromStorage(file, requestId, logger, { + maxBytes: MAX_FILE_SIZE, + }) audioFileName = file.name // file.type may be missing if the file came from a block that doesn't preserve it // Infer from filename extension as fallback @@ -143,7 +145,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - audioBuffer = await downloadFileFromStorage(file, requestId, logger) + audioBuffer = await downloadFileFromStorage(file, requestId, logger, { + maxBytes: MAX_FILE_SIZE, + }) audioFileName = file.name const ext = file.name.split('.').pop()?.toLowerCase() || '' diff --git a/apps/sim/app/api/tools/supabase/storage-upload/route.ts b/apps/sim/app/api/tools/supabase/storage-upload/route.ts index 7f02f7791ea..ac9d6bda605 100644 --- a/apps/sim/app/api/tools/supabase/storage-upload/route.ts +++ b/apps/sim/app/api/tools/supabase/storage-upload/route.ts @@ -6,7 +6,9 @@ import { parseToolRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateSupabaseProjectId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' @@ -152,7 +154,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let buffer: Buffer let resolvedContentType: string try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) buffer = resolved.buffer resolvedContentType = resolved.contentType } catch (error) { @@ -161,7 +165,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.error(`[${requestId}] Failed to download file for Supabase upload:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/app/api/tools/telegram/send-document/route.ts b/apps/sim/app/api/tools/telegram/send-document/route.ts index f454717e290..6d4cf533d64 100644 --- a/apps/sim/app/api/tools/telegram/send-document/route.ts +++ b/apps/sim/app/api/tools/telegram/send-document/route.ts @@ -5,6 +5,7 @@ import { telegramSendDocumentContract } from '@/lib/api/contracts/tools/communic import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -97,12 +98,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let buffer: Buffer let contentType: string try { - const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger) + const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: maxSize, + }) buffer = downloaded.buffer contentType = downloaded.contentType } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? userFile.size) / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `The following files exceed Telegram's 50MB limit: ${userFile.name} (${sizeMB}MB)`, + }, + { status: 400 } + ) + } logger.error(`[${requestId}] Failed to download document ${userFile.name}:`, error) return NextResponse.json( { diff --git a/apps/sim/app/api/tools/textract/shared.ts b/apps/sim/app/api/tools/textract/shared.ts index b6b570cb506..0150c8f0c99 100644 --- a/apps/sim/app/api/tools/textract/shared.ts +++ b/apps/sim/app/api/tools/textract/shared.ts @@ -12,6 +12,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import type { RawFileInput } from '@/lib/uploads/utils/file-utils' import { extractStorageKey, @@ -160,7 +161,10 @@ export async function resolveDocumentInput( const { buffer, contentType } = await downloadServableFileFromStorage( userFile, requestId, - logger + logger, + { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + } ) const resolvedContentType = contentType || userFile.type || 'application/octet-stream' diff --git a/apps/sim/app/api/tools/uptimerobot/server-utils.ts b/apps/sim/app/api/tools/uptimerobot/server-utils.ts index d7a1ef9ea8c..9c3c7d2f316 100644 --- a/apps/sim/app/api/tools/uptimerobot/server-utils.ts +++ b/apps/sim/app/api/tools/uptimerobot/server-utils.ts @@ -1,5 +1,6 @@ import type { Logger } from '@sim/logger' import { NextResponse } from 'next/server' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { assertToolFileAccess } from '@/app/api/files/authorization' @@ -47,7 +48,14 @@ async function appendPspImage( const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) if (denied) return denied - const { buffer, contentType } = await downloadServableFileFromStorage(userFile, requestId, logger) + const { buffer, contentType } = await downloadServableFileFromStorage( + userFile, + requestId, + logger, + { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + } + ) const mimeType = contentType || userFile.type || 'application/octet-stream' form.append(field, new Blob([new Uint8Array(buffer)], { type: mimeType }), userFile.name) return null diff --git a/apps/sim/app/api/tools/vanta/upload/route.ts b/apps/sim/app/api/tools/vanta/upload/route.ts index 81785b515c8..932b68658c7 100644 --- a/apps/sim/app/api/tools/vanta/upload/route.ts +++ b/apps/sim/app/api/tools/vanta/upload/route.ts @@ -5,6 +5,7 @@ import { vantaUploadContract } from '@/lib/api/contracts/tools/vanta' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -72,7 +73,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_UPLOAD_SIZE_BYTES, + }) fileBuffer = resolved.buffer fileName = params.fileName || userFile.name mimeType = @@ -80,6 +83,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { const notReady = docNotReadyResponse(error) if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + return uploadSizeError(error.observedBytes ?? userFile.size) + } logger.error(`[${requestId}] Failed to download Vanta upload file`, { error: getErrorMessage(error), }) diff --git a/apps/sim/app/api/tools/vision/analyze/route.ts b/apps/sim/app/api/tools/vision/analyze/route.ts index fb74630943f..74a0cb0560f 100644 --- a/apps/sim/app/api/tools/vision/analyze/route.ts +++ b/apps/sim/app/api/tools/vision/analyze/route.ts @@ -16,6 +16,7 @@ import { isModelSafeWorkspaceFileKey, MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { extractStorageKey, isInternalFileUrl, @@ -125,7 +126,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 400 } ) } - const buffer = await downloadFileFromStorage(userFile, requestId, logger) + // The three providers this route serves disagree too much for a single + // route-wide image limit to be right (Anthropic: 10MB base64 per image; + // OpenAI: 512MB total request payload), and picking the lowest would reject + // images the others accept. So bound the buffer we hold and let each provider + // reject what it will not take, with its own message. + const buffer = await downloadFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) base64 = buffer.toString('base64') bufferLength = buffer.length } diff --git a/apps/sim/app/api/tools/wordpress/upload/route.ts b/apps/sim/app/api/tools/wordpress/upload/route.ts index aac9e9e3354..b7b9ba8c2a1 100644 --- a/apps/sim/app/api/tools/wordpress/upload/route.ts +++ b/apps/sim/app/api/tools/wordpress/upload/route.ts @@ -5,7 +5,9 @@ import { wordpressUploadContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { getFileExtension, getMimeTypeFromExtension, @@ -93,7 +95,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let resolvedContentType: string try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) fileBuffer = servable.buffer resolvedContentType = servable.contentType } catch (error) { @@ -105,7 +109,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { success: false, error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } diff --git a/apps/sim/lib/collab-doc/persist.ts b/apps/sim/lib/collab-doc/persist.ts index 2e8e9692ff4..6d8f387ce6d 100644 --- a/apps/sim/lib/collab-doc/persist.ts +++ b/apps/sim/lib/collab-doc/persist.ts @@ -7,6 +7,7 @@ import { getWorkspaceFile, updateWorkspaceFileContent, } from '@/lib/uploads/contexts/workspace' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { collabDocStateSourceHash, hashMarkdown, saveCollabDocState } from './collab-state' import { canonicalizeYDoc, yDocToFileMarkdown } from './converter' @@ -94,7 +95,11 @@ export async function persistFileDoc( // nothing to clobber, so this reports the file's CURRENT durable version rather than conflicting on a // stale `expectedVersion`: it resynchronizes the relay's If-Match token instead of stranding it. if (record.size === markdownBuffer.length) { - const current = await fetchWorkspaceFileBuffer(record).catch(() => null) + // A byte-for-byte equality check: anything longer than what we are comparing against + // cannot match, so the buffer we are about to compare is itself the ceiling. + const current = await fetchWorkspaceFileBuffer(record, { + maxBytes: markdownBuffer.length, + }).catch(() => null) if (current?.equals(markdownBuffer)) { // Still refresh the cached snapshot: the markdown is unchanged (so its `sourceHash` tag stays // valid) but the doc state may have just been canonicalized, and a cold open should seed from @@ -191,7 +196,9 @@ async function recoverFromVersionConflict( try { const current = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) if (!current) return { status: 'missing' } - const durable = await fetchWorkspaceFileBuffer(current) + const durable = await fetchWorkspaceFileBuffer(current, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) if (hashMarkdown(durable) !== (await collabDocStateSourceHash(fileId))) return conflict() logger.info( `Persist token for file ${fileId} was stale, not the file; re-syncing and writing the projection` diff --git a/apps/sim/lib/copilot/tools/handlers/materialize-file.ts b/apps/sim/lib/copilot/tools/handlers/materialize-file.ts index 26f87478e51..8c13a9bbef8 100644 --- a/apps/sim/lib/copilot/tools/handlers/materialize-file.ts +++ b/apps/sim/lib/copilot/tools/handlers/materialize-file.ts @@ -39,6 +39,7 @@ import { hasCloudStorage, headObject } from '@/lib/uploads/core/storage-service' import { toLegacyWorkspaceFileSize } from '@/lib/uploads/shared/types' import { isArchiveFileName } from '@/lib/uploads/utils/file-utils' import { parseWorkflowJson } from '@/lib/workflows/operations/import-export' +import { MAX_IMPORT_BODY_BYTES } from '@/lib/workflows/operations/import-workflow' import { saveWorkflowToNormalizedTables } from '@/lib/workflows/persistence/utils' import { deduplicateWorkflowName } from '@/lib/workflows/utils' import { admitCreateWorkspaceFile } from '@/lib/workspace-files/application/create-workspace-file' @@ -260,7 +261,11 @@ async function executeImport( } } - const buffer = await fetchWorkspaceFileBuffer(toFileRecord(row)) + // The bytes are headed straight for `parseWorkflowJson`, so the import body ceiling is + // the real limit here — a larger file could not be imported even if it were read. + const buffer = await fetchWorkspaceFileBuffer(toFileRecord(row), { + maxBytes: MAX_IMPORT_BODY_BYTES, + }) const content = buffer.toString('utf-8') let parsed: unknown diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index b4a4756ff65..84a06683677 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -1659,7 +1659,7 @@ export async function getWorkspaceFile( */ export async function fetchServableWorkspaceFileBuffer( fileRecord: WorkspaceFileRecord, - options: { maxBytes?: number; signal?: AbortSignal; requestId?: string } = {} + options: { maxBytes: number; signal?: AbortSignal; requestId?: string } ): Promise<{ buffer: Buffer; contentType: string }> { const { downloadServableFileFromStorage } = await import('@/lib/uploads/utils/file-utils.server') @@ -1685,7 +1685,7 @@ export async function fetchServableWorkspaceFileBuffer( */ export async function fetchWorkspaceFileBuffer( fileRecord: WorkspaceFileRecord, - options: { maxBytes?: number } = {} + options: { maxBytes: number } ): Promise { logger.info(`Downloading workspace file: ${fileRecord.name}`) diff --git a/apps/sim/lib/uploads/shared/types.ts b/apps/sim/lib/uploads/shared/types.ts index e0fb1ffcb94..7d9ebc3adca 100644 --- a/apps/sim/lib/uploads/shared/types.ts +++ b/apps/sim/lib/uploads/shared/types.ts @@ -25,6 +25,25 @@ export const MAX_WORKSPACE_FORMDATA_FILE_SIZE = 100 * 1024 * 1024 /** Maximum size accepted by the knowledge-document parsing pipeline. */ export const MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE = 100 * 1024 * 1024 +/** + * Default ceiling for a read that holds the whole file resident as one `Buffer`. + * + * Workspace files are admitted at {@link MAX_WORKSPACE_FILE_SIZE} (5 GB) because they + * are streamed straight to object storage and never sit in the app process. A tool + * that pulls one back to hand it to a third party does not stream — it buffers, then + * usually copies again (base64, `Blob`, multipart), so peak resident memory is a + * multiple of the file. Sharing one ceiling keeps that multiple bounded no matter how + * many blocks run concurrently. + * + * 100 MB is the value this codebase already converged on for buffered work + * ({@link MAX_WORKSPACE_FORMDATA_FILE_SIZE}, `MAX_ARCHIVE_BYTES`, the 100 MB + * `maxResponseBytes` on the STT URL branch, and the ClickUp/Vanta/Daytona/Linq/SFTP + * upload routes). Use a destination's own documented limit instead whenever it is + * lower — failing here beats a slow round trip to a provider that will reject it. + * Genuinely large transfers belong on `downloadFileStream`, not on a bigger ceiling. + */ +export const MAX_BUFFERED_TRANSFER_BYTES = 100 * 1024 * 1024 + /** * Rejection wording shared by every surface that admits a knowledge document. * diff --git a/apps/sim/lib/uploads/utils/file-utils.server.test.ts b/apps/sim/lib/uploads/utils/file-utils.server.test.ts index 6cae5e88cfa..f93ce28d490 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.test.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.test.ts @@ -33,9 +33,12 @@ vi.mock('@/app/api/files/authorization', () => ({ })) import { createLogger } from '@sim/logger' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { downloadFileFromStorage, downloadServableFileFromStorage, + downloadServableFilesWithinBudget, } from '@/lib/uploads/utils/file-utils.server' import type { UserFile } from '@/executor/types' @@ -61,7 +64,9 @@ describe('downloadFileFromStorage context derivation', () => { context: 'og-images', } - await downloadFileFromStorage(userFile, 'req-1', createLogger('test')) + await downloadFileFromStorage(userFile, 'req-1', createLogger('test'), { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) expect(mockDownloadFile).toHaveBeenCalledTimes(1) expect(mockDownloadFile).toHaveBeenCalledWith( @@ -83,6 +88,7 @@ describe('downloadFileFromStorage context derivation', () => { const filePrincipal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } await downloadServableFileFromStorage(userFile, 'req-1', createLogger('test'), { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, filePrincipal, }) @@ -91,3 +97,112 @@ describe('downloadFileFromStorage context derivation', () => { ) }) }) + +describe('downloadFileFromStorage size ceiling', () => { + const logger = createLogger('test') + const fileOfSize = (size: number): UserFile => ({ + id: 'f1', + name: 'clip.wav', + url: '', + size, + type: 'audio/wav', + key: 'workspace/ws-1/1700000000000-abc1234-clip.wav', + }) + + beforeEach(() => { + vi.clearAllMocks() + mockParseWorkspaceFileKey.mockReturnValue(null) + }) + + it('rejects on the declared size before moving any bytes', async () => { + await expect( + downloadFileFromStorage(fileOfSize(2048), 'req-1', logger, { maxBytes: 1024 }) + ).rejects.toThrow(PayloadSizeLimitError) + + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('rejects on the delivered bytes when the declared size understated them', async () => { + mockDownloadFile.mockResolvedValue(Buffer.alloc(2048)) + + await expect( + downloadFileFromStorage(fileOfSize(1), 'req-1', logger, { maxBytes: 1024 }) + ).rejects.toThrow(PayloadSizeLimitError) + }) + + it('forwards the ceiling to the storage layer so a provider can stop mid-stream', async () => { + mockDownloadFile.mockResolvedValue(Buffer.alloc(512)) + + await downloadFileFromStorage(fileOfSize(512), 'req-1', logger, { maxBytes: 1024 }) + + expect(mockDownloadFile).toHaveBeenCalledWith(expect.objectContaining({ maxBytes: 1024 })) + }) +}) + +describe('downloadServableFilesWithinBudget', () => { + const logger = createLogger('test') + const fileOfSize = (name: string, size: number): UserFile => ({ + id: name, + name, + url: '', + size, + type: 'application/octet-stream', + key: `workspace/ws-1/1700000000000-abc1234-${name}`, + }) + + beforeEach(() => { + vi.clearAllMocks() + mockParseWorkspaceFileKey.mockReturnValue(null) + mockDownloadFile.mockImplementation(async ({ key }) => + Buffer.alloc(key.endsWith('big.bin') ? 900 : 400) + ) + }) + + it('spends the budget across the list rather than per file', async () => { + const resolved = await downloadServableFilesWithinBudget( + [fileOfSize('a.bin', 400), fileOfSize('b.bin', 400)], + 'req-1', + logger, + { totalMaxBytes: 1000, label: 'Total attachment size' } + ) + + expect(resolved.map((r) => r.buffer.length)).toEqual([400, 400]) + // The second file was only offered what the first left behind. + expect(mockDownloadFile).toHaveBeenNthCalledWith(2, expect.objectContaining({ maxBytes: 600 })) + }) + + it('rejects the combined size even when every file is individually under the limit', async () => { + const failure = await downloadServableFilesWithinBudget( + [fileOfSize('a.bin', 400), fileOfSize('b.bin', 400), fileOfSize('c.bin', 400)], + 'req-1', + logger, + { totalMaxBytes: 1000, label: 'Total attachment size' } + ).catch((error) => error) + + // Restated in the caller's terms: the whole set against the whole budget, not the + // third file against the 200 bytes the first two happened to leave. + expect(failure).toBeInstanceOf(PayloadSizeLimitError) + expect(failure).toMatchObject({ + label: 'Total attachment size', + maxBytes: 1000, + observedBytes: 1200, + }) + + // The third file's declared size already exceeds what the first two left, so it is + // refused without fetching its bytes — the whole set is never resident at once. + expect(mockDownloadFile).toHaveBeenCalledTimes(2) + }) + + it('refuses the next file on its declared size once the budget is spent', async () => { + await expect( + downloadServableFilesWithinBudget( + [fileOfSize('big.bin', 900), fileOfSize('a.bin', 400)], + 'req-1', + logger, + { totalMaxBytes: 1000, label: 'Total attachment size' } + ) + ).rejects.toThrow(PayloadSizeLimitError) + + expect(mockDownloadFile).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index e20b1900b49..579ad67f65d 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -9,6 +9,8 @@ import { import { assertKnownSizeWithinLimit, consumeOrCancelBody, + isPayloadSizeLimitError, + PayloadSizeLimitError, readResponseToBufferWithLimit, } from '@/lib/core/utils/stream-limits' import { StorageService } from '@/lib/uploads' @@ -335,29 +337,38 @@ export async function resolveInternalFileUrl( } /** - * Downloads a file from storage (execution or regular) + * Downloads a file from storage (execution or regular) into a single resident buffer. + * + * `maxBytes` is required, not optional. Workspace files are admitted at 5 GB, so a + * caller that forgets a ceiling inherits "unbounded" and can allocate gigabytes inside + * the shared app process. Making the parameter mandatory means a new call site has to + * name its limit — see {@link MAX_BUFFERED_TRANSFER_BYTES} for the default, and prefer + * the destination's own documented limit whenever it is lower. The size is checked + * twice: against the declared size before any bytes move, and against the delivered + * buffer, because the declared size comes from the caller and may be a lie. + * * @param userFile - UserFile object * @param requestId - Request ID for logging * @param logger - Logger instance + * @param options.maxBytes - Hard ceiling; throws `PayloadSizeLimitError` when exceeded * @returns Buffer containing file data */ export async function downloadFileFromStorage( userFile: UserFile, requestId: string, logger: Logger, - options: { maxBytes?: number } = {} + options: { maxBytes: number } ): Promise { + const { maxBytes } = options let buffer: Buffer - if (options.maxBytes !== undefined && userFile.size > options.maxBytes) { - assertKnownSizeWithinLimit(userFile.size, options.maxBytes, 'storage file download') - } + assertKnownSizeWithinLimit(userFile.size, maxBytes, 'storage file download') if (isExecutionFile(userFile)) { logger.info(`[${requestId}] Downloading from execution storage: ${userFile.key}`) const { downloadExecutionFile } = await import( '@/lib/uploads/contexts/execution/execution-file-manager' ) - buffer = await downloadExecutionFile(userFile, { maxBytes: options.maxBytes }) + buffer = await downloadExecutionFile(userFile, { maxBytes }) } else if (userFile.key) { const context = resolveTrustedFileContext(userFile.key, userFile.context) logger.info(`[${requestId}] Downloading from ${context} storage: ${userFile.key}`) @@ -366,15 +377,13 @@ export async function downloadFileFromStorage( buffer = await downloadFile({ key: userFile.key, context, - maxBytes: options.maxBytes, + maxBytes, }) } else { throw new Error('File has no key - cannot download') } - if (options.maxBytes !== undefined) { - assertKnownSizeWithinLimit(buffer.length, options.maxBytes, 'storage file download') - } + assertKnownSizeWithinLimit(buffer.length, maxBytes, 'storage file download') return buffer } @@ -404,17 +413,21 @@ export interface ServableFile { * * Throws `DocCompileUserError` when a generated doc's artifact is not ready (still * compiling) — callers should surface a retryable error rather than attach source. + * + * `maxBytes` is required for the reason given on {@link downloadFileFromStorage}: it + * bounds the source read. A compiled artifact is re-checked against the same ceiling + * below, since rendering can grow a small source into a large document. */ export async function downloadServableFileFromStorage( userFile: UserFile, requestId: string, logger: Logger, options: { - maxBytes?: number + maxBytes: number signal?: AbortSignal ownerKey?: string filePrincipal?: Principal - } = {} + } ): Promise { const buffer = await downloadFileFromStorage(userFile, requestId, logger, { maxBytes: options.maxBytes, @@ -430,10 +443,14 @@ export async function downloadServableFileFromStorage( const workspaceId = userFile.key ? (parseWorkspaceFileKey(userFile.key) ?? undefined) : undefined - return { - buffer: Buffer.from(await renderSimPageDocumentWithAssets(text, { workspaceId }), 'utf8'), - contentType: 'text/html', - } + const rendered = Buffer.from( + await renderSimPageDocumentWithAssets(text, { workspaceId }), + 'utf8' + ) + // Rendering inlines referenced assets, so a source well under the ceiling can + // resolve to a document well over it. + assertKnownSizeWithinLimit(rendered.length, options.maxBytes, 'servable page render') + return { buffer: rendered, contentType: 'text/html' } } } @@ -462,8 +479,58 @@ export async function downloadServableFileFromStorage( // Re-check: the raw download enforced maxBytes on the source, but a generated doc // resolves to a larger artifact. - if (options.maxBytes !== undefined && resolved.buffer.length > options.maxBytes) { - assertKnownSizeWithinLimit(resolved.buffer.length, options.maxBytes, 'servable file download') + assertKnownSizeWithinLimit(resolved.buffer.length, options.maxBytes, 'servable file download') + + return resolved +} + +/** + * Resolve every file of a multi-attachment request while bounding their COMBINED + * resident size. + * + * A per-file ceiling is not enough when a request carries an array: N attachments + * each just under the limit still cost N times the limit, and downloading them with + * `Promise.all` makes that the peak. Routes that pre-checked `sum(file.size)` were + * not protected either — the declared sizes come from the caller, which is why the + * downloader re-checks the delivered bytes. + * + * So this walks the list in order against a shrinking budget: each file may only use + * what the previous ones left. Sequential is the point — it is what keeps the peak at + * `totalMaxBytes` instead of the sum, and attachment lists are short enough that the + * lost parallelism does not register next to the provider round trip that follows. + * + * The overrun surfaces as a `PayloadSizeLimitError` restated in the caller's terms: + * the per-file failure underneath reports one file against whatever budget was left, + * which would read as a nonsense limit in a "total attachment size" message. `label` + * and `totalMaxBytes` are the caller's, and `observedBytes` is what the set needed. + */ +export async function downloadServableFilesWithinBudget( + userFiles: readonly UserFile[], + requestId: string, + logger: Logger, + options: { totalMaxBytes: number; label: string; signal?: AbortSignal } +): Promise { + const resolved: ServableFile[] = [] + let spent = 0 + + for (const userFile of userFiles) { + logger.info(`[${requestId}] Downloading ${userFile.name} (${userFile.size} bytes)`) + let servable: ServableFile + try { + servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: options.totalMaxBytes - spent, + signal: options.signal, + }) + } catch (error) { + if (!isPayloadSizeLimitError(error)) throw error + throw new PayloadSizeLimitError({ + label: options.label, + maxBytes: options.totalMaxBytes, + observedBytes: spent + (error.observedBytes ?? userFile.size), + }) + } + spent += servable.buffer.length + resolved.push(servable) } return resolved diff --git a/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts b/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts index f7456f44d73..725740ed411 100644 --- a/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts +++ b/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts @@ -13,7 +13,7 @@ const logger = createLogger('FetchServableWorkspaceFileBuffer') export async function fetchAuthorizedServableWorkspaceFileBuffer( fileRecord: WorkspaceFileRecord, filePrincipal: Principal, - options: { maxBytes?: number; signal?: AbortSignal; requestId?: string } = {} + options: { maxBytes: number; signal?: AbortSignal; requestId?: string } ): Promise<{ buffer: Buffer; contentType: string }> { return downloadServableFileFromStorage( { diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts index 39943c138e4..ca1ca6b784d 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts @@ -26,6 +26,7 @@ vi.mock('@sim/platform-authz/workspace', () => ({ resolveEffectiveWorkspacePermission: mocks.resolvePermission, })) +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { readWorkspaceFileContentByKey } from '@/lib/workspace-files/application/read-workspace-file-content-by-key' const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } @@ -75,7 +76,9 @@ describe('readWorkspaceFileContentByKey', () => { expect(mocks.getFile).toHaveBeenCalledWith(file.workspaceId, file.id, { throwOnError: true, }) - expect(mocks.fetchContent).toHaveBeenCalledWith(file) + expect(mocks.fetchContent).toHaveBeenCalledWith(file, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) }) it('rejects a stale key instead of serving the file current at the same ID', async () => { diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts index d5629ed275b..4a669fd940f 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts @@ -8,6 +8,7 @@ import { type WorkspaceFileRecord, } from '@/lib/uploads/contexts/workspace' import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { fileOperations } from '@/lib/workspace-files/application/operations' @@ -33,7 +34,10 @@ async function executeReadWorkspaceFileContentByKey({ throwOnError: true, }) if (!file || file.key !== input.key) throw new OrchestrationError('not_found', 'File not found') - return { file, content: await fetchWorkspaceFileBuffer(file) } + return { + file, + content: await fetchWorkspaceFileBuffer(file, { maxBytes: MAX_BUFFERED_TRANSFER_BYTES }), + } } export const readWorkspaceFileContentByKey = defineAuthorizedWorkspaceFileUseCase({ diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-content.ts index f35a14df784..bfd1020bd50 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-content.ts @@ -10,6 +10,7 @@ import { getBoundWorkspaceFileSecretProvenance, type WorkspaceFileSecretProvenance, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { fileOperations } from '@/lib/workspace-files/application/operations' import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/application/workspace-file-context' @@ -43,7 +44,9 @@ async function executeReadWorkspaceFileContent({ throwOnError: true, }) if (!file) throw new OrchestrationError('not_found', 'File not found') - const content = await fetchWorkspaceFileBuffer(file, { maxBytes: input.maxBytes }) + const content = await fetchWorkspaceFileBuffer(file, { + maxBytes: input.maxBytes ?? MAX_BUFFERED_TRANSFER_BYTES, + }) const secretProvenance = input.includeSecretProvenance ? await getBoundWorkspaceFileSecretProvenance(context.workspaceId, { fileId: file.id, diff --git a/apps/sim/lib/workspace-files/page-document.server.test.ts b/apps/sim/lib/workspace-files/page-document.server.test.ts new file mode 100644 index 00000000000..14375b70a16 --- /dev/null +++ b/apps/sim/lib/workspace-files/page-document.server.test.ts @@ -0,0 +1,121 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockDownloadFile, mockGetFileMetadataById, mockRenderSimPageDocument } = vi.hoisted(() => ({ + mockDownloadFile: vi.fn(), + mockGetFileMetadataById: vi.fn(), + mockRenderSimPageDocument: vi.fn(), +})) + +vi.mock('@/lib/uploads/core/storage-service', () => ({ + downloadFile: mockDownloadFile, +})) + +vi.mock('@/lib/uploads/server/metadata', () => ({ + getFileMetadataById: mockGetFileMetadataById, +})) + +vi.mock('@/lib/workspace-files/page-document', () => ({ + renderSimPageDocument: mockRenderSimPageDocument, +})) + +import { renderSimPageDocumentWithAssets } from '@/lib/workspace-files/page-document.server' + +const WORKSPACE_ID = 'ws-1' +const MB = 1024 * 1024 + +function imageRecord(id: string, size: number) { + return { + id, + key: `workspace/${WORKSPACE_ID}/${id}.png`, + context: 'workspace', + workspaceId: WORKSPACE_ID, + contentType: 'image/png', + size, + sizeBytes: size, + } +} + +function documentReferencing(ids: string[]) { + return ids.map((id) => ``).join('') +} + +describe('renderSimPageDocumentWithAssets memory bounds', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('charges the budget by delivered bytes, not by what the metadata claimed', async () => { + // Every row claims to be tiny; the objects are 8MB each. The budget must still + // stop at 32MB — planning off the recorded size would admit all six. + const ids = ['a', 'b', 'c', 'd', 'e', 'f'] + mockRenderSimPageDocument.mockReturnValue(documentReferencing(ids)) + mockGetFileMetadataById.mockImplementation(async (id: string) => imageRecord(id, 1024)) + mockDownloadFile.mockImplementation(async () => Buffer.alloc(8 * MB)) + + const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) + + expect(mockDownloadFile).toHaveBeenCalledTimes(4) + // The images past the budget keep their URL reference rather than failing the render. + expect(html).toContain('src="/api/files/view/e"') + expect(html).toContain('src="/api/files/view/f"') + }) + + it('offers each download only what the budget has left', async () => { + mockRenderSimPageDocument.mockReturnValue(documentReferencing(['a', 'b'])) + mockGetFileMetadataById.mockImplementation(async (id: string) => imageRecord(id, 1024)) + mockDownloadFile.mockImplementation(async () => Buffer.alloc(30 * MB)) + + await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) + + expect(mockDownloadFile).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ maxBytes: 8 * MB }) + ) + // 30MB delivered leaves 2MB, which is below the per-image limit and becomes the cap. + expect(mockDownloadFile).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ maxBytes: 2 * MB }) + ) + }) + + it('never offers a download more than the per-image limit', async () => { + mockRenderSimPageDocument.mockReturnValue(documentReferencing(['solo'])) + mockGetFileMetadataById.mockResolvedValue(imageRecord('solo', 1024)) + mockDownloadFile.mockResolvedValue(Buffer.from('png-bytes')) + + await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) + + expect(mockDownloadFile).toHaveBeenCalledWith( + expect.objectContaining({ maxBytes: 8 * MB, context: 'workspace' }) + ) + }) + + it('keeps the URL reference when a capped download rejects', async () => { + mockRenderSimPageDocument.mockReturnValue(documentReferencing(['big'])) + mockGetFileMetadataById.mockResolvedValue(imageRecord('big', 1024)) + mockDownloadFile.mockRejectedValue(new Error('storage download exceeds maximum size')) + + const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) + + expect(html).toContain('src="/api/files/view/big"') + }) + + it('inlines images that fit and leaves cross-workspace references alone', async () => { + mockRenderSimPageDocument.mockReturnValue(documentReferencing(['mine', 'theirs'])) + mockGetFileMetadataById.mockImplementation(async (id: string) => + id === 'mine' + ? imageRecord('mine', 1024) + : { ...imageRecord('theirs', 1024), workspaceId: 'ws-2' } + ) + mockDownloadFile.mockResolvedValue(Buffer.from('png-bytes')) + + const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID }) + + expect(mockDownloadFile).toHaveBeenCalledTimes(1) + expect(html).toContain(`data:image/png;base64,${Buffer.from('png-bytes').toString('base64')}`) + expect(html).toContain('src="/api/files/view/theirs"') + }) +}) diff --git a/apps/sim/lib/workspace-files/page-document.server.ts b/apps/sim/lib/workspace-files/page-document.server.ts index d0e2d5a7c22..4c4954e33dc 100644 --- a/apps/sim/lib/workspace-files/page-document.server.ts +++ b/apps/sim/lib/workspace-files/page-document.server.ts @@ -5,6 +5,13 @@ import { renderSimPageDocument } from '@/lib/workspace-files/page-document' /** Images past this size stay as URL references rather than bloating the document. */ const MAX_INLINE_IMAGE_BYTES = 8 * 1024 * 1024 +/** + * Ceiling on everything a single document inlines. A per-image limit does not bound + * the page on its own — N images each just under it still cost N times it. Images + * that do not fit what is left keep their URL reference, exactly like an oversized one. + */ +const MAX_INLINE_TOTAL_BYTES = 32 * 1024 * 1024 + const IMAGE_SRC = /src="[^"]*\/api\/files\/view\/([^"]+)"/g /** @@ -24,24 +31,41 @@ export async function renderSimPageDocumentWithAssets( const ids = [...new Set([...documentHtml.matchAll(IMAGE_SRC)].map((match) => match[1]))] if (ids.length === 0 || !options.workspaceId) return documentHtml - const inlined = new Map() - await Promise.all( + const candidates = await Promise.all( ids.map(async (id) => { - try { - const record = await getFileMetadataById(id) - if (!record || record.context !== 'workspace' || record.workspaceId !== options.workspaceId) - return - const bytes = await downloadFile({ key: record.key, context: 'workspace' }) - if (bytes.length > MAX_INLINE_IMAGE_BYTES) return - const mime = record.contentType?.startsWith('image/') - ? record.contentType - : 'application/octet-stream' - inlined.set(id, `data:${mime};base64,${bytes.toString('base64')}`) - } catch { - // A missing or unreadable image keeps its URL reference. - } + const record = await getFileMetadataById(id).catch(() => null) + if (!record || record.context !== 'workspace' || record.workspaceId !== options.workspaceId) + return null + return { id, record } }) ) + + // One image at a time, charged against the budget by what each download actually + // delivered. Fetching them concurrently made the peak the sum of every image rather + // than the largest one, and the ceiling on the finished document could only observe + // that after the fact. Each download is given whatever the budget has left, so an + // image that does not fit is refused by the read itself instead of after it lands. + const inlined = new Map() + let remaining = MAX_INLINE_TOTAL_BYTES + for (const candidate of candidates) { + if (!candidate) continue + if (remaining === 0) break + const { id, record } = candidate + try { + const bytes = await downloadFile({ + key: record.key, + context: 'workspace', + maxBytes: Math.min(MAX_INLINE_IMAGE_BYTES, remaining), + }) + remaining -= bytes.length + const mime = record.contentType?.startsWith('image/') + ? record.contentType + : 'application/octet-stream' + inlined.set(id, `data:${mime};base64,${bytes.toString('base64')}`) + } catch { + // A missing, unreadable or too-large image keeps its URL reference. + } + } if (inlined.size === 0) return documentHtml return documentHtml.replace(IMAGE_SRC, (match, id: string) => { const dataUri = inlined.get(id) diff --git a/apps/sim/tools/microsoft_teams/server-utils.ts b/apps/sim/tools/microsoft_teams/server-utils.ts index c5558ad8512..75df5c83fe6 100644 --- a/apps/sim/tools/microsoft_teams/server-utils.ts +++ b/apps/sim/tools/microsoft_teams/server-utils.ts @@ -5,6 +5,7 @@ */ import type { Logger } from '@sim/logger' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { FileAccessDeniedError, verifyFileAccess } from '@/app/api/files/authorization' @@ -79,11 +80,19 @@ export async function uploadFilesForTeamsMessage(params: { throw new FileAccessDeniedError() } - // Download file from storage - const { buffer, contentType } = await downloadServableFileFromStorage(file, requestId, log) - - if (buffer.length > MAX_TEAMS_FILE_SIZE) { - const sizeMB = (buffer.length / (1024 * 1024)).toFixed(2) + // Download file from storage, bounded by the same 4MB limit checked above — the + // declared size is the caller's claim, so the ceiling has to reach the read itself. + let buffer: Buffer + let contentType: string + try { + const servable = await downloadServableFileFromStorage(file, requestId, log, { + maxBytes: MAX_TEAMS_FILE_SIZE, + }) + buffer = servable.buffer + contentType = servable.contentType + } catch (error) { + if (!isPayloadSizeLimitError(error)) throw error + const sizeMB = ((error.observedBytes ?? file.size) / (1024 * 1024)).toFixed(2) throw new Error( `File "${file.name}" (${sizeMB}MB) exceeds the 4MB limit for Teams attachments. Use smaller files or upload to SharePoint/OneDrive first.` )