diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts index 78b4ce5311f..b7b1074db0f 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts @@ -11,10 +11,8 @@ import { } from '@/lib/execution/private-tool-metadata' const { - mockIsFeatureEnabled, mockGetTableById, mockListTables, - mockQueryRows, mockGetOrCreateTableSnapshot, mockDownloadFile, mockGeneratePresignedDownloadUrl, @@ -33,13 +31,10 @@ const { mockMaterializeCopilotCodeSecrets, mockHasWorkspaceSandboxAccess, mockImportWorkspaceFileSecretProvenanceForRuntime, - mockLoadTableRowSecretProvenance, - mockIsTableSnapshotSafeForModelMount, + mockGetTableSnapshotModelMountSafety, } = vi.hoisted(() => ({ - mockIsFeatureEnabled: vi.fn(), mockGetTableById: vi.fn(), mockListTables: vi.fn(), - mockQueryRows: vi.fn(), mockGetOrCreateTableSnapshot: vi.fn(), mockDownloadFile: vi.fn(), mockGeneratePresignedDownloadUrl: vi.fn(), @@ -58,20 +53,16 @@ const { mockMaterializeCopilotCodeSecrets: vi.fn(), mockHasWorkspaceSandboxAccess: vi.fn(), mockImportWorkspaceFileSecretProvenanceForRuntime: vi.fn(), - mockLoadTableRowSecretProvenance: vi.fn(), - mockIsTableSnapshotSafeForModelMount: vi.fn(), + mockGetTableSnapshotModelMountSafety: vi.fn(), })) -vi.mock('@/lib/core/config/feature-flags', () => ({ isFeatureEnabled: mockIsFeatureEnabled })) vi.mock('@/lib/core/security/encryption', () => encryptionMock) vi.mock('@/lib/table/service', () => ({ getTableById: mockGetTableById, listTables: mockListTables, })) -vi.mock('@/lib/table/rows/service', () => ({ queryRows: mockQueryRows })) vi.mock('@/lib/table/rows/secret-provenance', () => ({ - isTableSnapshotSafeForModelMount: mockIsTableSnapshotSafeForModelMount, - loadTableRowSecretProvenance: mockLoadTableRowSecretProvenance, + getTableSnapshotModelMountSafety: mockGetTableSnapshotModelMountSafety, })) vi.mock('@/lib/table/snapshot-cache', () => ({ getOrCreateTableSnapshot: mockGetOrCreateTableSnapshot, @@ -133,7 +124,7 @@ import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-tr const table = { id: 'tbl_1', workspaceId: 'ws_1', - rowCount: 1000, + rowCount: 1, schema: { columns: [{ id: 'col_name', name: 'name', type: 'string' }] }, } @@ -151,20 +142,12 @@ function mountedFiles() { return params._sandboxFiles ?? [] } -const snapshotCacheOn = (flag: string) => Promise.resolve(flag === 'table-snapshot-cache') - function resetExecutionMocks(): void { vi.clearAllMocks() mockExecuteTool.mockReset() mockMaterializeCopilotCodeSecrets.mockReset() - mockLoadTableRowSecretProvenance.mockReset() - mockIsTableSnapshotSafeForModelMount.mockReset() - mockLoadTableRowSecretProvenance.mockResolvedValue({ - version: 1, - complete: true, - entries: [], - }) - mockIsTableSnapshotSafeForModelMount.mockResolvedValue(true) + mockGetTableSnapshotModelMountSafety.mockReset() + mockGetTableSnapshotModelMountSafety.mockResolvedValue('safe') mockListWorkspaceFiles.mockResolvedValue([]) mockListWorkspaceFileFolders.mockResolvedValue([]) mockListAllWorkspaceFiles.mockImplementation(async () => { @@ -576,72 +559,12 @@ describe('executeFunctionExecute table mounts', () => { resetExecutionMocks() mockExecuteTool.mockResolvedValue({ success: true }) mockGetTableById.mockResolvedValue(table) - mockIsFeatureEnabled.mockResolvedValue(false) - // Row data is keyed by stable column id at rest, not display name. - mockQueryRows.mockResolvedValue({ rows: [{ data: { col_name: 'Ada' } }] }) mockHasCloudStorage.mockReturnValue(true) mockGeneratePresignedDownloadUrl.mockResolvedValue('https://s3.example/presigned?sig=abc') }) - it('flag OFF: drains the table inline via queryRows (existing path)', async () => { - await executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never) - - expect(mockQueryRows).toHaveBeenCalledTimes(1) - expect(mockGetOrCreateTableSnapshot).not.toHaveBeenCalled() - const files = mountedFiles() - expect(files[0].path).toBe('/home/user/tables/tbl_1.csv') - expect(files[0].content).toBe('name\nAda') - }) - - it('mounts CSV with display-name headers and id-keyed values, never column ids', async () => { - mockGetTableById.mockResolvedValue({ - id: 'tbl_2', - workspaceId: 'ws_1', - rowCount: 2, - schema: { - columns: [ - { id: 'col_name', name: 'name', type: 'string' }, - { id: 'col_company', name: 'company', type: 'string' }, - ], - }, - }) - mockQueryRows.mockResolvedValue({ - rows: [ - { data: { col_name: 'Ada', col_company: 'Analytical Engine' } }, - { data: { col_name: 'Grace', col_company: 'Navy, Inc' } }, - ], - }) - - await executeFunctionExecute({ inputTables: ['tbl_2'] }, context as never) - - const csv = mountedFiles()[0].content as string - const lines = csv.split('\n') - expect(lines[0]).toBe('name,company') - expect(lines[1]).toBe('Ada,Analytical Engine') - // Value containing a comma is quoted. - expect(lines[2]).toBe('Grace,"Navy, Inc"') - // No stable column id leaks into the mounted file. - expect(csv).not.toContain('col_name') - expect(csv).not.toContain('col_company') - }) - - it('reads values by column id for legacy name-keyed rows too', async () => { - // Legacy column with no id: getColumnId falls back to name, so name-keyed data is correct. - mockGetTableById.mockResolvedValue({ - id: 'tbl_legacy', - workspaceId: 'ws_1', - rowCount: 1, - schema: { columns: [{ name: 'email', type: 'string' }] }, - }) - mockQueryRows.mockResolvedValue({ rows: [{ data: { email: 'a@b.com' } }] }) - - await executeFunctionExecute({ inputTables: ['tbl_legacy'] }, context as never) - - expect(mountedFiles()[0].content).toBe('email\na@b.com') - }) - - it('flag ON + cloud storage: mounts by presigned URL, no bytes through web', async () => { - mockIsFeatureEnabled.mockImplementation(snapshotCacheOn) + it('mounts every table by presigned snapshot URL', async () => { + mockGetTableById.mockResolvedValue({ ...table, rowCount: 0 }) mockGetOrCreateTableSnapshot.mockResolvedValue({ key: 'table-snapshots/ws_1/tbl_1/v5.csv', size: 9, @@ -651,7 +574,6 @@ describe('executeFunctionExecute table mounts', () => { await executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never) expect(mockGetOrCreateTableSnapshot).toHaveBeenCalledTimes(1) - expect(mockQueryRows).not.toHaveBeenCalled() expect(mockDownloadFile).not.toHaveBeenCalled() expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledWith( 'table-snapshots/ws_1/tbl_1/v5.csv', @@ -665,8 +587,7 @@ describe('executeFunctionExecute table mounts', () => { }) }) - it('flag ON + local storage: falls back to a buffered content mount', async () => { - mockIsFeatureEnabled.mockImplementation(snapshotCacheOn) + it('mounts a complete snapshot through a bounded buffer with local storage', async () => { mockHasCloudStorage.mockReturnValue(false) mockGetOrCreateTableSnapshot.mockResolvedValue({ key: 'table-snapshots/ws_1/tbl_1/v5.csv', @@ -687,9 +608,8 @@ describe('executeFunctionExecute table mounts', () => { expect(file.type).toBeUndefined() }) - it('flag ON + unknown snapshot provenance still mounts and taints model egress', async () => { - mockIsFeatureEnabled.mockImplementation(snapshotCacheOn) - mockIsTableSnapshotSafeForModelMount.mockResolvedValue(false) + it('unknown snapshot provenance still mounts and taints model egress', async () => { + mockGetTableSnapshotModelMountSafety.mockResolvedValue('unsafe-provenance') mockGetOrCreateTableSnapshot.mockResolvedValue({ key: 'table-snapshots/ws_1/tbl_1/v5.csv', size: 9, @@ -717,46 +637,22 @@ describe('executeFunctionExecute table mounts', () => { expect(projectToolResultForCopilot(result, parentRegistry)).toEqual({ success: true }) }) - it('flag OFF + unknown row provenance still mounts and taints model egress', async () => { - mockLoadTableRowSecretProvenance.mockResolvedValue({ - version: 1, - complete: false, - entries: [], - }) - mockExecuteTool.mockResolvedValue({ success: true, output: { result: 'raw output' } }) - const parentRegistry = new ResolvedSecretTraceRegistry([], { - userId: 'u1', - workspaceId: 'ws_1', - }) - - const result = await executeFunctionExecute( - { inputTables: ['tbl_1'] }, - { ...context, resolvedSecretTraceRegistry: parentRegistry } - ) - - expect(mountedFiles()[0].content).toBe('name\nAda') - expect(mockExecuteTool.mock.calls[0]?.[1]?.[PRIVATE_SECRET_PROVENANCE_FIELD]).toEqual({ - version: 1, - complete: false, - selections: [], + it('rejects a snapshot that becomes stale before mounting', async () => { + mockGetTableSnapshotModelMountSafety.mockResolvedValue('stale') + mockGetOrCreateTableSnapshot.mockResolvedValue({ + key: 'table-snapshots/ws_1/tbl_1/v5.csv', + size: 9, + version: 5, }) - expect(result).toEqual({ success: true, output: { result: 'raw output' } }) - expect(parentRegistry.isComplete()).toBe(false) - expect(projectToolResultForCopilot(result, parentRegistry)).toEqual({ success: true }) - }) - - it('flag ON but small table stays on the inline path', async () => { - mockIsFeatureEnabled.mockImplementation(snapshotCacheOn) - mockGetTableById.mockResolvedValue({ ...table, rowCount: 10 }) - - await executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never) - expect(mockGetOrCreateTableSnapshot).not.toHaveBeenCalled() - expect(mockQueryRows).toHaveBeenCalledTimes(1) + await expect( + executeFunctionExecute({ inputTables: ['tbl_1'] }, context as never) + ).rejects.toThrow(/changed while preparing its snapshot/) + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + expect(mockExecuteTool).not.toHaveBeenCalled() }) - it('flag ON + cloud: throws when the snapshot exceeds the table mount limit', async () => { - mockIsFeatureEnabled.mockImplementation(snapshotCacheOn) + it('throws when a cloud snapshot exceeds the table mount limit', async () => { mockGetOrCreateTableSnapshot.mockResolvedValue({ key: 'table-snapshots/ws_1/tbl_1/v5.csv', size: 600 * 1024 * 1024, @@ -769,8 +665,22 @@ describe('executeFunctionExecute table mounts', () => { expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() }) - it('flag ON + local: throws when the snapshot exceeds the per-file mount limit', async () => { - mockIsFeatureEnabled.mockImplementation(snapshotCacheOn) + it('throws when cloud snapshots exceed the aggregate URL mount limit', async () => { + mockGetTableById.mockImplementation(async (tableId: string) => ({ ...table, id: tableId })) + mockGetOrCreateTableSnapshot.mockImplementation(async (mountedTable: typeof table) => ({ + key: `table-snapshots/ws_1/${mountedTable.id}/v5.csv`, + size: 500 * 1024 * 1024, + version: 5, + })) + const tableIds = Array.from({ length: 5 }, (_, index) => `tbl_${index}`) + + await expect( + executeFunctionExecute({ inputTables: tableIds }, context as never) + ).rejects.toThrow(/total mount limit/) + expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledTimes(4) + }) + + it('throws when a local snapshot exceeds the per-file mount limit', async () => { mockHasCloudStorage.mockReturnValue(false) mockGetOrCreateTableSnapshot.mockResolvedValue({ key: 'table-snapshots/ws_1/tbl_1/v5.csv', @@ -809,7 +719,6 @@ describe('executeFunctionExecute file mounts', () => { beforeEach(() => { resetExecutionMocks() mockExecuteTool.mockResolvedValue({ success: true }) - mockIsFeatureEnabled.mockResolvedValue(false) mockHasCloudStorage.mockReturnValue(true) mockGeneratePresignedDownloadUrl.mockResolvedValue('https://s3.example/file?sig=abc') mockListWorkspaceFiles.mockResolvedValue([fileRecord]) @@ -1167,7 +1076,6 @@ describe('executeFunctionExecute unmountable namespaces', () => { beforeEach(() => { resetExecutionMocks() mockExecuteTool.mockResolvedValue({ success: true }) - mockIsFeatureEnabled.mockResolvedValue(false) mockHasCloudStorage.mockReturnValue(true) mockListWorkspaceFiles.mockResolvedValue([]) mockFindWorkspaceFileRecord.mockReturnValue(null) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index 84d78343427..0d40949bc7e 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -11,8 +11,6 @@ import { materializeCopilotCodeSecrets, } from '@/lib/copilot/tools/secret-mount-materializer.server' import { decodeVfsPathSegments, encodeVfsPathSegments } from '@/lib/copilot/vfs/path-utils' -import { isFeatureEnabled } from '@/lib/core/config/feature-flags' -import { neutralizeCsvFormula, toCsvRow } from '@/lib/core/utils/csv' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import type { PrivateSecretProvenanceBundleV1 } from '@/lib/execution/model-input-provenance' import { @@ -21,14 +19,7 @@ import { } from '@/lib/execution/private-tool-metadata' import { MAX_PLAN_REQUIRED } from '@/lib/execution/remote-sandbox/workspace-sandboxes' import { recordSecretUsage } from '@/lib/secrets/usage/record' -import { getColumnId } from '@/lib/table/column-keys' -import { TABLE_LIMITS } from '@/lib/table/constants' -import { formatCsvCell } from '@/lib/table/export-format' -import { - isTableSnapshotSafeForModelMount, - loadTableRowSecretProvenance, -} from '@/lib/table/rows/secret-provenance' -import { queryRows } from '@/lib/table/rows/service' +import { getTableSnapshotModelMountSafety } from '@/lib/table/rows/secret-provenance' import { getTableById, listTables } from '@/lib/table/service' import { getOrCreateTableSnapshot, SNAPSHOT_MAX_BYTES } from '@/lib/table/snapshot-cache' import { @@ -62,13 +53,6 @@ const MAX_FILE_SIZE = 10 * 1024 * 1024 const MAX_TOTAL_SIZE = 50 * 1024 * 1024 const MAX_MOUNTED_FILES = 500 -/** - * Below this row count a table mounts via the direct inline CSV path — the version-keyed snapshot - * cache (storage round-trip) only pays off for larger/hot tables. Behind the feature flag either - * way; this just keeps tiny one-shot tables on the cheaper path. - */ -const SNAPSHOT_MIN_ROWS = 500 - /** * Lifetime of a presigned URL handed to the sandbox to fetch a mounted object (table snapshot or * workspace file). Long enough to download a large file at sandbox startup; the URL grants read to @@ -318,7 +302,6 @@ export async function resolveInputFiles( inputFiles?: unknown[], inputTables?: unknown[], inputDirectories?: unknown[], - provenanceUserId?: string, resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry, filePrincipal?: Principal ): Promise { @@ -475,7 +458,6 @@ export async function resolveInputFiles( const tablePathLookup = hasTablePathRefs ? new Map((await listTables(workspaceId)).map((table) => [table.name, table])) : undefined - const snapshotCacheEnabled = await isFeatureEnabled('table-snapshot-cache') for (const tableRef of inputTables) { const tableId = typeof tableRef === 'string' @@ -496,109 +478,64 @@ export async function resolveInputFiles( : undefined const mountPath = sandboxPath || `/home/user/tables/${table.id}.csv` - // Large/hot tables mount by reference from a version-keyed CSV snapshot in object storage. - if (snapshotCacheEnabled && table.rowCount >= SNAPSHOT_MIN_ROWS) { - const snapshot = await getOrCreateTableSnapshot(table, 'copilot-fn-exec') - if (!resolvedSecretTraceRegistry) { - throw new Error( - `Input table "${tableId}" cannot be mounted because its secret provenance is unavailable.` - ) - } - try { - const safeForModelMount = await isTableSnapshotSafeForModelMount({ - tableId: table.id, - workspaceId, - rowsVersion: snapshot.version, - }) - if (!safeForModelMount) - resolvedSecretTraceRegistry.markIncomplete('table-snapshot-unsafe-for-mount') - } catch { - resolvedSecretTraceRegistry.markIncomplete('table-snapshot-unsafe-for-mount') - } - - if (hasCloudStorage()) { - // Mount by reference: the sandbox fetches the snapshot straight from storage via a - // presigned URL, so the bytes never pass through the web process — the only ceiling is - // sandbox disk (enforced at materialization by SNAPSHOT_MAX_BYTES). - if (snapshot.size > SNAPSHOT_MAX_BYTES) { - throw new Error( - `Input table "${tableId}" is ${Math.round(snapshot.size / 1024 / 1024)}MB, over the ${SNAPSHOT_MAX_BYTES / 1024 / 1024}MB table mount limit.` - ) - } - const url = await generatePresignedDownloadUrl( - snapshot.key, - 'execution', - MOUNT_URL_TTL_SECONDS - ) - sandboxFiles.push({ type: 'url', path: mountPath, url }) - continue - } + const snapshot = await getOrCreateTableSnapshot(table, 'copilot-fn-exec') + if (!resolvedSecretTraceRegistry) { + throw new Error( + `Input table "${tableId}" cannot be mounted because its secret provenance is unavailable.` + ) + } + const mountSafety = await getTableSnapshotModelMountSafety({ + tableId: table.id, + workspaceId, + rowsVersion: snapshot.version, + }) + if (mountSafety === 'stale') { + throw new Error(`Input table "${tableId}" changed while preparing its snapshot. Retry.`) + } + if (mountSafety === 'unsafe-provenance') { + resolvedSecretTraceRegistry.markIncomplete('table-snapshot-unsafe-for-mount') + } - // Local storage: a presigned URL is an app-internal serve path a remote sandbox can't - // reach, so fall back to buffering the bytes through the web process (file-mount guards). - if (snapshot.size > MAX_FILE_SIZE) { + if (hasCloudStorage()) { + if (snapshot.size > SNAPSHOT_MAX_BYTES) { throw new Error( - `Input table "${tableId}" is ${Math.round(snapshot.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.` + `Input table "${tableId}" is ${Math.round(snapshot.size / 1024 / 1024)}MB, over the ${SNAPSHOT_MAX_BYTES / 1024 / 1024}MB table mount limit.` ) } - if (mounted.buffered + snapshot.size > MAX_TOTAL_SIZE) { + if (mounted.url + snapshot.size > MAX_TOTAL_URL_BYTES) { throw new Error( - `Mounting "${tableId}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller tables.` + `Mounting "${tableId}" would exceed the ${MAX_TOTAL_URL_BYTES / 1024 / 1024 / 1024}GB total mount limit. Mount fewer or smaller files and tables.` ) } - const buffer = await downloadFile({ - key: snapshot.key, - context: 'execution', - maxBytes: MAX_FILE_SIZE, - }) - mounted.buffered += buffer.length - sandboxFiles.push({ path: mountPath, content: buffer.toString('utf-8') }) + const url = await generatePresignedDownloadUrl( + snapshot.key, + 'execution', + MOUNT_URL_TTL_SECONDS + ) + sandboxFiles.push({ type: 'url', path: mountPath, url }) + mounted.url += snapshot.size continue } - // Keep the prior bounded mount — draining the whole table here was backed - // out for OOM, so don't ride the new unbounded queryRows default. - const rows = await queryRows( - table, - { limit: TABLE_LIMITS.DEFAULT_QUERY_LIMIT }, - 'copilot-fn-exec' - ) - if (!resolvedSecretTraceRegistry) { + // Local storage: a presigned URL is an app-internal serve path a remote sandbox can't + // reach, so fall back to buffering the bytes through the web process (file-mount guards). + if (snapshot.size > MAX_FILE_SIZE) { throw new Error( - `Input table "${tableId}" cannot be mounted because its secret provenance is unavailable.` + `Input table "${tableId}" is ${Math.round(snapshot.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.` ) } - try { - const provenance = await loadTableRowSecretProvenance(rows.rows, { - userId: provenanceUserId ?? 'opaque-model-mount', - workspaceId, - }) - if ( - !provenance.complete || - !(await resolvedSecretTraceRegistry.importProvenance(provenance, { - trusted: true, - origin: 'copilotFunctionExecute.result', - })) - ) { - resolvedSecretTraceRegistry.markIncomplete('source-provenance-incomplete', { - origin: 'copilotFunctionExecute.result', - }) - } - } catch { - resolvedSecretTraceRegistry.markIncomplete('source-provenance-incomplete', { - origin: 'copilotFunctionExecute.result', - }) - } - - const columns = table.schema.columns - const csvLines = [toCsvRow(columns.map((column) => neutralizeCsvFormula(column.name)))] - for (const row of rows.rows) { - csvLines.push( - toCsvRow(columns.map((column) => formatCsvCell(column, row.data[getColumnId(column)]))) + if (mounted.buffered + snapshot.size > MAX_TOTAL_SIZE) { + throw new Error( + `Mounting "${tableId}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller tables.` ) } - const csvContent = csvLines.join('\n') - sandboxFiles.push({ path: mountPath, content: csvContent }) + const buffer = await downloadFile({ + key: snapshot.key, + context: 'execution', + maxBytes: MAX_FILE_SIZE, + }) + mounted.buffered += buffer.length + sandboxFiles.push({ path: mountPath, content: buffer.toString('utf-8') }) } } @@ -737,7 +674,6 @@ export async function executeFunctionExecute( inputFiles, inputTables, inputDirectories, - secretActorUserId ?? context.userId, mountedRegistry, inputFiles.length > 0 || inputDirectories.length > 0 ? resolveCopilotFilePrincipal(context) diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index c39c3310ec5..b33f02f4fa9 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -180,7 +180,6 @@ export const env = createEnv({ BILLING_CONCURRENCY_LIMIT_TEAM: z.string().optional(), // In-flight executions per Max-tier billing account (Max and Max for Teams) BILLING_CONCURRENCY_LIMIT_ENTERPRISE: z.string().optional(), // In-flight executions per Enterprise billing account (metadata-overridable) BILLING_ENABLED: z.boolean().optional(), // Enable billing enforcement and usage tracking - TABLE_SNAPSHOT_CACHE: z.boolean().optional(), // Mount tables into sandboxes by reference via a version-keyed CSV snapshot in object storage instead of draining the whole table into web-process heap PII_REDACTION: z.boolean().optional(), // Redact PII from workflow logs via configurable Data Retention rules (Presidio at the logger persist choke point) and expose the Data Retention config UI PII_GRANULAR_REDACTION: z.boolean().optional(), // Expose the execution-altering PII redaction stages (redact workflow input + block outputs in-flight) in the Data Retention config; layered on top of PII_REDACTION TRIGGER_EU_REGION: z.boolean().optional(), // Route Trigger.dev runs to eu-central-1 instead of the default us-east-1 (fallback for the trigger-eu-region flag when AppConfig is not the source of truth) diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index 9376d2031ec..39d91747ffb 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -56,14 +56,6 @@ interface FeatureFlagDefinition { /** The single registry of known flags. To add a flag, add one entry here. */ const FEATURE_FLAGS = { - 'table-snapshot-cache': { - description: - 'Mount Sim tables into code sandboxes by reference via a version-keyed CSV snapshot in ' + - 'object storage (reused across runs until the table mutates) instead of draining the whole ' + - 'table into web-process heap. resolveInputFiles evaluates without user context — use ' + - 'enabled:true for global rollout rather than per-user targeting.', - fallback: 'TABLE_SNAPSHOT_CACHE', - }, 'pii-redaction': { description: 'Redact PII from workflow logs via configurable Data Retention rules (Presidio at the ' + diff --git a/apps/sim/lib/table/rows/secret-provenance.test.ts b/apps/sim/lib/table/rows/secret-provenance.test.ts index acb20fc811c..b2966a0e9d3 100644 --- a/apps/sim/lib/table/rows/secret-provenance.test.ts +++ b/apps/sim/lib/table/rows/secret-provenance.test.ts @@ -20,7 +20,7 @@ vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ import type { DbTransaction } from '@/lib/table/planner' import { classifyTableRowSecretProvenanceForCopy, - isTableSnapshotSafeForModelMount, + getTableSnapshotModelMountSafety, loadTableRowSecretProvenance, mutateTableRowsWithSecretProvenance, updateTableRowsWithDerivedSecretProvenance, @@ -67,30 +67,31 @@ describe('table row secret provenance', () => { queueTableRows(userTableRows, []) await expect( - isTableSnapshotSafeForModelMount({ + getTableSnapshotModelMountSafety({ tableId: 'table-1', workspaceId: 'workspace-1', rowsVersion: 7, }) - ).resolves.toBe(true) + ).resolves.toBe('safe') expect(dbChainMockFns.limit).toHaveBeenCalledTimes(3) expect(dbChainMockFns.orderBy).not.toHaveBeenCalled() }) - it('rejects the snapshot as soon as an unsafe row exists', async () => { + it('classifies unsafe provenance after confirming the snapshot remains current', async () => { queueTableRows(userTableDefinitions, [{ rowsVersion: 7 }]) queueTableRows(userTableRows, [{ id: 'unsafe-row' }]) + queueTableRows(userTableDefinitions, [{ rowsVersion: 7 }]) await expect( - isTableSnapshotSafeForModelMount({ + getTableSnapshotModelMountSafety({ tableId: 'table-1', workspaceId: 'workspace-1', rowsVersion: 7, }) - ).resolves.toBe(false) + ).resolves.toBe('unsafe-provenance') - expect(dbChainMockFns.limit).toHaveBeenCalledTimes(2) + expect(dbChainMockFns.limit).toHaveBeenCalledTimes(3) }) it('rejects a snapshot when the table changes during the safety check', async () => { @@ -99,12 +100,12 @@ describe('table row secret provenance', () => { queueTableRows(userTableRows, []) await expect( - isTableSnapshotSafeForModelMount({ + getTableSnapshotModelMountSafety({ tableId: 'table-1', workspaceId: 'workspace-1', rowsVersion: 7, }) - ).resolves.toBe(false) + ).resolves.toBe('stale') }) it('keeps untouched legacy rows readable with exact-empty provenance', async () => { diff --git a/apps/sim/lib/table/rows/secret-provenance.ts b/apps/sim/lib/table/rows/secret-provenance.ts index 48e9ac0e73d..a02dd4b696a 100644 --- a/apps/sim/lib/table/rows/secret-provenance.ts +++ b/apps/sim/lib/table/rows/secret-provenance.ts @@ -705,17 +705,19 @@ async function readTableRowsVersion(tableId: string, workspaceId: string): Promi return table?.rowsVersion ?? null } +export type TableSnapshotModelMountSafety = 'safe' | 'unsafe-provenance' | 'stale' + /** - * Verifies that a version-pinned table contains no secret-bearing or unknown - * cells before its opaque CSV bytes cross into a model-controlled sandbox. + * Classifies whether a version-pinned table snapshot can cross into a model-controlled sandbox. + * A version change takes precedence over provenance because stale bytes must never be mounted. */ -export async function isTableSnapshotSafeForModelMount(options: { +export async function getTableSnapshotModelMountSafety(options: { tableId: string workspaceId: string rowsVersion: number -}): Promise { +}): Promise { if ((await readTableRowsVersion(options.tableId, options.workspaceId)) !== options.rowsVersion) { - return false + return 'stale' } const [unsafeRow] = await db @@ -748,9 +750,11 @@ export async function isTableSnapshotSafeForModelMount(options: { ) .limit(1) - if (unsafeRow) return false + if ((await readTableRowsVersion(options.tableId, options.workspaceId)) !== options.rowsVersion) { + return 'stale' + } - return (await readTableRowsVersion(options.tableId, options.workspaceId)) === options.rowsVersion + return unsafeRow ? 'unsafe-provenance' : 'safe' } /**