From 9873df03881b5455a0dcb39e596525d9ea45ef85 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 31 Oct 2026 16:08:36 -0700 Subject: [PATCH 1/3] fix(files): stream served files with byte ranges instead of buffering them under 100 MB --- .../api/files/serve/[...path]/route.test.ts | 273 +++++++++++++++--- .../app/api/files/serve/[...path]/route.ts | 211 ++++++++++---- apps/sim/app/api/files/utils.ts | 71 +++++ .../components/file-viewer/file-viewer.tsx | 152 +++++++++- .../agent/memory-harness.integration.ts | 2 +- apps/sim/hooks/queries/workspace-files.ts | 28 +- apps/sim/hooks/use-file-content-source.tsx | 8 + apps/sim/lib/uploads/client/download.test.ts | 46 +++ apps/sim/lib/uploads/client/download.ts | 34 ++- apps/sim/lib/uploads/core/storage-service.ts | 19 +- apps/sim/lib/uploads/providers/blob/client.ts | 8 +- .../providers/google-cloud-storage/client.ts | 9 +- apps/sim/lib/uploads/providers/s3/client.ts | 10 +- .../sim/lib/uploads/server/byte-range.test.ts | 113 ++++++++ apps/sim/lib/uploads/server/byte-range.ts | 79 +++++ apps/sim/lib/uploads/server/delivery.ts | 9 + apps/sim/lib/uploads/shared/types.ts | 6 + ...read-workspace-file-record-by-key.test.ts} | 59 +--- ...s => read-workspace-file-record-by-key.ts} | 41 +-- .../application/read-workspace-inline-file.ts | 21 +- .../stream-authorized-workspace-file.test.ts | 107 +++++++ .../stream-authorized-workspace-file.ts | 48 +++ scripts/check-unused-exports.baseline.json | 2 - 23 files changed, 1100 insertions(+), 256 deletions(-) create mode 100644 apps/sim/lib/uploads/server/byte-range.test.ts create mode 100644 apps/sim/lib/uploads/server/byte-range.ts rename apps/sim/lib/workspace-files/application/{read-workspace-file-content-by-key.test.ts => read-workspace-file-record-by-key.test.ts} (66%) rename apps/sim/lib/workspace-files/application/{read-workspace-file-content-by-key.ts => read-workspace-file-record-by-key.ts} (63%) create mode 100644 apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.test.ts create mode 100644 apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.ts diff --git a/apps/sim/app/api/files/serve/[...path]/route.test.ts b/apps/sim/app/api/files/serve/[...path]/route.test.ts index 7cfca662081..19a2a0cb4a2 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.test.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.test.ts @@ -2,6 +2,7 @@ * Tests for file serve API route */ +import { Readable } from 'node:stream' import { authMockFns, hybridAuthMockFns, @@ -41,7 +42,7 @@ import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' const { mockReadFile, mockAuthenticateWorkspaceFile, - mockReadWorkspaceFileContentByKey, + mockStreamAuthorizedWorkspaceFile, mockReadWorkspaceFileRecordByKey, mockAuthorizeWorkspaceFileRecordByKey, mockResolveServableDocBytes, @@ -64,7 +65,7 @@ const { mockReadOrganizationAssistantImage: vi.fn(), mockReadFile: vi.fn(), mockAuthenticateWorkspaceFile: vi.fn(), - mockReadWorkspaceFileContentByKey: vi.fn(), + mockStreamAuthorizedWorkspaceFile: vi.fn(), mockReadWorkspaceFileRecordByKey: vi.fn(), mockAuthorizeWorkspaceFileRecordByKey: vi.fn(async () => undefined), mockResolveServableDocBytes: vi.fn(), @@ -117,8 +118,11 @@ vi.mock('@/lib/workspace-files/api', () => ({ internalWorkspaceFileServeAuth: { authenticate: mockAuthenticateWorkspaceFile }, })) -vi.mock('@/lib/workspace-files/application/read-workspace-file-content-by-key', () => ({ - readWorkspaceFileContentByKey: { execute: mockReadWorkspaceFileContentByKey }, +vi.mock('@/lib/workspace-files/application/stream-authorized-workspace-file', () => ({ + streamAuthorizedWorkspaceFile: mockStreamAuthorizedWorkspaceFile, +})) + +vi.mock('@/lib/workspace-files/application/read-workspace-file-record-by-key', () => ({ readWorkspaceFileRecordByKey: { execute: mockReadWorkspaceFileRecordByKey, authorize: mockAuthorizeWorkspaceFileRecordByKey, @@ -129,7 +133,9 @@ vi.mock('@/lib/uploads/documents/compile', () => ({ resolveServableDocBytes: mockResolveServableDocBytes, })) -vi.mock('@/app/api/files/utils', () => ({ +vi.mock('@/app/api/files/utils', async (importOriginal) => ({ + createStoredFileStreamResponse: (await importOriginal()) + .createStoredFileStreamResponse, FileNotFoundError, createFileResponse: mockCreateFileResponse, createConditionalFileResponse: mockCreateConditionalFileResponse, @@ -151,6 +157,10 @@ const serveLogger = getMockLogger('FilesServeAPI') const mockDownloadCopilotFile = uploadsMockFns.mockDownloadCopilotFile const mockIsUsingCloudStorage = uploadsMockFns.mockIsUsingCloudStorage +function webStream(text: string): ReadableStream { + return new Blob([text]).stream() +} + describe('File Serve API Route', () => { beforeEach(() => { hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ @@ -170,14 +180,7 @@ describe('File Serve API Route', () => { mockAuthenticateWorkspaceFile.mockResolvedValue( createSessionPrincipal({ userId: 'test-user-id' }) ) - mockReadWorkspaceFileContentByKey.mockResolvedValue({ - file: { - id: 'file-1', - workspaceId: 'test-workspace-id', - name: 'report.pdf', - }, - content: Buffer.from('generated source'), - }) + mockStreamAuthorizedWorkspaceFile.mockImplementation(async () => webStream('stored bytes')) mockReadWorkspaceFileRecordByKey.mockResolvedValue({ file: { id: 'file-1', @@ -248,11 +251,11 @@ describe('File Serve API Route', () => { mockDownloadCopilotFile.mockResolvedValue(Buffer.from('bytes')) await GET( - createMockRequest({ url: 'http://localhost:3000/api/files/serve/copilot/doc.txt' }), - createRouteContext({ path: ['copilot', 'doc.txt'] }) + createMockRequest({ url: 'http://localhost:3000/api/files/serve/copilot/doc.pdf' }), + createRouteContext({ path: ['copilot', 'doc.pdf'] }) ) - expect(mockDownloadCopilotFile).toHaveBeenCalledWith('copilot/doc.txt', { + expect(mockDownloadCopilotFile).toHaveBeenCalledWith('copilot/doc.pdf', { maxBytes: MAX_BUFFERED_TRANSFER_BYTES, }) }) @@ -287,9 +290,9 @@ describe('File Serve API Route', () => { it('bounds the local read rather than trusting the stored size', async () => { await GET( createMockRequest({ - url: 'http://localhost:3000/api/files/serve/workspace/ws/test-file.txt', + url: 'http://localhost:3000/api/files/serve/workspace/ws/report.pdf', }), - createRouteContext({ path: ['workspace', 'ws', 'test-file.txt'] }) + createRouteContext({ path: ['workspace', 'ws', 'report.pdf'] }) ) expect(mockReadLocalFileWithinLimit).toHaveBeenCalledWith( @@ -318,8 +321,8 @@ describe('File Serve API Route', () => { ) const response = await GET( - createMockRequest({ url: 'http://localhost:3000/api/files/serve/workspace/ws/huge.bin' }), - createRouteContext({ path: ['workspace', 'ws', 'huge.bin'] }) + createMockRequest({ url: 'http://localhost:3000/api/files/serve/workspace/ws/huge.pdf' }), + createRouteContext({ path: ['workspace', 'ws', 'huge.pdf'] }) ) expect(response.status).toBe(413) @@ -334,10 +337,10 @@ describe('File Serve API Route', () => { }) const response = await GET( - new NextRequest('http://localhost:3000/api/files/serve/workspace/ws/report.md', { + new NextRequest('http://localhost:3000/api/files/serve/workspace/ws/report.pdf', { signal: controller.signal, }), - createRouteContext({ path: ['workspace', 'ws', 'report.md'] }) + createRouteContext({ path: ['workspace', 'ws', 'report.pdf'] }) ) expect(response.status).toBe(499) @@ -400,36 +403,220 @@ describe('File Serve API Route', () => { const response = await serveVersionedDoc(true) expect(response.headers.get('Cache-Control')).toBe('private, no-cache, must-revalidate') }) + }) - it('serves stored bytes under the type their extension names, not the recorded type', async () => { + describe('streaming stored objects', () => { + const principal = createSessionPrincipal({ userId: 'test-user-id' }) + const video = { + id: 'file-1', + workspaceId: 'test-workspace-id', + name: 'clip.mp4', + key: 'workspace/test-workspace-id/clip.mp4', + type: 'application/octet-stream', + size: 300_000_000, + updatedAt: new Date('2026-10-31T20:28:58Z'), + contentUpdatedAt: new Date('2026-10-31T20:28:58Z'), + } + + function serveWorkspaceFile( + file: typeof video, + headers: Record = {}, + query = '' + ) { mockResolveStoredFileContext.mockResolvedValue('workspace') mockParseWorkspaceFileKey.mockReturnValue('test-workspace-id') mockAuthenticateWorkspaceFile.mockResolvedValue(principal) - mockReadWorkspaceFileRecordByKey.mockResolvedValue({ - file: { - id: 'file-1', - workspaceId: 'test-workspace-id', - name: 'export.csv', - key: 'workspace/test-workspace-id/export.csv', - type: 'application/vnd.ms-excel', - size: 3, - contentUpdatedAt: new Date('2026-09-06'), - }, - }) - fileUtilsServerMockFns.mockDownloadServableFileFromStorage.mockResolvedValue({ - buffer: Buffer.from('a,b'), - contentType: 'application/vnd.ms-excel', - }) + mockReadWorkspaceFileRecordByKey.mockResolvedValue({ file }) + const segments = file.key.split('/') + return GET( + new NextRequest(`http://localhost:3000/api/files/serve/${file.key}${query}`, { headers }), + createRouteContext({ path: segments }) + ) + } - const response = await GET( - createMockRequest({ - url: 'http://localhost:3000/api/files/serve/workspace/test-workspace-id/export.csv', - }), - createRouteContext({ path: ['workspace', 'test-workspace-id', 'export.csv'] }) + it('streams a file past the transfer ceiling instead of refusing it', async () => { + const response = await serveWorkspaceFile(video) + + expect(response.status).toBe(200) + expect(response.headers.get('Content-Length')).toBe(String(video.size)) + expect(response.headers.get('Content-Type')).toBe('video/mp4') + expect(response.headers.get('Accept-Ranges')).toBe('bytes') + expect(await response.text()).toBe('stored bytes') + expect(mockStreamAuthorizedWorkspaceFile).toHaveBeenCalledWith( + expect.objectContaining({ file: video, principal, range: undefined }) + ) + expect(fileUtilsServerMockFns.mockDownloadServableFileFromStorage).not.toHaveBeenCalled() + }) + + it('rechecks access through the same use case that admitted the file', async () => { + await serveWorkspaceFile(video) + const [{ authorize }] = mockStreamAuthorizedWorkspaceFile.mock.calls[0] + await authorize() + expect(mockAuthorizeWorkspaceFileRecordByKey).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + input: { key: video.key, assertedWorkspaceId: 'test-workspace-id' }, + }) + ) + }) + + it('answers a byte range with 206 and reads only that span', async () => { + const response = await serveWorkspaceFile(video, { Range: 'bytes=100-199' }) + + expect(response.status).toBe(206) + expect(response.headers.get('Content-Range')).toBe(`bytes 100-199/${video.size}`) + expect(response.headers.get('Content-Length')).toBe('100') + expect(mockStreamAuthorizedWorkspaceFile).toHaveBeenCalledWith( + expect.objectContaining({ range: { start: 100, end: 199 } }) ) + }) + + it('refuses a range past the end with 416 without opening storage', async () => { + const response = await serveWorkspaceFile(video, { Range: `bytes=${video.size}-` }) + + expect(response.status).toBe(416) + expect(response.headers.get('Content-Range')).toBe(`bytes */${video.size}`) + expect(mockStreamAuthorizedWorkspaceFile).not.toHaveBeenCalled() + }) + + it('answers a revalidation of unchanged bytes with 304 without opening storage', async () => { + const first = await serveWorkspaceFile(video) + const etag = first.headers.get('ETag') + expect(etag).toBeTruthy() + mockStreamAuthorizedWorkspaceFile.mockClear() + + const response = await serveWorkspaceFile(video, { 'If-None-Match': etag ?? '' }) + + expect(response.status).toBe(304) + expect(mockStreamAuthorizedWorkspaceFile).not.toHaveBeenCalled() + }) + + it('stops matching once a content write moves the file to new bytes', async () => { + const etag = (await serveWorkspaceFile(video)).headers.get('ETag') ?? '' + const rewritten = { + ...video, + key: 'workspace/test-workspace-id/clip-v2.mp4', + contentUpdatedAt: new Date('2026-11-01T00:00:00Z'), + } + + const revalidated = await serveWorkspaceFile(rewritten, { 'If-None-Match': etag }) + const resumed = await serveWorkspaceFile(rewritten, { Range: 'bytes=0-9', 'If-Range': etag }) + + expect(revalidated.status).toBe(200) + expect(resumed.status).toBe(200) + expect(resumed.headers.get('Content-Length')).toBe(String(video.size)) + }) + + it('serves stored bytes under the type their extension names, not the recorded type', async () => { + const response = await serveWorkspaceFile({ + ...video, + name: 'export.csv', + key: 'workspace/test-workspace-id/export.csv', + type: 'application/vnd.ms-excel', + size: 3, + }) expect(response.status).toBe(200) expect(response.headers.get('Content-Type')).toBe('text/csv') }) + + it('streams the raw source of a page rather than rendering it', async () => { + mockGetContentType.mockReturnValue('text/html') + const response = await serveWorkspaceFile( + { ...video, name: 'page.html', key: 'workspace/test-workspace-id/page.html', size: 12 }, + {}, + '?raw=1' + ) + + expect(response.status).toBe(200) + expect(await response.text()).toBe('stored bytes') + }) + + it.each([ + ['a generated document', { name: 'report.pdf', type: 'text/x-pdflibjs' }, ''], + ['a page', { name: 'page.html', type: 'text/html' }, ''], + [ + 'a preview, which may be a HEIC transcode', + { name: 'photo.jpg', type: 'image/jpeg' }, + '?preview=1', + ], + ])('keeps %s on the buffered path', async (_label, override, query) => { + fileUtilsServerMockFns.mockDownloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('rendered'), + contentType: 'application/octet-stream', + }) + await serveWorkspaceFile( + { ...video, ...override, key: `workspace/test-workspace-id/${override.name}`, size: 8 }, + {}, + query + ) + + expect(mockStreamAuthorizedWorkspaceFile).not.toHaveBeenCalled() + expect(fileUtilsServerMockFns.mockDownloadServableFileFromStorage).toHaveBeenCalled() + }) + + describe('outside the workspace-file model', () => { + const key = 'workspace/ws/1790000000000-0123456789abcdef-clip.mp4' + + function serveAttachment(headers: Record = {}) { + mockIsUsingCloudStorage.mockReturnValue(true) + storageServiceMockFns.mockDownloadFileStream.mockImplementation(async () => + Readable.from([Buffer.from('attachment')]) + ) + return GET( + new NextRequest(`http://localhost:3000/api/files/serve/${key}`, { headers }), + createRouteContext({ path: key.split('/') }) + ) + } + + it('streams a range of an attachment sized by storage, never buffering it', async () => { + storageServiceMockFns.mockHeadObject.mockResolvedValue({ + size: 500_000_000, + version: '"v1"', + }) + + const response = await serveAttachment({ Range: 'bytes=0-1023' }) + + expect(response.status).toBe(206) + expect(response.headers.get('Content-Range')).toBe('bytes 0-1023/500000000') + expect(response.headers.get('ETag')).toBeTruthy() + expect(await response.text()).toBe('attachment') + expect(storageServiceMockFns.mockDownloadFileStream).toHaveBeenCalledWith({ + key, + context: 'mothership', + range: { start: 0, end: 1023 }, + }) + expect(storageServiceMockFns.mockDownloadFile).not.toHaveBeenCalled() + }) + + it('still checks access before asking storage anything', async () => { + mockVerifyFileAccess.mockResolvedValue(false) + + const response = await serveAttachment() + + expect(response.status).toBe(404) + expect(storageServiceMockFns.mockHeadObject).not.toHaveBeenCalled() + expect(storageServiceMockFns.mockDownloadFileStream).not.toHaveBeenCalled() + }) + + it('answers 404 for an object storage no longer has', async () => { + storageServiceMockFns.mockHeadObject.mockResolvedValue(null) + + const response = await serveAttachment() + + expect(response.status).toBe(404) + expect(storageServiceMockFns.mockDownloadFileStream).not.toHaveBeenCalled() + }) + + it('sends everything when storage offers no version to resume against', async () => { + storageServiceMockFns.mockHeadObject.mockResolvedValue({ size: 10 }) + + const response = await serveAttachment({ Range: 'bytes=0-4', 'If-Range': '"v1"' }) + + expect(response.status).toBe(200) + expect(response.headers.get('ETag')).toBeNull() + expect(response.headers.get('Content-Length')).toBe('10') + }) + }) }) }) diff --git a/apps/sim/app/api/files/serve/[...path]/route.ts b/apps/sim/app/api/files/serve/[...path]/route.ts index ec99bbfb84b..70e666c0479 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.ts @@ -11,6 +11,7 @@ import { } from '@/lib/api/server/routes' import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' +import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { CopilotFiles, isUsingCloudStorage } from '@/lib/uploads' @@ -18,21 +19,27 @@ import type { StorageContext } from '@/lib/uploads/config' import { readOrganizationChatAttachment } from '@/lib/uploads/contexts/organization-assistant/application' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { parseWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/file-manager' -import { downloadFile } from '@/lib/uploads/core/storage-service' +import { downloadFile, downloadFileStream, headObject } from '@/lib/uploads/core/storage-service' import { resolveServableDocBytes } from '@/lib/uploads/documents/compile' import { DocCompileUserError } from '@/lib/uploads/documents/compile-error' -import { FILE_CACHE_CONTROL, workspaceFileCacheControl } from '@/lib/uploads/server/delivery' +import { + FILE_CACHE_CONTROL, + storedObjectEtag, + workspaceFileCacheControl, +} from '@/lib/uploads/server/delivery' import { resolveServableImageBytes } from '@/lib/uploads/server/image-derivative' import { resolveStoredFileContext } from '@/lib/uploads/server/metadata' import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { inferContextFromKey, resolveEffectiveMimeType } from '@/lib/uploads/utils/file' +import { + inferContextFromKey, + isRenderableDocumentName, + resolveEffectiveMimeType, +} from '@/lib/uploads/utils/file' import { internalWorkspaceFileServeAuth } from '@/lib/workspace-files/api' import { fetchAuthorizedServableWorkspaceFileBuffer } from '@/lib/workspace-files/application/fetch-servable-workspace-file-buffer' import { finishFileDelivery } from '@/lib/workspace-files/application/finish-file-delivery' -import { - readWorkspaceFileContentByKey, - readWorkspaceFileRecordByKey, -} from '@/lib/workspace-files/application/read-workspace-file-content-by-key' +import { readWorkspaceFileRecordByKey } from '@/lib/workspace-files/application/read-workspace-file-record-by-key' +import { streamAuthorizedWorkspaceFile } from '@/lib/workspace-files/application/stream-authorized-workspace-file' import { isSimPageSource, SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' import { renderSimPageDocumentWithAssets } from '@/lib/workspace-files/page-document.server' import { type KnowledgeFileAccess, verifyFileAccess } from '@/app/api/files/authorization' @@ -40,6 +47,7 @@ import { createConditionalFileResponse, createFileErrorResponse, createFileResponse, + createStoredFileStreamResponse, FileNotFoundError, type FileResponse, findLocalFile, @@ -191,6 +199,73 @@ async function resolveTransformedBytes(params: { } } +/** + * Whether a request is answered with the stored object byte-for-byte, so its bytes can stream from + * storage instead of being read whole and capped at the transfer ceiling. + * + * Every other answer is produced from the bytes and stays buffered: a Sim page renders, a generated + * document compiles, and a `preview` may be a HEIC transcode — sniffed from the content, so no name + * or type rules it out. What streams is exactly what {@link resolveServableBytes} and the + * workspace resolver would have passed through unchanged. + */ +function servesStoredObject( + filename: string, + recordedType: string | undefined, + options: ServeOptions +): boolean { + if (options.raw) return true + if (options.preview) return false + if (recordedType === SIM_PAGE_CONTENT_TYPE || filename.toLowerCase().endsWith('.html')) { + return false + } + return !isRenderableDocumentName(filename) +} + +/** + * The type a stored object streams under: what the buffered path names the same bytes — the + * extension's type for a served file, the route's own table for `raw`. + */ +function storedObjectContentType(filename: string, options: ServeOptions): string { + return options.raw ? getContentType(filename) : resolveEffectiveMimeType(undefined, filename) +} + +/** + * Streams a stored object outside the workspace-file model (execution outputs, chat and copilot + * attachments, knowledge documents) when the request is answered with it as-is, or returns null + * for the buffered path. The object is its only record, so storage is asked for its length and + * version, and the version is what proves a revalidation still holds. + */ +async function streamStoredObject(params: { + key: string + context: StorageContext + filename: string + options: ServeOptions + request: NextRequest +}): Promise { + const { key, context, filename, options } = params + if (!servesStoredObject(filename, undefined, options)) return null + const stored = await headObject(key, context) + if (!stored) throw new FileNotFoundError(`File not found: ${key}`) + + logger.info('Streaming stored file', { key, context, size: stored.size }) + return createStoredFileStreamResponse( + { + filename, + contentType: storedObjectContentType(filename, options), + cacheControl: + resolveServeCacheControl(options.versioned, context, false) ?? FILE_CACHE_CONTROL.private, + size: stored.size, + etag: + stored.version === undefined + ? undefined + : storedObjectEtag([context, key, stored.size, stored.version]), + open: async (range) => + nodeReadableToWebStream(await downloadFileStream({ key, context, range })), + }, + params.request.headers + ) +} + const STORAGE_KEY_PREFIX_RE = /^\d{13}-[a-z0-9]{7}-/ function stripStorageKeyPrefix(segment: string): string { @@ -372,10 +447,9 @@ export const GET = withRouteHandler( cloudKey, userId, options, - request.signal, + request, storageContext, - knowledgeAccess, - request.method === 'HEAD' + knowledgeAccess ) } @@ -383,10 +457,9 @@ export const GET = withRouteHandler( cloudKey, userId, options, - request.signal, + request, storageContext, - knowledgeAccess, - request.method === 'HEAD' + knowledgeAccess ) } catch (error) { if (request.signal.aborted) throw error @@ -471,40 +544,47 @@ async function handleWorkspaceFile( headers: { 'Cache-Control': FILE_CACHE_CONTROL.noStore, 'X-Content-Type-Options': 'nosniff' }, }) } - let file: WorkspaceFileRecord - let resolved: ServableBytes - if (options.raw) { - const read = await readWorkspaceFileContentByKey.execute({ principal, input, request }) - file = read.file - resolved = { - buffer: read.content, - contentType: getContentType(file.name), - dependsOnReferencedFiles: false, - } - } else { - file = (await readWorkspaceFileRecordByKey.execute({ principal, input, request })).file - const rendered = await fetchAuthorizedServableWorkspaceFileBuffer(file, principal, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - signal: request.signal, - }) - const preview = options.preview ? await resolveServableImageBytes(rendered.buffer, key) : null - resolved = { - buffer: preview?.buffer ?? rendered.buffer, - contentType: preview?.contentType ?? renderedWorkspaceContentType(file, rendered.contentType), - dependsOnReferencedFiles: rendered.dependsOnReferencedFiles, - } - assertKnownSizeWithinLimit( - resolved.buffer.length, - MAX_BUFFERED_TRANSFER_BYTES, - 'served file response' + const { file } = await readWorkspaceFileRecordByKey.execute({ principal, input, request }) + const authorize = () => readWorkspaceFileRecordByKey.authorize({ principal, input, request }) + + if (servesStoredObject(file.name, file.type, options)) { + logger.info('Streaming workspace file', { fileId: file.id, workspaceId, size: file.size }) + return createStoredFileStreamResponse( + { + filename: file.name, + contentType: storedObjectContentType(file.name, options), + cacheControl: workspaceFileCacheControl(options.versioned), + size: file.size, + // A content write uploads under a fresh key and stamps `contentUpdatedAt`, so the pair + // changes whenever the bytes do. + etag: storedObjectEtag([ + file.key, + file.size, + (file.contentUpdatedAt ?? file.updatedAt).getTime(), + ]), + open: (range) => streamAuthorizedWorkspaceFile({ file, principal, authorize, range }), + }, + request.headers ) - await finishFileDelivery({ - principal, - authorize: () => readWorkspaceFileRecordByKey.authorize({ principal, input, request }), - receipt: rendered.receipt, - }) } + const rendered = await fetchAuthorizedServableWorkspaceFileBuffer(file, principal, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: request.signal, + }) + const preview = options.preview ? await resolveServableImageBytes(rendered.buffer, key) : null + const resolved: ServableBytes = { + buffer: preview?.buffer ?? rendered.buffer, + contentType: preview?.contentType ?? renderedWorkspaceContentType(file, rendered.contentType), + dependsOnReferencedFiles: rendered.dependsOnReferencedFiles, + } + assertKnownSizeWithinLimit( + resolved.buffer.length, + MAX_BUFFERED_TRANSFER_BYTES, + 'served file response' + ) + await finishFileDelivery({ principal, authorize, receipt: rendered.receipt }) + logger.info('Workspace file served', { fileId: file.id, workspaceId, @@ -529,12 +609,12 @@ async function handleLocalFile( filename: string, userId: string, options: ServeOptions, - signal: AbortSignal | undefined, + request: NextRequest, context: StorageContext, - knowledgeAccess: KnowledgeFileAccess | undefined, - head: boolean + knowledgeAccess: KnowledgeFileAccess | undefined ): Promise { const ownerKey = `user:${userId}` + const { signal } = request try { const hasAccess = await verifyFileAccess( filename, @@ -549,7 +629,7 @@ async function handleLocalFile( throw new FileNotFoundError(`File not found: ${filename}`) } - if (head) + if (request.method === 'HEAD') return new NextResponse(null, { headers: { 'Cache-Control': FILE_CACHE_CONTROL.noStore, @@ -562,13 +642,22 @@ async function handleLocalFile( throw new FileNotFoundError(`File not found: ${filename}`) } + const segment = filename.split('/').pop() || filename + const displayName = stripStorageKeyPrefix(segment) + const streamed = await streamStoredObject({ + key: filename, + context, + filename: displayName, + options, + request, + }) + if (streamed) return streamed + const rawBuffer = await readLocalFileWithinLimit( filePath, MAX_BUFFERED_TRANSFER_BYTES, 'served file' ) - const segment = filename.split('/').pop() || filename - const displayName = stripStorageKeyPrefix(segment) const workspaceId = getWorkspaceIdForCompile(filename) const { buffer: fileBuffer, @@ -610,12 +699,12 @@ async function handleCloudProxy( cloudKey: string, userId: string, options: ServeOptions, - signal: AbortSignal | undefined, + request: NextRequest, context: StorageContext, - knowledgeAccess: KnowledgeFileAccess | undefined, - head: boolean + knowledgeAccess: KnowledgeFileAccess | undefined ): Promise { const ownerKey = `user:${userId}` + const { signal } = request try { logger.info(`Resolved context: ${context} for key: ${cloudKey}`) @@ -632,13 +721,24 @@ async function handleCloudProxy( throw new FileNotFoundError(`File not found: ${cloudKey}`) } - if (head) + if (request.method === 'HEAD') return new NextResponse(null, { headers: { 'Cache-Control': FILE_CACHE_CONTROL.noStore, 'X-Content-Type-Options': 'nosniff', }, }) + const segment = cloudKey.split('/').pop() || 'download' + const displayName = stripStorageKeyPrefix(segment) + const streamed = await streamStoredObject({ + key: cloudKey, + context, + filename: displayName, + options, + request, + }) + if (streamed) return streamed + let rawBuffer: Buffer if (context === 'copilot') { @@ -652,9 +752,6 @@ async function handleCloudProxy( maxBytes: MAX_BUFFERED_TRANSFER_BYTES, }) } - - const segment = cloudKey.split('/').pop() || 'download' - const displayName = stripStorageKeyPrefix(segment) const workspaceId = getWorkspaceIdForCompile(cloudKey) const { buffer: fileBuffer, diff --git a/apps/sim/app/api/files/utils.ts b/apps/sim/app/api/files/utils.ts index f8b018c6c7f..ae79767abee 100644 --- a/apps/sim/app/api/files/utils.ts +++ b/apps/sim/app/api/files/utils.ts @@ -4,11 +4,13 @@ import { isPayloadSizeLimitError, readNodeStreamToBufferWithLimit, } from '@/lib/core/utils/stream-limits' +import { resolveByteRange } from '@/lib/uploads/server/byte-range' import { bufferedRepresentationEtag, FILE_CACHE_CONTROL, fileDeliveryHeaders, } from '@/lib/uploads/server/delivery' +import type { ByteRange } from '@/lib/uploads/shared/types' import { sanitizeFileKey } from '@/lib/uploads/utils/file' const logger = createLogger('FilesUtils') @@ -218,6 +220,75 @@ export function createConditionalFileResponse( return response } +/** A file answered straight from its stored object, read only as far as the response needs. */ +interface StoredFileStream { + filename: string + contentType: string + cacheControl: string + /** The stored object's exact length, which `Content-Length` and every range are measured against. */ + size: number + /** + * Strong validator for the stored bytes. Without one, a revalidation is never answered 304 and + * `If-Range` never matches, so a resumed transfer restarts from the first byte. + */ + etag?: string + /** Opens the bytes to send: the whole object, or one inclusive span of it. */ + open(range: ByteRange | undefined): Promise> +} + +/** + * Answers a request from a stored object without materializing it: a 304 when the client already + * holds these bytes, a 206 for the one byte range it asked for, a 416 for a range past the end, and + * otherwise a 200 streaming the whole object. + * + * Ranges are what let a media element seek, and Safari will not play a video at all from a server + * that refuses them. A rejected range is answered before storage is opened, so it costs no read. + */ +export async function createStoredFileStreamResponse( + file: StoredFileStream, + requestHeaders: Headers +): Promise { + const headers = fileDeliveryHeaders(file) + headers.set('Accept-Ranges', 'bytes') + if (file.etag) headers.set('ETag', file.etag) + + if (file.etag && ifNoneMatchHolds(requestHeaders.get('if-none-match'), file.etag)) { + return new NextResponse(null, { status: 304, headers }) + } + + const resolution = resolveByteRange({ + range: requestHeaders.get('range'), + ifRange: requestHeaders.get('if-range'), + etag: file.etag, + size: file.size, + }) + + if (resolution.kind === 'unsatisfiable') { + return NextResponse.json( + { error: 'RangeNotSatisfiable', message: 'Requested range is outside the file' }, + { + status: 416, + headers: { + 'Accept-Ranges': 'bytes', + 'Cache-Control': FILE_CACHE_CONTROL.noStore, + 'Content-Range': `bytes */${file.size}`, + 'X-Content-Type-Options': 'nosniff', + }, + } + ) + } + + if (resolution.kind === 'full') { + headers.set('Content-Length', String(file.size)) + return new NextResponse(await file.open(undefined), { status: 200, headers }) + } + + const { range } = resolution + headers.set('Content-Length', String(range.end - range.start + 1)) + headers.set('Content-Range', `bytes ${range.start}-${range.end}/${file.size}`) + return new NextResponse(await file.open(range), { status: 206, headers }) +} + export function createFileErrorResponse(error: Error, status = 500): NextResponse { const statusCode = error instanceof FileNotFoundError diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index 1332f42984d..99b52a3285f 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -11,6 +11,7 @@ import { import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { resolveMediaMimeType } from '@/lib/uploads/utils/file' import { + useReresolveFileRecord, useWorkspaceFileBinary, useWorkspaceFileContent, useWorkspaceImageDimensionsAdapter, @@ -19,6 +20,7 @@ import { createWorkspaceFileContentSource, type FileContentSource, FileContentSourceProvider, + useFileContentSource, } from '@/hooks/use-file-content-source' import { CsvTablePreview } from './csv-table-preview' import { DocxPreview } from './docx-preview' @@ -408,20 +410,99 @@ function useBlobUrl(workspaceId: string, fileId: string, fileKey: string) { return { fileData, isLoading, error, blobUrl, replaceBlobUrl } } +interface MediaPreviewProps { + file: WorkspaceFileRecord + workspaceId: string + kind: 'audio' | 'video' +} + /** - * Shared blob-backed preview for audio and video files — the fetch, blob-URL - * lifecycle, and error/loading handling are identical; only the rendered - * player differs. + * Audio and video preview. Where the content source answers byte ranges the player streams straight + * from its URL, so playback and seeking start while the file downloads, at any size. Elsewhere the + * file is fetched whole into a blob first. */ -const MediaPreview = memo(function MediaPreview({ +const MediaPreview = memo(function MediaPreview({ file, workspaceId, kind }: MediaPreviewProps) { + const source = useFileContentSource() + if (!source.servesByteRanges) { + return + } + /** `v` lets the browser cache the bytes across re-opens; a content write moves the key. */ + const src = source.buildUrl(file.key, { + version: Number(new Date(file.updatedAt)) || file.size, + }) + /** Keyed by URL so a re-resolved key starts the player, and its load state, afresh. */ + return ( + + ) +}) + +function StreamedMediaPlayer({ + src, file, workspaceId, kind, -}: { - file: WorkspaceFileRecord - workspaceId: string - kind: 'audio' | 'video' -}) { +}: MediaPreviewProps & { src: string }) { + const [status, setStatus] = useState<'loading' | 'ready' | 'unplayable' | 'unavailable'>( + 'loading' + ) + const reresolveRecord = useReresolveFileRecord(workspaceId, file.id) + const diagnosing = useRef(false) + + /** + * A media element reports only that it failed, not why, so the URL is asked directly. A 404 means + * the key was replaced; the record is re-resolved, and a moved key re-renders this preview at the + * new URL. Bytes that are there but would not play are the browser's format support. + */ + const handleError = useCallback(async () => { + if (diagnosing.current) return + diagnosing.current = true + // boundary-raw-fetch: HEAD probe of the binary serve path, which has no contract + const probe = await fetch(src, { method: 'HEAD', cache: 'no-store' }).catch(() => null) + if (probe?.status === 404) await reresolveRecord() + setStatus(probe?.ok ? 'unplayable' : 'unavailable') + }, [src, reresolveRecord]) + + /** + * The element starts loading as soon as it is in the document, which can be before hydration + * attaches its listeners — a small file then finishes before `loadedmetadata` is heard. Reading + * the element's own state on mount covers what the listeners missed. + */ + const syncWithElement = useCallback( + (media: HTMLMediaElement | null) => { + if (!media || status !== 'loading') return + if (media.error) void handleError() + else if (media.readyState >= HTMLMediaElement.HAVE_METADATA) setStatus('ready') + }, + [status, handleError] + ) + + if (status === 'unplayable') { + return ( + + ) + } + if (status === 'unavailable') { + return + } + + return ( + setStatus('ready')} + onError={status === 'loading' ? handleError : undefined} + /> + ) +} + +/** Fetches the whole file into a blob, for content sources that send only whole bodies. */ +function BufferedMediaPreview({ file, workspaceId, kind }: MediaPreviewProps) { const { fileData, isLoading, @@ -444,27 +525,66 @@ const MediaPreview = memo(function MediaPreview({ return } + return +} + +interface MediaPlayerProps { + file: WorkspaceFileRecord + kind: 'audio' | 'video' + src: string | null + /** Covers the player until it can describe the media, for players that load in place. */ + loading?: boolean + mediaRef?: (media: HTMLMediaElement | null) => void + onLoadedMetadata?: () => void + onError?: () => void +} + +function MediaPlayer({ + file, + kind, + src, + loading, + mediaRef, + onLoadedMetadata, + onError, +}: MediaPlayerProps) { if (kind === 'audio') { return ( -
+

{file.name}

- {blobUrl && ( + {src && ( // biome-ignore lint/a11y/useMediaCaption: audio from workspace files -
) } return ( -
- {blobUrl && ( +
+ {src && ( // biome-ignore lint/a11y/useMediaCaption: video from workspace files -
) -}) +} diff --git a/apps/sim/executor/handlers/agent/memory-harness.integration.ts b/apps/sim/executor/handlers/agent/memory-harness.integration.ts index c6fd1683db0..d03d64dcd57 100644 --- a/apps/sim/executor/handlers/agent/memory-harness.integration.ts +++ b/apps/sim/executor/handlers/agent/memory-harness.integration.ts @@ -83,7 +83,7 @@ import { } from '@/lib/uploads/contexts/workspace/file-secret-provenance' import { deleteFile, uploadFile } from '@/lib/uploads/core/storage-service' import { insertImmutableFileMetadata } from '@/lib/uploads/server/metadata' -import { readWorkspaceFileRecordByKey } from '@/lib/workspace-files/application/read-workspace-file-content-by-key' +import { readWorkspaceFileRecordByKey } from '@/lib/workspace-files/application/read-workspace-file-record-by-key' import { AgentBlockHandler } from '@/executor/handlers/agent/handler' import type { AgentInputs, Message } from '@/executor/handlers/agent/types' import type { ExecutionContext, StreamingExecution, UserFile } from '@/executor/types' diff --git a/apps/sim/hooks/queries/workspace-files.ts b/apps/sim/hooks/queries/workspace-files.ts index 3fd234a473b..59f8aa70f1f 100644 --- a/apps/sim/hooks/queries/workspace-files.ts +++ b/apps/sim/hooks/queries/workspace-files.ts @@ -778,6 +778,14 @@ function fileAddressFilter(workspaceId: string, fileId: string) { } } +/** Refetches every active read that can have handed out `fileId`'s storage key. */ +function reresolveFileRecord(queryClient: QueryClient, workspaceId: string, fileId: string) { + return queryClient.refetchQueries( + { ...fileAddressFilter(workspaceId, fileId), type: 'active' }, + { cancelRefetch: true } + ) +} + /** * Re-resolve a file's address after a read found its storage key superseded. * @@ -809,17 +817,25 @@ function useStaleKeyRecovery(workspaceId: string, fileId: string): (error: unkno // // `cancelRefetch` because a re-resolution has to OBSERVE the rotation: a record read already in // flight was started before it, so it can only hand back the key we already know is dead. - void Promise.resolve().then(() => - queryClient.refetchQueries( - { ...fileAddressFilter(workspaceId, fileId), type: 'active' }, - { cancelRefetch: true } - ) - ) + void Promise.resolve().then(() => reresolveFileRecord(queryClient, workspaceId, fileId)) }, [queryClient, workspaceId, fileId] ) } +/** + * Re-resolves a file's record for a reader that loads bytes outside react-query — a media element + * streaming from the serve URL — the way a content query does on a 404 from a replaced key. Resolves + * once the record is current: if it moved, the reader re-renders at the new key's URL. + */ +export function useReresolveFileRecord(workspaceId: string, fileId: string): () => Promise { + const queryClient = useQueryClient() + return useCallback( + () => reresolveFileRecord(queryClient, workspaceId, fileId), + [queryClient, workspaceId, fileId] + ) +} + /** * Fetch file content as text via a content-source URL */ diff --git a/apps/sim/hooks/use-file-content-source.tsx b/apps/sim/hooks/use-file-content-source.tsx index bcedf42bfd6..9dd838cc9b6 100644 --- a/apps/sim/hooks/use-file-content-source.tsx +++ b/apps/sim/hooks/use-file-content-source.tsx @@ -63,6 +63,12 @@ export interface ImageDimensionsSource { */ export interface FileContentSource { buildUrl: (key: string, opts?: FileContentUrlOptions) => string + /** + * The URLs {@link buildUrl} returns answer byte-range requests, so a media element can play and + * seek a file straight from them while it downloads. Absent where the route sends only whole + * bodies, from which a player can stream only start to end — and Safari not at all. + */ + servesByteRanges?: boolean /** * Map an embedded image `src` to a display URL scoped to the current context: the in-app source * points at the workspace-scoped inline route, the public source at the token-scoped cascade route. @@ -125,6 +131,7 @@ export function createWorkspaceFileContentSource( `/api/workspaces/${workspaceId}/files/inline`, { entityType: 'workspace', entityId: workspaceId } ), + servesByteRanges: true, ...imageDimensions, } } @@ -181,6 +188,7 @@ export function createProjectFileContentSource( export const workspaceFileContentSource: FileContentSource = { buildUrl: buildServeUrl, resolveImageSrc: (src) => src, + servesByteRanges: true, } const FileContentSourceContext = createContext(workspaceFileContentSource) diff --git a/apps/sim/lib/uploads/client/download.test.ts b/apps/sim/lib/uploads/client/download.test.ts index d88acc3ef00..b3d7883061b 100644 --- a/apps/sim/lib/uploads/client/download.test.ts +++ b/apps/sim/lib/uploads/client/download.test.ts @@ -121,3 +121,49 @@ describe('file download snapshots', () => { } ) }) + +describe('large file downloads', () => { + const video: WorkspaceFileRecord = { + ...file, + name: 'clip.mp4', + key: 'workspace/workspace-1/clip.mp4', + type: 'video/mp4', + size: 300_000_000, + } + let downloadedHref = '' + + beforeEach(() => { + downloadedHref = '' + vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(function () { + downloadedHref = this.getAttribute('href') ?? '' + downloadedName = this.download + click() + }) + }) + + it('hands a file too large to hold in the tab to the browser instead of buffering it', async () => { + fetchMock.mockResolvedValue(new Response(null, { status: 200 })) + + await triggerFileDownload(video) + + expect(fetchMock).toHaveBeenCalledOnce() + expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({ method: 'HEAD' }) + expect(createObjectURL).not.toHaveBeenCalled() + expect(downloadedHref).toMatch(/^\/api\/files\/serve\/workspace%2Fworkspace-1%2Fclip\.mp4\?/) + expect(downloadedName).toBe('clip.mp4') + }) + + it('surfaces a refused download in place rather than starting one that fails', async () => { + fetchMock.mockResolvedValue(new Response(null, { status: 404 })) + + await expect(triggerFileDownload(video)).rejects.toThrow('Failed to download "clip.mp4"') + expect(click).not.toHaveBeenCalled() + }) + + it('keeps fetching a file small enough to hold, so a rejection still surfaces as an error', async () => { + await triggerFileDownload({ ...video, size: 1024 }) + + expect(fetchMock.mock.calls[0]?.[1]).not.toMatchObject({ method: 'HEAD' }) + expect(createObjectURL).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/uploads/client/download.ts b/apps/sim/lib/uploads/client/download.ts index 4c6c1d1561f..c2e07aaf5a5 100644 --- a/apps/sim/lib/uploads/client/download.ts +++ b/apps/sim/lib/uploads/client/download.ts @@ -13,10 +13,18 @@ export interface FileDownloadSource { getContent: () => string | null } -export function saveBlob(blob: Blob, fileName: string): void { - const objectUrl = URL.createObjectURL(blob) +/** + * Above this a file is handed to the browser's download manager instead of being read into the + * page. Holding it as a `Blob` costs its whole size in tab memory and shows no progress until the + * last byte lands; the download manager streams it to disk. Below it, fetching first keeps a + * rejection — a document still compiling — on screen as an error rather than as a failed entry + * in the browser's downloads list. + */ +const IN_PAGE_DOWNLOAD_MAX_BYTES = 100 * 1024 * 1024 + +function clickDownloadLink(href: string, fileName: string): void { const anchor = document.createElement('a') - anchor.href = objectUrl + anchor.href = href anchor.download = fileName // Attached before clicking: a detached anchor works in current browsers, but every // other download helper in the app attaches, and a silent no-op here would look @@ -24,6 +32,11 @@ export function saveBlob(blob: Blob, fileName: string): void { document.body.appendChild(anchor) anchor.click() document.body.removeChild(anchor) +} + +export function saveBlob(blob: Blob, fileName: string): void { + const objectUrl = URL.createObjectURL(blob) + clickDownloadLink(objectUrl, fileName) // Deferred: revoking synchronously after click() can race the download starting. setTimeout(() => URL.revokeObjectURL(objectUrl), 0) } @@ -76,12 +89,21 @@ export async function triggerFileDownload( return } - const url = + const exportsMarkdown = isMarkdown && record.vfsNamespace !== 'uploads' && (record.storageContext ?? 'workspace') === 'workspace' - ? `/api/files/export/${encodeURIComponent(record.id)}` - : `/api/files/serve/${encodeURIComponent(record.key)}?context=${record.storageContext ?? 'workspace'}&t=${Date.now()}` + const url = exportsMarkdown + ? `/api/files/export/${encodeURIComponent(record.id)}` + : `/api/files/serve/${encodeURIComponent(record.key)}?context=${record.storageContext ?? 'workspace'}&t=${Date.now()}` + + if (!exportsMarkdown && record.size > IN_PAGE_DOWNLOAD_MAX_BYTES) { + // boundary-raw-fetch: HEAD probe of a binary path with no contract, so a refusal still surfaces here + const probe = await fetch(url, { method: 'HEAD', cache: 'no-store' }) + if (!probe.ok) throw new Error(`Failed to download "${record.name}"`) + clickDownloadLink(url, record.name) + return + } // boundary-raw-fetch: binary download read as a blob; these paths have no contract const response = await fetch(url, { cache: 'no-store' }) diff --git a/apps/sim/lib/uploads/core/storage-service.ts b/apps/sim/lib/uploads/core/storage-service.ts index 17972c0ee5c..269602bee22 100644 --- a/apps/sim/lib/uploads/core/storage-service.ts +++ b/apps/sim/lib/uploads/core/storage-service.ts @@ -17,6 +17,7 @@ import type { } from '@/lib/uploads/providers/google-cloud-storage/types' import type { S3Config, S3MultipartPart } from '@/lib/uploads/providers/s3/types' import type { + ByteRange, DeleteFileOptions, DownloadFileOptions, FileInfo, @@ -660,35 +661,40 @@ export async function downloadFile(options: DownloadFileOptions): Promise { - const { key, context } = options + const { key, context, range } = options const config = getStorageConfig(context) if (USE_BLOB_STORAGE) { const { downloadFromBlobStream } = await import('@/lib/uploads/providers/blob/client') - return downloadFromBlobStream(key, createBlobConfig(config)) + return downloadFromBlobStream(key, createBlobConfig(config), range) } if (USE_S3_STORAGE) { const { downloadFromS3Stream } = await import('@/lib/uploads/providers/s3/client') - return downloadFromS3Stream(key, createS3Config(config)) + return downloadFromS3Stream(key, createS3Config(config), range) } if (USE_GCS_STORAGE) { const { downloadFromGcsStream } = await import( '@/lib/uploads/providers/google-cloud-storage/client' ) - return downloadFromGcsStream(key, createGcsConfig(config)) + return downloadFromGcsStream(key, createGcsConfig(config), range) } const { createReadStream } = await import('fs') const { join } = await import('path') const { UPLOAD_DIR_SERVER } = await getLocalStorageSetup() - return createReadStream(join(UPLOAD_DIR_SERVER, sanitizeFileKey(key))) + return createReadStream(join(UPLOAD_DIR_SERVER, sanitizeFileKey(key)), range) } /** @@ -802,7 +808,8 @@ export async function headObject( const { UPLOAD_DIR_SERVER } = await getLocalStorageSetup() try { const file = await stat(join(UPLOAD_DIR_SERVER, sanitizeFileKey(key))) - return { size: file.size } + // A local write replaces the file in place, so its modification time is what changes with it. + return { size: file.size, version: `${file.mtimeMs}` } } catch (error) { const code = (error as NodeJS.ErrnoException).code if (code === 'ENOENT') return null diff --git a/apps/sim/lib/uploads/providers/blob/client.ts b/apps/sim/lib/uploads/providers/blob/client.ts index a1f4397b8a0..9fa37766b0c 100644 --- a/apps/sim/lib/uploads/providers/blob/client.ts +++ b/apps/sim/lib/uploads/providers/blob/client.ts @@ -15,6 +15,7 @@ import type { BlobConfig, } from '@/lib/uploads/providers/blob/types' import type { + ByteRange, FileInfo, MultipartCompletionPolicy, StoredObjectInfo, @@ -410,7 +411,8 @@ export async function downloadFromBlob( */ export async function downloadFromBlobStream( key: string, - customConfig?: BlobConfig + customConfig?: BlobConfig, + range?: ByteRange ): Promise { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType @@ -440,7 +442,9 @@ export async function downloadFromBlobStream( const containerClient = blobServiceClient.getContainerClient(containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) - const downloadBlockBlobResponse = await blockBlobClient.download() + const downloadBlockBlobResponse = range + ? await blockBlobClient.download(range.start, range.end - range.start + 1) + : await blockBlobClient.download() if (!downloadBlockBlobResponse.readableStreamBody) { throw new Error('Failed to get readable stream from blob download') } diff --git a/apps/sim/lib/uploads/providers/google-cloud-storage/client.ts b/apps/sim/lib/uploads/providers/google-cloud-storage/client.ts index 64e8849fcbd..80430e2de5d 100644 --- a/apps/sim/lib/uploads/providers/google-cloud-storage/client.ts +++ b/apps/sim/lib/uploads/providers/google-cloud-storage/client.ts @@ -17,6 +17,7 @@ import type { GcsPartUploadUrl, } from '@/lib/uploads/providers/google-cloud-storage/types' import type { + ByteRange, FileInfo, MultipartCompletionPolicy, StoredObjectInfo, @@ -362,11 +363,15 @@ export async function downloadFromGcs( */ export async function downloadFromGcsStream( key: string, - customConfig?: GcsConfig + customConfig?: GcsConfig, + range?: ByteRange ): Promise { const config = customConfig || { bucket: GCS_CONFIG.bucket } const storage = await getGcsClient() - return storage.bucket(config.bucket).file(key).createReadStream() + return storage + .bucket(config.bucket) + .file(key) + .createReadStream(range ? { start: range.start, end: range.end } : undefined) } /** diff --git a/apps/sim/lib/uploads/providers/s3/client.ts b/apps/sim/lib/uploads/providers/s3/client.ts index 226e439a9e4..4c6d60a5dc4 100644 --- a/apps/sim/lib/uploads/providers/s3/client.ts +++ b/apps/sim/lib/uploads/providers/s3/client.ts @@ -30,6 +30,7 @@ import type { S3PartUploadUrl, } from '@/lib/uploads/providers/s3/types' import type { + ByteRange, FileInfo, MultipartCompletionPolicy, StoredObjectInfo, @@ -276,10 +277,15 @@ export async function downloadFromS3( */ export async function downloadFromS3Stream( key: string, - customConfig?: S3Config + customConfig?: S3Config, + range?: ByteRange ): Promise { const config = customConfig || { bucket: S3_CONFIG.bucket, region: S3_CONFIG.region } - const command = new GetObjectCommand({ Bucket: config.bucket, Key: key }) + const command = new GetObjectCommand({ + Bucket: config.bucket, + Key: key, + ...(range ? { Range: `bytes=${range.start}-${range.end}` } : {}), + }) const response = await getS3Client().send(command) if (!response.Body) { throw new Error(`S3 object has no body: ${key}`) diff --git a/apps/sim/lib/uploads/server/byte-range.test.ts b/apps/sim/lib/uploads/server/byte-range.test.ts new file mode 100644 index 00000000000..f777b1f5ae4 --- /dev/null +++ b/apps/sim/lib/uploads/server/byte-range.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from 'vitest' +import { resolveByteRange } from '@/lib/uploads/server/byte-range' + +const SIZE = 1000 +const ETAG = '"abc"' + +function resolve(range: string | null, options?: { ifRange?: string | null; etag?: string }) { + return resolveByteRange({ + range, + ifRange: options?.ifRange ?? null, + etag: options && 'etag' in options ? options.etag : ETAG, + size: SIZE, + }) +} + +describe('resolveByteRange', () => { + it('serves the full representation when no range is asked for', () => { + expect(resolve(null)).toEqual({ kind: 'full' }) + expect(resolve('')).toEqual({ kind: 'full' }) + }) + + it('reads a closed range as inclusive offsets', () => { + expect(resolve('bytes=0-499')).toEqual({ kind: 'partial', range: { start: 0, end: 499 } }) + expect(resolve('bytes=10-10')).toEqual({ kind: 'partial', range: { start: 10, end: 10 } }) + }) + + it('runs an open-ended range to the last byte', () => { + expect(resolve('bytes=900-')).toEqual({ kind: 'partial', range: { start: 900, end: 999 } }) + }) + + it('reads a suffix range as the final N bytes', () => { + expect(resolve('bytes=-100')).toEqual({ kind: 'partial', range: { start: 900, end: 999 } }) + }) + + it('clamps an end past the representation to its last byte', () => { + expect(resolve('bytes=500-5000')).toEqual({ kind: 'partial', range: { start: 500, end: 999 } }) + }) + + it('serves every byte for a suffix longer than the representation', () => { + expect(resolve('bytes=-5000')).toEqual({ kind: 'partial', range: { start: 0, end: 999 } }) + }) + + it('treats the range unit case-insensitively and tolerates surrounding whitespace', () => { + expect(resolve('Bytes= 0-1 ')).toEqual({ kind: 'partial', range: { start: 0, end: 1 } }) + }) + + it('refuses a range that starts at or past the end', () => { + expect(resolve('bytes=1000-')).toEqual({ kind: 'unsatisfiable' }) + expect(resolve('bytes=1000-1200')).toEqual({ kind: 'unsatisfiable' }) + }) + + it('refuses an empty suffix', () => { + expect(resolve('bytes=-0')).toEqual({ kind: 'unsatisfiable' }) + }) + + it.each([ + 'items=0-1', + 'bytes', + 'bytes=', + 'bytes=-', + 'bytes=abc', + 'bytes=5-2', + 'bytes=1-2-3', + 'bytes=-1-2', + 'bytes=+1-2', + 'bytes=0x10-20', + 'bytes=1.5-2', + 'bytes=99999999999999999999-', + ])('ignores the malformed header %j and serves everything', (header) => { + expect(resolve(header)).toEqual({ kind: 'full' }) + }) + + it('ignores a multi-range request rather than assembling a multipart body', () => { + expect(resolve('bytes=0-1,5-6')).toEqual({ kind: 'full' }) + }) + + it('serves an empty representation whole, since no range of it exists', () => { + expect(resolveByteRange({ range: 'bytes=0-', ifRange: null, etag: ETAG, size: 0 })).toEqual({ + kind: 'full', + }) + }) + + describe('If-Range', () => { + it('honors the range while the client still holds the current bytes', () => { + expect(resolve('bytes=0-9', { ifRange: ETAG })).toEqual({ + kind: 'partial', + range: { start: 0, end: 9 }, + }) + }) + + it('sends everything once the bytes have changed, so stale parts are never spliced', () => { + expect(resolve('bytes=0-9', { ifRange: '"other"' })).toEqual({ kind: 'full' }) + }) + + it('never matches a weak validator, which cannot vouch for byte-identical parts', () => { + expect(resolve('bytes=0-9', { ifRange: `W/${ETAG}` })).toEqual({ kind: 'full' }) + }) + + it('sends everything for the date form, since no Last-Modified is ever issued', () => { + expect(resolve('bytes=0-9', { ifRange: 'Wed, 21 Oct 2015 07:28:00 GMT' })).toEqual({ + kind: 'full', + }) + }) + + it('sends everything when the representation has no validator to compare against', () => { + expect(resolve('bytes=0-9', { ifRange: ETAG, etag: undefined })).toEqual({ kind: 'full' }) + }) + + it('does not turn an unsatisfiable range into a 416 once the bytes have changed', () => { + expect(resolve('bytes=5000-', { ifRange: '"other"' })).toEqual({ kind: 'full' }) + }) + }) +}) diff --git a/apps/sim/lib/uploads/server/byte-range.ts b/apps/sim/lib/uploads/server/byte-range.ts new file mode 100644 index 00000000000..99ea6e84f9a --- /dev/null +++ b/apps/sim/lib/uploads/server/byte-range.ts @@ -0,0 +1,79 @@ +import type { ByteRange } from '@/lib/uploads/shared/types' + +/** + * What to send for a request's `Range`, per RFC 9110 §14. + * + * - `full` — the whole representation, as a 200. Also the answer to every header this declines to + * honor: a server may always ignore `Range`, so malformed and multi-range requests degrade to it. + * - `partial` — the one span asked for, as a 206. + * - `unsatisfiable` — a well-formed range that selects no byte, as a 416. + */ +type ByteRangeResolution = + | { kind: 'full' } + | { kind: 'partial'; range: ByteRange } + | { kind: 'unsatisfiable' } + +const FULL: ByteRangeResolution = { kind: 'full' } +const UNSATISFIABLE: ByteRangeResolution = { kind: 'unsatisfiable' } + +const RANGE_SPEC = /^(\d*)-(\d*)$/ + +/** An omitted offset reads as `undefined`; one too large to represent exactly is not an offset. */ +function isOffset(value: number | undefined): boolean { + return value === undefined || Number.isSafeInteger(value) +} + +/** + * Whether `If-Range` still names the bytes being served, so parts the client already holds can be + * completed rather than restarted. Only a strong entity-tag qualifies (RFC 9110 §13.1.5): a weak one + * cannot vouch for byte-identical parts, and the date form is never matched because no + * `Last-Modified` is issued. + */ +function ifRangeHolds(ifRange: string | null, etag: string | undefined): boolean { + if (ifRange === null) return true + return etag !== undefined && ifRange.trim() === etag +} + +/** + * Resolves a request's `Range` against a representation of `size` bytes. + * + * One span is served at most. A multi-range request is answered whole instead of as a + * `multipart/byteranges` body: media players and download managers ask for one span at a time, and + * the standard permits ignoring `Range` outright. An empty representation is likewise sent whole, + * because no range of it can exist. + */ +export function resolveByteRange(input: { + range: string | null + ifRange: string | null + etag: string | undefined + size: number +}): ByteRangeResolution { + const { range, size } = input + if (!range || size <= 0) return FULL + if (!ifRangeHolds(input.ifRange, input.etag)) return FULL + + const separator = range.indexOf('=') + if (separator === -1 || range.slice(0, separator).trim().toLowerCase() !== 'bytes') return FULL + + const specs = range.slice(separator + 1).split(',') + if (specs.length !== 1) return FULL + const match = RANGE_SPEC.exec(specs[0].trim()) + if (!match) return FULL + + const first = match[1] ? Number(match[1]) : undefined + const last = match[2] ? Number(match[2]) : undefined + if (!isOffset(first) || !isOffset(last)) return FULL + + if (first === undefined) { + if (last === undefined) return FULL + if (last === 0) return UNSATISFIABLE + return { kind: 'partial', range: { start: Math.max(0, size - last), end: size - 1 } } + } + + if (last !== undefined && last < first) return FULL + if (first >= size) return UNSATISFIABLE + return { + kind: 'partial', + range: { start: first, end: last === undefined ? size - 1 : Math.min(last, size - 1) }, + } +} diff --git a/apps/sim/lib/uploads/server/delivery.ts b/apps/sim/lib/uploads/server/delivery.ts index a3ed7d483cd..f62677352ce 100644 --- a/apps/sim/lib/uploads/server/delivery.ts +++ b/apps/sim/lib/uploads/server/delivery.ts @@ -150,6 +150,15 @@ export function bufferedRepresentationEtag(buffer: Buffer): string { return `"${createHash('sha256').update(buffer).digest('base64url')}"` } +/** + * Strong validator for bytes streamed straight from a stored object. Those bytes are never held, so + * they cannot be digested; the validator is derived from what identifies the object instead, and + * every part passed must change whenever the bytes do. + */ +export function storedObjectEtag(identity: readonly (string | number)[]): string { + return `"${createHash('sha256').update(identity.join('\u0000')).digest('base64url')}"` +} + /** Binary presenters share header assembly without coupling storage mechanics to a route framework. */ export function presentFileDelivery(input: { body: Buffer | ReadableStream diff --git a/apps/sim/lib/uploads/shared/types.ts b/apps/sim/lib/uploads/shared/types.ts index 097718ffccd..b99750cd190 100644 --- a/apps/sim/lib/uploads/shared/types.ts +++ b/apps/sim/lib/uploads/shared/types.ts @@ -176,3 +176,9 @@ export interface StoredObjectInfo { uploadId?: string version?: string } + +/** A span of a stored object's bytes, with both offsets inclusive as HTTP `Range` counts them. */ +export interface ByteRange { + start: number + end: number +} diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-record-by-key.test.ts similarity index 66% rename from apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts rename to apps/sim/lib/workspace-files/application/read-workspace-file-record-by-key.test.ts index d19acf2e6f2..161dfc29d43 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-record-by-key.test.ts @@ -1,5 +1,4 @@ import { - createDelegatedPrincipal, createSessionPrincipal, createWorkspaceApiKeyPrincipal, } from '@sim/testing/factories/principal.factory' @@ -9,21 +8,12 @@ import { uploadsMetadataMockFns, } from '@sim/testing/mocks/uploads-metadata.mock' import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' -import { - workspaceFileSecretProvenanceMock, - workspaceFileSecretProvenanceMockFns, -} from '@sim/testing/mocks/workspace-file-secret-provenance.mock' +import { workspaceFileSecretProvenanceMock } from '@sim/testing/mocks/workspace-file-secret-provenance.mock' import { workspaceUploadsMock, workspaceUploadsMockFns, } from '@sim/testing/mocks/workspace-uploads.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' -import { - ContentRefusedError, - observeContentDelivery, -} from '@/lib/core/application/content-delivery' -import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' -import { recheckFileReadReceipt } from '@/lib/workspace-files/read-receipt' vi.mock('@/lib/workspace-files/read-receipt', () => fileReadReceiptMock) vi.mock( @@ -37,14 +27,9 @@ vi.mock('@/lib/uploads/server/metadata', () => uploadsMetadataMock) vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { - readWorkspaceFileContentByKey, - readWorkspaceFileRecordByKey, -} from '@/lib/workspace-files/application/read-workspace-file-content-by-key' +import { readWorkspaceFileRecordByKey } from '@/lib/workspace-files/application/read-workspace-file-record-by-key' const mocks = { - fetchContent: workspaceUploadsMockFns.mockFetchWorkspaceFileBuffer, getFile: workspaceUploadsMockFns.mockGetWorkspaceFile, loadContext: workspaceUploadsMockFns.mockLoadActiveWorkspaceFileContext, resolvePermission: workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission, @@ -72,7 +57,7 @@ const file = { updatedAt: new Date('2026-08-01T00:00:00Z'), } -describe('readWorkspaceFileContentByKey', () => { +describe('readWorkspaceFileRecordByKey', () => { beforeEach(() => { mocks.getMetadata.mockResolvedValue({ id: file.id, @@ -83,51 +68,17 @@ describe('readWorkspaceFileContentByKey', () => { mocks.loadContext.mockResolvedValue(context) mocks.resolvePermission.mockResolvedValue('read') mocks.getFile.mockResolvedValue(file) - mocks.fetchContent.mockResolvedValue(Buffer.from('source')) - }) - - it('refuses bytes when final receipt provenance changes after storage opens', async () => { - const chat = createDelegatedPrincipal({ audience: WORKSPACE_FILES_DELEGATION_AUDIENCE }) - workspaceFileSecretProvenanceMockFns.mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue( - { status: 'exact', entries: [] } - ) - vi.mocked(recheckFileReadReceipt).mockResolvedValueOnce({ status: 'unknown' }) - await expect( - observeContentDelivery( - { admit: async (sources) => sources.map((source) => source.provenance.status === 'exact') }, - () => readWorkspaceFileContentByKey.execute({ principal: chat, input: { key: file.key } }) - ) - ).rejects.toBeInstanceOf(ContentRefusedError) - expect(mocks.fetchContent).toHaveBeenCalled() - }) - - it('authorizes the canonical file before returning its stored bytes', async () => { - await expect( - readWorkspaceFileContentByKey.execute({ - principal, - input: { key: file.key, assertedWorkspaceId: file.workspaceId }, - }) - ).resolves.toEqual({ file, content: Buffer.from('source') }) - - expect(mocks.getFile).toHaveBeenCalledWith(file.workspaceId, file.id, { - throwOnError: true, - includeOwnedFiles: true, - }) - expect(mocks.fetchContent).toHaveBeenCalledWith(file, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) }) it('rejects a stale key instead of serving the file current at the same ID', async () => { mocks.getFile.mockResolvedValue({ ...file, key: `${file.key}.new` }) await expect( - readWorkspaceFileContentByKey.execute({ + readWorkspaceFileRecordByKey.execute({ principal, input: { key: file.key, assertedWorkspaceId: file.workspaceId }, }) ).rejects.toMatchObject({ code: 'not_found' }) - expect(mocks.fetchContent).not.toHaveBeenCalled() }) it('authorizes an exact-key record read for a workspace API key without a human fallback', async () => { @@ -145,7 +96,6 @@ describe('readWorkspaceFileContentByKey', () => { throwOnError: true, includeOwnedFiles: true, }) - expect(mocks.fetchContent).not.toHaveBeenCalled() }) it('authorizes an actorless deployment executor by its preserved workflow authority', async () => { @@ -181,7 +131,6 @@ describe('readWorkspaceFileContentByKey', () => { ).resolves.toEqual({ file }) expect(mocks.resolvePermission).not.toHaveBeenCalled() - expect(mocks.fetchContent).not.toHaveBeenCalled() }) it('conceals an exact key asserted under a different workspace before authorization', async () => { diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-record-by-key.ts similarity index 63% rename from apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts rename to apps/sim/lib/workspace-files/application/read-workspace-file-record-by-key.ts index 94fe159d275..08f14b0b878 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-record-by-key.ts @@ -1,33 +1,23 @@ import type { Principal } from '@sim/auth/principal' -import type { AuthorizedWorkspaceUseCaseContext } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { type ActiveWorkspaceFileContext, - fetchWorkspaceFileBuffer, getWorkspaceFile, loadActiveWorkspaceFileContext, type WorkspaceFileRecord, } from '@/lib/uploads/contexts/workspace' import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { requireAdmittedWorkspaceFile } from '@/lib/workspace-files/application/admit-file-content' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { assertOwnedFileAccess } from '@/lib/workspace-files/application/file-context' -import { finishFileDelivery } from '@/lib/workspace-files/application/finish-file-delivery' import { fileOperations } from '@/lib/workspace-files/application/operations' import { OWNED_FILE_CONTEXTS, ownedFileKind } from '@/lib/workspace-files/owned-files' -import { createFileReadReceipt } from '@/lib/workspace-files/read-receipt' -export interface ReadWorkspaceFileByKeyInput { +interface ReadWorkspaceFileByKeyInput { key: string assertedWorkspaceId?: string } -export interface ReadWorkspaceFileContentByKeyResult { - file: WorkspaceFileRecord - content: Buffer -} - export interface ReadWorkspaceFileRecordByKeyResult { file: WorkspaceFileRecord } @@ -44,29 +34,6 @@ async function loadCurrentWorkspaceFileByKey( return file } -async function executeReadWorkspaceFileContentByKey({ - input, - context, - principal, -}: AuthorizedWorkspaceUseCaseContext< - typeof fileOperations.readContent, - ReadWorkspaceFileByKeyInput, - ActiveWorkspaceFileContext ->): Promise { - const file = await loadCurrentWorkspaceFileByKey(input, context) - await requireAdmittedWorkspaceFile(principal, context.workspaceId, file, 'opaque') - const content = await fetchWorkspaceFileBuffer(file, { maxBytes: MAX_BUFFERED_TRANSFER_BYTES }) - const provenance = await finishFileDelivery({ - principal, - authorize: () => readWorkspaceFileContentByKey.authorize({ principal, input }), - receipt: createFileReadReceipt({ entityType: 'workspace', entityId: context.workspaceId }, [ - { ...file, contentUpdatedAt: file.contentUpdatedAt ?? file.updatedAt }, - ]), - }) - await requireAdmittedWorkspaceFile(principal, context.workspaceId, file, 'opaque', provenance) - return { file, content } -} - /** Workspace files and owned files share the `workspace/` key prefix; the row says which. */ const KEY_READABLE_CONTEXTS = new Set(['workspace', ...OWNED_FILE_CONTEXTS]) @@ -106,9 +73,3 @@ export const readWorkspaceFileRecordByKey = defineAuthorizedWorkspaceFileUseCase return { file } }, }) - -export const readWorkspaceFileContentByKey = defineAuthorizedWorkspaceFileUseCase({ - operation: fileOperations.readContent, - resolveContext: resolveWorkspaceFileByKeyContext, - execute: executeReadWorkspaceFileContentByKey, -}) diff --git a/apps/sim/lib/workspace-files/application/read-workspace-inline-file.ts b/apps/sim/lib/workspace-files/application/read-workspace-inline-file.ts index ef45d1aaf00..69fa5c911e4 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-inline-file.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-inline-file.ts @@ -1,19 +1,16 @@ import type { AuthorizedWorkspaceUseCaseContext } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' -import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' import { type ActiveWorkspaceFileContext, getWorkspaceFile, loadActiveWorkspaceFileContext, type WorkspaceFileRecord, } from '@/lib/uploads/contexts/workspace/file-manager' -import { downloadFileStream } from '@/lib/uploads/core/storage-service' import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' import { requireAdmittedWorkspaceFile } from '@/lib/workspace-files/application/admit-file-content' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' -import { finishFileDelivery } from '@/lib/workspace-files/application/finish-file-delivery' import { fileOperations } from '@/lib/workspace-files/application/operations' -import { createFileReadReceipt } from '@/lib/workspace-files/read-receipt' +import { streamAuthorizedWorkspaceFile } from '@/lib/workspace-files/application/stream-authorized-workspace-file' export interface ReadWorkspaceInlineFileInput { workspaceId: string @@ -52,23 +49,11 @@ async function executeReadWorkspaceInlineFile({ if (!file) throw new OrchestrationError('not_found', 'Not found') await requireAdmittedWorkspaceFile(principal, context.workspaceId, file, 'opaque') - const stream = nodeReadableToWebStream( - await downloadFileStream({ key: file.key, context: 'workspace' }) - ) - const provenance = await finishFileDelivery({ + const stream = await streamAuthorizedWorkspaceFile({ + file, principal, authorize: () => readWorkspaceInlineFile.authorize({ principal, input }), - receipt: createFileReadReceipt({ entityType: 'workspace', entityId: context.workspaceId }, [ - { ...file, contentUpdatedAt: file.contentUpdatedAt ?? file.updatedAt }, - ]), - stream, }) - try { - await requireAdmittedWorkspaceFile(principal, context.workspaceId, file, 'opaque', provenance) - } catch (error) { - await stream.cancel(error).catch(() => undefined) - throw error - } return { file, stream, diff --git a/apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.test.ts b/apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.test.ts new file mode 100644 index 00000000000..afcf8a6d6cc --- /dev/null +++ b/apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.test.ts @@ -0,0 +1,107 @@ +import { Readable } from 'node:stream' +import { storageServiceMock, storageServiceMockFns } from '@sim/testing' +import { + createDelegatedPrincipal, + createSessionPrincipal, +} from '@sim/testing/factories/principal.factory' +import { fileReadReceiptMock } from '@sim/testing/mocks/file-read-receipt.mock' +import { + workspaceFileSecretProvenanceMock, + workspaceFileSecretProvenanceMockFns, +} from '@sim/testing/mocks/workspace-file-secret-provenance.mock' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + ContentRefusedError, + observeContentDelivery, +} from '@/lib/core/application/content-delivery' +import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' +import { recheckFileReadReceipt } from '@/lib/workspace-files/read-receipt' + +vi.mock('@/lib/workspace-files/read-receipt', () => fileReadReceiptMock) +vi.mock( + '@/lib/uploads/contexts/workspace/file-secret-provenance', + () => workspaceFileSecretProvenanceMock +) +vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) + +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { streamAuthorizedWorkspaceFile } from '@/lib/workspace-files/application/stream-authorized-workspace-file' + +const file = { + id: 'file-1', + workspaceId: 'workspace-1', + name: 'clip.mp4', + key: 'workspace/workspace-1/clip.mp4', + path: '/api/files/serve/workspace/workspace-1/clip.mp4', + size: 6, + type: 'video/mp4', + uploadedBy: 'user-1', + uploadedAt: new Date('2026-08-01T00:00:00Z'), + updatedAt: new Date('2026-08-01T00:00:00Z'), +} satisfies WorkspaceFileRecord + +let source: Readable + +async function readAll(stream: ReadableStream): Promise { + return Buffer.from(await new Response(stream).arrayBuffer()).toString() +} + +describe('streamAuthorizedWorkspaceFile', () => { + beforeEach(() => { + source = Readable.from([Buffer.from('stream')]) + storageServiceMockFns.mockDownloadFileStream.mockResolvedValue(source) + }) + + it('streams the stored bytes once access still holds after storage opens', async () => { + const stream = await streamAuthorizedWorkspaceFile({ + file, + principal: createSessionPrincipal(), + authorize: async () => undefined, + }) + await expect(readAll(stream)).resolves.toBe('stream') + }) + + it('reads only the requested span from storage', async () => { + await streamAuthorizedWorkspaceFile({ + file, + principal: createSessionPrincipal(), + authorize: async () => undefined, + range: { start: 2, end: 4 }, + }) + expect(storageServiceMockFns.mockDownloadFileStream).toHaveBeenCalledWith({ + key: file.key, + context: 'workspace', + range: { start: 2, end: 4 }, + }) + }) + + it('disposes the opened stream when access is revoked before it is handed out', async () => { + await expect( + streamAuthorizedWorkspaceFile({ + file, + principal: createSessionPrincipal(), + authorize: async () => { + throw new Error('revoked') + }, + }) + ).rejects.toThrow('revoked') + expect(source.destroyed).toBe(true) + }) + + it('refuses and disposes the stream when provenance changes after storage opens', async () => { + const chat = createDelegatedPrincipal({ audience: WORKSPACE_FILES_DELEGATION_AUDIENCE }) + workspaceFileSecretProvenanceMockFns.mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue( + { status: 'exact', entries: [] } + ) + vi.mocked(recheckFileReadReceipt).mockResolvedValueOnce({ status: 'unknown' }) + + await expect( + observeContentDelivery( + { admit: async (sources) => sources.map((entry) => entry.provenance.status === 'exact') }, + () => + streamAuthorizedWorkspaceFile({ file, principal: chat, authorize: async () => undefined }) + ) + ).rejects.toBeInstanceOf(ContentRefusedError) + expect(source.destroyed).toBe(true) + }) +}) diff --git a/apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.ts b/apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.ts new file mode 100644 index 00000000000..0ece44596e8 --- /dev/null +++ b/apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.ts @@ -0,0 +1,48 @@ +import type { Principal } from '@sim/auth/principal' +import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { downloadFileStream } from '@/lib/uploads/core/storage-service' +import type { ByteRange } from '@/lib/uploads/shared/types' +import { requireAdmittedWorkspaceFile } from '@/lib/workspace-files/application/admit-file-content' +import { finishFileDelivery } from '@/lib/workspace-files/application/finish-file-delivery' +import { createFileReadReceipt } from '@/lib/workspace-files/read-receipt' + +/** + * Opens an already-authorized workspace file's stored bytes for delivery, whole or as one inclusive + * span, without reading them into memory. + * + * Access is rechecked once storage has opened and before the stream is handed out, as every + * streamed read does: a revocation, a content write, or a provenance change landing in between + * disposes the stream instead of publishing it. `authorize` is the caller's own use-case check, so + * the recheck applies the same policy that admitted the file. + */ +export async function streamAuthorizedWorkspaceFile(input: { + file: WorkspaceFileRecord + principal: Principal + authorize(): Promise + range?: ByteRange +}): Promise> { + const { file, principal } = input + const stream = nodeReadableToWebStream( + await downloadFileStream({ + key: file.key, + context: file.storageContext ?? 'workspace', + range: input.range, + }) + ) + const provenance = await finishFileDelivery({ + principal, + authorize: input.authorize, + receipt: createFileReadReceipt({ entityType: 'workspace', entityId: file.workspaceId }, [ + { ...file, contentUpdatedAt: file.contentUpdatedAt ?? file.updatedAt }, + ]), + stream, + }) + try { + await requireAdmittedWorkspaceFile(principal, file.workspaceId, file, 'opaque', provenance) + } catch (error) { + await stream.cancel(error).catch(() => undefined) + throw error + } + return stream +} diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index b319df489bf..e4448b682b2 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -5198,8 +5198,6 @@ "apps/sim/lib/workspace-files/application/folders.ts#ListWorkspaceFileFoldersResult", "apps/sim/lib/workspace-files/application/folders.ts#UpdateWorkspaceFileFolderResult", "apps/sim/lib/workspace-files/application/move-workspace-file-items.ts#MoveWorkspaceFileItemsResult", - "apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts#ReadWorkspaceFileByKeyInput", - "apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts#ReadWorkspaceFileContentByKeyResult", "apps/sim/lib/workspace-files/application/read-workspace-file-content.ts#ReadWorkspaceFileContentInput", "apps/sim/lib/workspace-files/application/read-workspace-file-content.ts#ReadWorkspaceFileContentResult", "apps/sim/lib/workspace-files/application/read-workspace-file-metadata.ts#ReadWorkspaceFileMetadataResult", From fc1e8ea5daa64400d6de43c0744caa556d7635eb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 31 Oct 2026 16:16:50 -0700 Subject: [PATCH 2/3] fix(files): stream finished document uploads, diagnose playback errors, prove ranges on real bytes --- .../api/files/serve/[...path]/route.test.ts | 13 +++++ .../app/api/files/serve/[...path]/route.ts | 6 +- .../components/file-viewer/file-viewer.tsx | 2 +- apps/sim/lib/uploads/client/download.test.ts | 4 +- .../stream-authorized-workspace-file.test.ts | 56 +++++++++++-------- 5 files changed, 55 insertions(+), 26 deletions(-) diff --git a/apps/sim/app/api/files/serve/[...path]/route.test.ts b/apps/sim/app/api/files/serve/[...path]/route.test.ts index 19a2a0cb4a2..d785e481f9c 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.test.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.test.ts @@ -520,6 +520,19 @@ describe('File Serve API Route', () => { expect(response.headers.get('Content-Type')).toBe('text/csv') }) + it('streams an uploaded document, which is already the finished file', async () => { + const response = await serveWorkspaceFile({ + ...video, + name: 'scan.pdf', + key: 'workspace/test-workspace-id/scan.pdf', + type: 'application/pdf', + }) + + expect(response.status).toBe(200) + expect(response.headers.get('Content-Length')).toBe(String(video.size)) + expect(fileUtilsServerMockFns.mockDownloadServableFileFromStorage).not.toHaveBeenCalled() + }) + it('streams the raw source of a page rather than rendering it', async () => { mockGetContentType.mockReturnValue('text/html') const response = await serveWorkspaceFile( diff --git a/apps/sim/app/api/files/serve/[...path]/route.ts b/apps/sim/app/api/files/serve/[...path]/route.ts index 70e666c0479..88fcccac774 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.ts @@ -32,7 +32,7 @@ import { resolveStoredFileContext } from '@/lib/uploads/server/metadata' import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { inferContextFromKey, - isRenderableDocumentName, + needsRenderedArtifact, resolveEffectiveMimeType, } from '@/lib/uploads/utils/file' import { internalWorkspaceFileServeAuth } from '@/lib/workspace-files/api' @@ -218,7 +218,9 @@ function servesStoredObject( if (recordedType === SIM_PAGE_CONTENT_TYPE || filename.toLowerCase().endsWith('.html')) { return false } - return !isRenderableDocumentName(filename) + // A recorded type tells a finished upload from a generation source; without one, every + // document name may be a source. + return !needsRenderedArtifact(recordedType, filename) } /** diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index 99b52a3285f..d743ce901ba 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -496,7 +496,7 @@ function StreamedMediaPlayer({ loading={status === 'loading'} mediaRef={syncWithElement} onLoadedMetadata={() => setStatus('ready')} - onError={status === 'loading' ? handleError : undefined} + onError={handleError} /> ) } diff --git a/apps/sim/lib/uploads/client/download.test.ts b/apps/sim/lib/uploads/client/download.test.ts index b3d7883061b..3524b6c98a2 100644 --- a/apps/sim/lib/uploads/client/download.test.ts +++ b/apps/sim/lib/uploads/client/download.test.ts @@ -134,7 +134,9 @@ describe('large file downloads', () => { beforeEach(() => { downloadedHref = '' - vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(function () { + vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(function ( + this: HTMLAnchorElement + ) { downloadedHref = this.getAttribute('href') ?? '' downloadedName = this.download click() diff --git a/apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.test.ts b/apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.test.ts index afcf8a6d6cc..38c4cf6209d 100644 --- a/apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.test.ts +++ b/apps/sim/lib/workspace-files/application/stream-authorized-workspace-file.test.ts @@ -1,10 +1,14 @@ -import { Readable } from 'node:stream' -import { storageServiceMock, storageServiceMockFns } from '@sim/testing' +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' import { createDelegatedPrincipal, createSessionPrincipal, } from '@sim/testing/factories/principal.factory' import { fileReadReceiptMock } from '@sim/testing/mocks/file-read-receipt.mock' +import { uploadsConfigMock, uploadsConfigMockFns } from '@sim/testing/mocks/uploads-config.mock' +import { uploadsMetadataMock } from '@sim/testing/mocks/uploads-metadata.mock' +import { setUploadDirServer, uploadsSetupMock } from '@sim/testing/mocks/uploads-setup.mock' import { workspaceFileSecretProvenanceMock, workspaceFileSecretProvenanceMockFns, @@ -22,60 +26,65 @@ vi.mock( '@/lib/uploads/contexts/workspace/file-secret-provenance', () => workspaceFileSecretProvenanceMock ) -vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) +vi.mock('@/lib/uploads/config', () => uploadsConfigMock) +vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock) +vi.mock('@/lib/uploads/server/metadata', () => uploadsMetadataMock) import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { streamAuthorizedWorkspaceFile } from '@/lib/workspace-files/application/stream-authorized-workspace-file' +const CONTENT = Buffer.from('0123456789abcdefghij') + const file = { id: 'file-1', workspaceId: 'workspace-1', name: 'clip.mp4', key: 'workspace/workspace-1/clip.mp4', path: '/api/files/serve/workspace/workspace-1/clip.mp4', - size: 6, + size: CONTENT.length, type: 'video/mp4', uploadedBy: 'user-1', uploadedAt: new Date('2026-08-01T00:00:00Z'), updatedAt: new Date('2026-08-01T00:00:00Z'), } satisfies WorkspaceFileRecord -let source: Readable - -async function readAll(stream: ReadableStream): Promise { - return Buffer.from(await new Response(stream).arrayBuffer()).toString() +async function readAll(stream: ReadableStream): Promise { + return Buffer.from(await new Response(stream).arrayBuffer()) } describe('streamAuthorizedWorkspaceFile', () => { beforeEach(() => { - source = Readable.from([Buffer.from('stream')]) - storageServiceMockFns.mockDownloadFileStream.mockResolvedValue(source) + const uploads = mkdtempSync(join(tmpdir(), 'stream-authorized-')) + mkdirSync(dirname(join(uploads, file.key)), { recursive: true }) + writeFileSync(join(uploads, file.key), CONTENT) + setUploadDirServer(uploads) + uploadsConfigMockFns.mockGetStorageConfig.mockReturnValue({ bucket: 'b', region: 'r' }) }) - it('streams the stored bytes once access still holds after storage opens', async () => { + it('streams every stored byte once access still holds after storage opens', async () => { const stream = await streamAuthorizedWorkspaceFile({ file, principal: createSessionPrincipal(), authorize: async () => undefined, }) - await expect(readAll(stream)).resolves.toBe('stream') + + expect(await readAll(stream)).toEqual(CONTENT) }) - it('reads only the requested span from storage', async () => { - await streamAuthorizedWorkspaceFile({ + it('reads exactly the requested inclusive span and nothing else', async () => { + const stream = await streamAuthorizedWorkspaceFile({ file, principal: createSessionPrincipal(), authorize: async () => undefined, - range: { start: 2, end: 4 }, - }) - expect(storageServiceMockFns.mockDownloadFileStream).toHaveBeenCalledWith({ - key: file.key, - context: 'workspace', - range: { start: 2, end: 4 }, + range: { start: 5, end: 9 }, }) + + expect((await readAll(stream)).toString()).toBe('56789') }) it('disposes the opened stream when access is revoked before it is handed out', async () => { + const cancel = vi.spyOn(ReadableStream.prototype, 'cancel') + await expect( streamAuthorizedWorkspaceFile({ file, @@ -85,7 +94,8 @@ describe('streamAuthorizedWorkspaceFile', () => { }, }) ).rejects.toThrow('revoked') - expect(source.destroyed).toBe(true) + expect(cancel).toHaveBeenCalled() + cancel.mockRestore() }) it('refuses and disposes the stream when provenance changes after storage opens', async () => { @@ -94,6 +104,7 @@ describe('streamAuthorizedWorkspaceFile', () => { { status: 'exact', entries: [] } ) vi.mocked(recheckFileReadReceipt).mockResolvedValueOnce({ status: 'unknown' }) + const cancel = vi.spyOn(ReadableStream.prototype, 'cancel') await expect( observeContentDelivery( @@ -102,6 +113,7 @@ describe('streamAuthorizedWorkspaceFile', () => { streamAuthorizedWorkspaceFile({ file, principal: chat, authorize: async () => undefined }) ) ).rejects.toBeInstanceOf(ContentRefusedError) - expect(source.destroyed).toBe(true) + expect(cancel).toHaveBeenCalled() + cancel.mockRestore() }) }) From 75dacb4fefe590d6aa6b52fbba0bb37c2e2a146f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 31 Oct 2026 16:28:19 -0700 Subject: [PATCH 3/3] fix(files): probe served bytes before handing off, keep generated documents in-page, version local objects by inode --- .../components/file-viewer/file-viewer.tsx | 14 +++++++---- apps/sim/lib/uploads/client/download.test.ts | 12 +++++++-- apps/sim/lib/uploads/client/download.ts | 25 ++++++++++++------- apps/sim/lib/uploads/core/storage-service.ts | 4 +-- .../lib/uploads/upload-session/provider.ts | 10 +++++--- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index d743ce901ba..71fcdf559ce 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -449,15 +449,19 @@ function StreamedMediaPlayer({ const diagnosing = useRef(false) /** - * A media element reports only that it failed, not why, so the URL is asked directly. A 404 means - * the key was replaced; the record is re-resolved, and a moved key re-renders this preview at the - * new URL. Bytes that are there but would not play are the browser's format support. + * A media element reports only that it failed, not why, so the URL is asked directly for its + * first byte, which opens storage. A 404 means the key was replaced; the record is re-resolved, + * and a moved key re-renders this preview at the new URL. Bytes that are there but would not play + * are the browser's format support. */ const handleError = useCallback(async () => { if (diagnosing.current) return diagnosing.current = true - // boundary-raw-fetch: HEAD probe of the binary serve path, which has no contract - const probe = await fetch(src, { method: 'HEAD', cache: 'no-store' }).catch(() => null) + // boundary-raw-fetch: ranged probe of the binary serve path, which has no contract + const probe = await fetch(src, { headers: { Range: 'bytes=0-0' }, cache: 'no-store' }).catch( + () => null + ) + await probe?.body?.cancel() if (probe?.status === 404) await reresolveRecord() setStatus(probe?.ok ? 'unplayable' : 'unavailable') }, [src, reresolveRecord]) diff --git a/apps/sim/lib/uploads/client/download.test.ts b/apps/sim/lib/uploads/client/download.test.ts index 3524b6c98a2..3d6311b5efa 100644 --- a/apps/sim/lib/uploads/client/download.test.ts +++ b/apps/sim/lib/uploads/client/download.test.ts @@ -149,7 +149,7 @@ describe('large file downloads', () => { await triggerFileDownload(video) expect(fetchMock).toHaveBeenCalledOnce() - expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({ method: 'HEAD' }) + expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({ headers: { Range: 'bytes=0-0' } }) expect(createObjectURL).not.toHaveBeenCalled() expect(downloadedHref).toMatch(/^\/api\/files\/serve\/workspace%2Fworkspace-1%2Fclip\.mp4\?/) expect(downloadedName).toBe('clip.mp4') @@ -165,7 +165,15 @@ describe('large file downloads', () => { it('keeps fetching a file small enough to hold, so a rejection still surfaces as an error', async () => { await triggerFileDownload({ ...video, size: 1024 }) - expect(fetchMock.mock.calls[0]?.[1]).not.toMatchObject({ method: 'HEAD' }) + expect(fetchMock.mock.calls[0]?.[1]).not.toHaveProperty('headers') + expect(createObjectURL).toHaveBeenCalledOnce() + }) + + it('keeps fetching a large generated document, which must compile before it can be served', async () => { + await triggerFileDownload({ ...video, name: 'report.pdf', type: 'text/x-pdflibjs' }) + + expect(fetchMock).toHaveBeenCalledOnce() + expect(fetchMock.mock.calls[0]?.[1]).not.toHaveProperty('headers') expect(createObjectURL).toHaveBeenCalledOnce() }) }) diff --git a/apps/sim/lib/uploads/client/download.ts b/apps/sim/lib/uploads/client/download.ts index c2e07aaf5a5..fced8c56b5d 100644 --- a/apps/sim/lib/uploads/client/download.ts +++ b/apps/sim/lib/uploads/client/download.ts @@ -3,7 +3,7 @@ import { requestRaw } from '@/lib/api/client/request' import { downloadWorkspaceFileItemsContract } from '@/lib/api/contracts/workspace-file-folders' import { exportWorkspaceFileSnapshotContract } from '@/lib/api/contracts/workspace-files' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -import { isMarkdownFile } from '@/lib/uploads/utils/file' +import { isMarkdownFile, needsRenderedArtifact } from '@/lib/uploads/utils/file' import { normalizeMimeType } from '@/lib/uploads/utils/mime' /** Action-time content from the mounted viewer, scoped so another file cannot consume it. */ @@ -14,11 +14,11 @@ export interface FileDownloadSource { } /** - * Above this a file is handed to the browser's download manager instead of being read into the - * page. Holding it as a `Blob` costs its whole size in tab memory and shows no progress until the - * last byte lands; the download manager streams it to disk. Below it, fetching first keeps a - * rejection — a document still compiling — on screen as an error rather than as a failed entry - * in the browser's downloads list. + * Above this a stored file is handed to the browser's download manager instead of being read into + * the page. Holding it as a `Blob` costs its whole size in tab memory and shows no progress until + * the last byte lands; the download manager streams it to disk. Below it, and for a generated + * document, which must compile first, fetching keeps a rejection on screen as an error rather than + * as a failed entry in the browser's downloads list. */ const IN_PAGE_DOWNLOAD_MAX_BYTES = 100 * 1024 * 1024 @@ -97,9 +97,16 @@ export async function triggerFileDownload( ? `/api/files/export/${encodeURIComponent(record.id)}` : `/api/files/serve/${encodeURIComponent(record.key)}?context=${record.storageContext ?? 'workspace'}&t=${Date.now()}` - if (!exportsMarkdown && record.size > IN_PAGE_DOWNLOAD_MAX_BYTES) { - // boundary-raw-fetch: HEAD probe of a binary path with no contract, so a refusal still surfaces here - const probe = await fetch(url, { method: 'HEAD', cache: 'no-store' }) + if ( + !exportsMarkdown && + !needsRenderedArtifact(record.type, record.name) && + record.size > IN_PAGE_DOWNLOAD_MAX_BYTES + ) { + // A first-byte read opens storage itself, so a missing object is refused here, in place, + // rather than as a failed entry in the browser's downloads list. + // boundary-raw-fetch: ranged probe of a binary path with no contract + const probe = await fetch(url, { headers: { Range: 'bytes=0-0' }, cache: 'no-store' }) + await probe.body?.cancel() if (!probe.ok) throw new Error(`Failed to download "${record.name}"`) clickDownloadLink(url, record.name) return diff --git a/apps/sim/lib/uploads/core/storage-service.ts b/apps/sim/lib/uploads/core/storage-service.ts index 269602bee22..769db1dcf02 100644 --- a/apps/sim/lib/uploads/core/storage-service.ts +++ b/apps/sim/lib/uploads/core/storage-service.ts @@ -808,8 +808,8 @@ export async function headObject( const { UPLOAD_DIR_SERVER } = await getLocalStorageSetup() try { const file = await stat(join(UPLOAD_DIR_SERVER, sanitizeFileKey(key))) - // A local write replaces the file in place, so its modification time is what changes with it. - return { size: file.size, version: `${file.mtimeMs}` } + const { localObjectVersion } = await import('@/lib/uploads/upload-session/provider') + return { size: file.size, version: localObjectVersion(file) } } catch (error) { const code = (error as NodeJS.ErrnoException).code if (code === 'ENOENT') return null diff --git a/apps/sim/lib/uploads/upload-session/provider.ts b/apps/sim/lib/uploads/upload-session/provider.ts index 84cdad89315..a0c81aa96a2 100644 --- a/apps/sim/lib/uploads/upload-session/provider.ts +++ b/apps/sim/lib/uploads/upload-session/provider.ts @@ -790,7 +790,7 @@ async function headLocalObject(key: string): Promise { size: file.size, contentType: metadata.contentType, uploadId: metadata.uploadId, - version: localVersion(file), + version: localObjectVersion(file), } } @@ -803,7 +803,7 @@ async function deleteLocalObjectVersion(key: string, version: string): Promise { @@ -841,7 +841,11 @@ async function readLocalMetadata(path: string): Promise { } } -function localVersion(file: Awaited>): string { +/** + * Version of a local upload object: what changes whenever its bytes do, including a same-size + * replacement within one timestamp tick. + */ +export function localObjectVersion(file: Awaited>): string { return `${file.dev}:${file.ino}:${file.size}:${file.mtimeMs}` }