From 4030b6f1b83b69c6ad53b8dcd9e6eda06135f90b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 2 Nov 2026 18:31:15 -0800 Subject: [PATCH 1/2] improvement(sandbox): return a timed-out workbench run's output and the files it already wrote A run_code call killed at its own timeout returned an empty stdout and only "timed out after Nms", so models reran long jobs from scratch. E2B's SDK throws a stream-less TimeoutError, which dropped the output the stream callbacks had already observed. - E2B adapter: a timed-out command returns the stdout/stderr its callbacks observed - Workbench runs (session sandboxes) that hit their own timeout throw a timeout abort carrying the output tail (64 KB) and the files already in the call's output directory; every existing timeout check still accepts it - The route returns that output in stdout with a 408 and says the workbench and its files persist, so the next call can rerun only the remaining work - Output goes through the same session provenance check and secret-name reporting as any other result; one-shot sandboxes and billing are unchanged --- .../remote-sandbox/e2b-session.test.ts | 30 +++++ apps/sim/lib/execution/remote-sandbox/e2b.ts | 27 ++-- .../sim/lib/execution/remote-sandbox/index.ts | 55 +++++++- .../execution/remote-sandbox/output-limits.ts | 5 +- .../remote-sandbox/session-sandbox.test.ts | 122 ++++++++++++++++++ .../lib/execution/remote-sandbox/timeout.ts | 64 +++++++++ .../execute-request.test.ts | 34 +++++ .../lib/function-execution/execute-request.ts | 24 +++- 8 files changed, 341 insertions(+), 20 deletions(-) create mode 100644 apps/sim/lib/execution/remote-sandbox/timeout.ts diff --git a/apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts b/apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts index 6c9611e3c26..732cacb6fed 100644 --- a/apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts @@ -613,6 +613,36 @@ describe('E2B session recovery', () => { expect(stopCommand).toHaveBeenCalledOnce() expect(killSandbox).not.toHaveBeenCalled() }) + + it('returns what a session command printed before its own timeout stopped it', async () => { + runCommand.mockImplementation( + async ( + _command: string, + options: { onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void } + ) => { + options.onStdout?.('rendered 1 of 3\n') + options.onStderr?.('slow frame\n') + options.onStdout?.('rendered 2 of 3\n') + return { + pid: 7, + wait: async () => { + throw Object.assign( + new Error("deadline_exceeded: This error is likely due to exceeding 'timeoutMs'"), + { name: 'TimeoutError' } + ) + }, + } + } + ) + const sandbox = await sessionSandbox('reconnected') + const result = await sandbox?.runCommand('render', { timeoutMs: 1000 }) + expect(result).toMatchObject({ + timedOut: true, + exitCode: 124, + stdout: 'rendered 1 of 3\nrendered 2 of 3\n', + stderr: 'slow frame\n', + }) + }) }) describe('E2B session lease', () => { diff --git a/apps/sim/lib/execution/remote-sandbox/e2b.ts b/apps/sim/lib/execution/remote-sandbox/e2b.ts index fb40976763d..425c2c70525 100644 --- a/apps/sim/lib/execution/remote-sandbox/e2b.ts +++ b/apps/sim/lib/execution/remote-sandbox/e2b.ts @@ -373,6 +373,14 @@ export async function stopE2BSessionProcess( if (sandbox) await stopSessionCommand(sandbox, process.id, signal) } +/** Joins what the stream callbacks observed with the streams the SDK returned, without repeating either. */ +function mergeObservedOutput(observed: string, returned: string): string { + if (!observed) return returned + if (!returned || observed.endsWith(returned)) return observed + if (returned.startsWith(observed)) return returned + return observed + returned +} + class E2BSandboxHandle implements SandboxHandle { private killed = false private killPromise: Promise | null = null @@ -676,12 +684,6 @@ class E2BSandboxHandle implements SandboxHandle { // process handle because `background: true` is fixed above. result = started } - const mergeObservedOutput = (observed: string, returned: string): string => { - if (!observed) return returned - if (!returned || observed.endsWith(returned)) return observed - if (returned.startsWith(observed)) return returned - return observed + returned - } const stdout = mergeObservedOutput(observedStdout, result.stdout ?? '') const stderr = mergeObservedOutput(observedStderr, result.stderr ?? '') assertSandboxProcessOutputWithinLimit([stdout, stderr], options.maxOutputBytes) @@ -727,16 +729,21 @@ class E2BSandboxHandle implements SandboxHandle { ) const tailIfStreamed = (value: string | undefined, retain: boolean) => retain || value === undefined ? value : tailStreamedSandboxOutput(value) - const failureStdout = tailIfStreamed(failure.stdout, retainStdout) - const failureStderr = tailIfStreamed(failure.stderr, retainStderr) if (isE2BExecutionTimeout(error)) { + // The SDK's timeout error carries no streams; the callbacks observed the run's output. + const stdout = mergeObservedOutput(observedStdout, failure.stdout ?? '') + const stderr = mergeObservedOutput(observedStderr, failure.stderr ?? '') return { - stdout: failureStdout ?? '', - stderr: failureStderr ?? failure.message ?? '', + stdout: retainStdout ? stdout : tailStreamedSandboxOutput(stdout), + stderr: + (retainStderr ? stderr : tailStreamedSandboxOutput(stderr)) || + `Command timed out after ${options.timeoutMs}ms`, exitCode: 124, timedOut: true, } } + const failureStdout = tailIfStreamed(failure.stdout, retainStdout) + const failureStderr = tailIfStreamed(failure.stderr, retainStderr) return { stdout: failureStdout ?? '', stderr: failureStderr ?? failure.message ?? getErrorMessage(error), diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index 265271b7b51..c9dd37350a2 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -9,6 +9,7 @@ import { isTimeoutAbortReason, } from '@/lib/core/execution-limits' import { recordSandboxTeardownFailure } from '@/lib/core/execution-limits/metrics' +import { withinDeadline } from '@/lib/core/utils/deadline' import { buildJavaScriptRuntimeBindingsSource } from '@/lib/execution/code-placeholders/javascript-runtime' import { SANDBOX_SYSTEM_PATH } from '@/lib/execution/remote-sandbox/cli-tools.server' import { describeProcessExit } from '@/lib/execution/remote-sandbox/code-failure' @@ -65,6 +66,10 @@ import { recordSessionFileInput, } from '@/lib/execution/remote-sandbox/session-file-provenance' import { withSandboxSessionLock } from '@/lib/execution/remote-sandbox/session-lock' +import { + SandboxExecutionTimeoutError, + sandboxTimeoutProgress, +} from '@/lib/execution/remote-sandbox/timeout' import type { CreateSandboxOptions, SandboxCodeResult, @@ -258,6 +263,36 @@ function throwIfSandboxTimedOut(result: { timedOut?: boolean }): void { if (result.timedOut) throw new DOMException('timeout', 'AbortError') } +/** Bounds the output-directory listing a timeout report adds; a slow listing reports no files. */ +const TIMED_OUT_OUTPUT_LISTING_MS = 3_000 + +/** + * A workbench run stopped by its own timeout reports what it produced, because the workbench and + * its files outlive the call and the next call can resume from them. A one-shot sandbox is torn + * down with everything in it, so it keeps the plain timeout. + */ +async function throwIfRunTimedOut( + result: { timedOut?: boolean }, + streams: Array, + run: { + sandbox: SandboxHandle + lease: SandboxLease + outputSandboxDir: string | undefined + timeoutMs: number + } +): Promise { + if (!result.timedOut) return + if (!run.lease.session) throw new DOMException('timeout', 'AbortError') + const { outputSandboxDir, sandbox } = run + const listed = outputSandboxDir + ? await withinDeadline( + () => sandbox.listFiles(outputSandboxDir, { depth: SANDBOX_OUTPUT_DIR_MAX_DEPTH }), + Date.now() + TIMED_OUT_OUTPUT_LISTING_MS + ).catch(() => []) + : [] + throw new SandboxExecutionTimeoutError(sandboxTimeoutProgress(streams, run.timeoutMs, listed)) +} + function bindSandboxAbort( sandbox: SandboxHandle, provider: SandboxProviderId, @@ -969,10 +1004,11 @@ async function executeInSandboxWithinBudget( Object.keys(privateInputFiles.environment).length > 0 let execution: SandboxCodeResult + const runTimeoutMs = codeBudgetMs(req, signal) try { const endRun = phases.start('run') execution = await sandbox.runCode(code, { - timeoutMs: codeBudgetMs(req, signal), + timeoutMs: runTimeoutMs, javascriptPreload: buildJavaScriptRuntimeBindingsSource(req.runtimeBindings ?? []), maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, signal, @@ -984,7 +1020,12 @@ async function executeInSandboxWithinBudget( throw error } throwIfAborted(signal) - throwIfSandboxTimedOut(execution) + await throwIfRunTimedOut(execution, [execution.text, execution.stdout, execution.stderr], { + sandbox, + lease, + outputSandboxDir: req.outputSandboxDir, + timeoutMs: runTimeoutMs, + }) if (execution.error) { const errorMessage = `${execution.error.name}: ${execution.error.value}` @@ -1166,6 +1207,7 @@ async function executeShellInSandboxWithinBudget( endPrepare() let result: SandboxCommandResult + const runTimeoutMs = codeBudgetMs(req, signal) try { const endRun = phases.start('run') result = await sandbox.runCommand(code, { @@ -1177,7 +1219,7 @@ async function executeShellInSandboxWithinBudget( ...privateInputFiles.environment, ...(req.session && req.outputSandboxDir ? { SIM_OUTPUT_DIR: req.outputSandboxDir } : {}), }, - timeoutMs: codeBudgetMs(req, signal), + timeoutMs: runTimeoutMs, maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, signal, rootUser: !lease.session, @@ -1189,7 +1231,12 @@ async function executeShellInSandboxWithinBudget( throw error } throwIfAborted(signal) - throwIfSandboxTimedOut(result) + await throwIfRunTimedOut(result, [result.stdout, result.stderr], { + sandbox, + lease, + outputSandboxDir: req.outputSandboxDir, + timeoutMs: runTimeoutMs, + }) const stdout = [result.stdout, result.stderr].filter(Boolean).join('\n') diff --git a/apps/sim/lib/execution/remote-sandbox/output-limits.ts b/apps/sim/lib/execution/remote-sandbox/output-limits.ts index e260660aa19..209246cc79b 100644 --- a/apps/sim/lib/execution/remote-sandbox/output-limits.ts +++ b/apps/sim/lib/execution/remote-sandbox/output-limits.ts @@ -48,7 +48,8 @@ const STREAMED_OUTPUT_TRUNCATION_NOTE = */ export function tailStreamedSandboxOutput( value: string | undefined, - limitBytes = MAX_SANDBOX_STREAMED_OUTPUT_TAIL_BYTES + limitBytes = MAX_SANDBOX_STREAMED_OUTPUT_TAIL_BYTES, + truncationNote = STREAMED_OUTPUT_TRUNCATION_NOTE ): string { if (!value) return '' const buffer = Buffer.from(value, 'utf8') @@ -56,7 +57,7 @@ export function tailStreamedSandboxOutput( let start = buffer.length - limitBytes while (start < buffer.length && (buffer[start] & 0xc0) === 0x80) start += 1 - return `${STREAMED_OUTPUT_TRUNCATION_NOTE}${buffer.subarray(start).toString('utf8')}` + return `${truncationNote}${buffer.subarray(start).toString('utf8')}` } /** diff --git a/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts b/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts index 20562abee59..d5da31fc11f 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts @@ -19,6 +19,7 @@ import type { SandboxCommandResult, SandboxHandle, SandboxProvider, + SandboxSessionRequest, } from '@/lib/execution/remote-sandbox/types' const { mockCreate, mockFindSessionSandbox, mockResolveWorkspaceSandbox } = vi.hoisted(() => ({ @@ -41,6 +42,7 @@ vi.mock('@/lib/core/execution-limits/metrics', () => ({ recordSandboxProviderLimit: vi.fn(), })) +import { isTimeoutAbortReason } from '@/lib/core/execution-limits' import { executeInSandbox, executeShellInSandbox, @@ -56,6 +58,10 @@ import { recordSessionFileInput, } from '@/lib/execution/remote-sandbox/session-file-provenance' import { writeSessionSandboxFile } from '@/lib/execution/remote-sandbox/session-files' +import { + MAX_SANDBOX_TIMEOUT_OUTPUT_TAIL_BYTES, + SandboxExecutionTimeoutError, +} from '@/lib/execution/remote-sandbox/timeout' remoteSandboxProviderMockFns.mockResolveProvider.mockImplementation( (): SandboxProvider => ({ @@ -941,6 +947,122 @@ describe('session sandbox lease', () => { expect(calls.killed).toBe(false) }) + describe('a workbench run stopped by its own timeout', () => { + const OUTPUT_DIR = '/tmp/sim/outputs/call-timeout' + + async function timedOutRun( + kind: 'code' | 'shell', + streams: { stdout: string; stderr: string }, + listFiles: SandboxHandle['listFiles'], + acceptOutputProvenance?: SandboxSessionRequest['acceptOutputProvenance'] + ): Promise { + const { handle } = fakeSandbox(`timeout-${kind}`) + mockFindSessionSandbox.mockResolvedValue(handle) + handle.runCode = async () => ({ text: '', ...streams, timedOut: true }) + handle.runCommand = async () => ({ ...streams, exitCode: 124, timedOut: true }) + handle.listFiles = listFiles + const request = { + ...CODE_REQUEST, + session: { + key: `timeout-${kind}`, + ...(acceptOutputProvenance ? { acceptOutputProvenance } : {}), + }, + outputSandboxDir: OUTPUT_DIR, + } + return ( + kind === 'code' + ? executeInSandbox(request) + : executeShellInSandbox({ ...request, envs: {} }) + ).then( + () => { + throw new Error('Expected the run to time out') + }, + (error: unknown) => error + ) + } + + it.each(['code', 'shell'] as const)( + 'reports the %s output so far and the files already exported', + async (kind) => { + const error = await timedOutRun( + kind, + { stdout: 'rendered 1 of 3\nrendered 2 of 3', stderr: 'slow frame' }, + async () => [ + { + path: `${OUTPUT_DIR}/${SANDBOX_OUTPUT_DIR_SENTINEL}`, + relativePath: SANDBOX_OUTPUT_DIR_SENTINEL, + kind: 'file', + size: 0, + }, + { path: `${OUTPUT_DIR}/frames`, relativePath: 'frames', kind: 'directory', size: 0 }, + { + path: `${OUTPUT_DIR}/frames/2.png`, + relativePath: 'frames/2.png', + kind: 'file', + size: 9, + }, + { + path: `${OUTPUT_DIR}/frames/1.png`, + relativePath: 'frames/1.png', + kind: 'file', + size: 9, + }, + ] + ) + expect(isTimeoutAbortReason(error)).toBe(true) + if (!(error instanceof SandboxExecutionTimeoutError)) throw error + expect(error.progress.output).toBe('rendered 1 of 3\nrendered 2 of 3\nslow frame') + expect(error.progress.outputFiles).toEqual([ + `${OUTPUT_DIR}/frames/1.png`, + `${OUTPUT_DIR}/frames/2.png`, + ]) + expect(error.progress.stoppedAfterMs).toBeGreaterThan(0) + expect(error.progress.stoppedAfterMs).toBeLessThanOrEqual(CODE_REQUEST.timeoutMs) + } + ) + + it.each([ + ['fails', async () => Promise.reject(new Error('listing unavailable'))], + ['never answers', () => new Promise(() => {})], + ] as const)('still reports the output when the file listing %s', async (_case, listFiles) => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + try { + const pending = timedOutRun('shell', { stdout: 'step 1', stderr: '' }, listFiles) + await vi.runAllTimersAsync() + const error = await pending + if (!(error instanceof SandboxExecutionTimeoutError)) throw error + expect(error.progress.output).toBe('step 1') + expect(error.progress.outputFiles).toEqual([]) + } finally { + vi.useRealTimers() + } + }) + + it('withholds the output when the workbench secret provenance is unavailable', async () => { + const error = await timedOutRun( + 'shell', + { stdout: 'token=secret-value', stderr: '' }, + async () => [], + async () => { + throw new Error('Workbench output withheld because its secret provenance is unavailable') + } + ) + expect(error).not.toBeInstanceOf(SandboxExecutionTimeoutError) + expect(getErrorMessage(error)).not.toContain('secret-value') + }) + + it('keeps only the end of an output too large to return', async () => { + const lines = Array.from({ length: 20_000 }, (_, index) => `frame ${index}`).join('\n') + const error = await timedOutRun('shell', { stdout: lines, stderr: '' }, async () => []) + if (!(error instanceof SandboxExecutionTimeoutError)) throw error + expect(Buffer.byteLength(error.progress.output)).toBeLessThanOrEqual( + MAX_SANDBOX_TIMEOUT_OUTPUT_TAIL_BYTES + 64 + ) + expect(error.progress.output.endsWith('frame 19999')).toBe(true) + expect(error.progress.output.startsWith('[earlier output omitted]')).toBe(true) + }) + }) + it('keeps the one-shot teardown when no session is requested', async () => { const { handle, calls } = fakeSandbox('sb-oneshot') mockCreate.mockResolvedValue(handle) diff --git a/apps/sim/lib/execution/remote-sandbox/timeout.ts b/apps/sim/lib/execution/remote-sandbox/timeout.ts new file mode 100644 index 00000000000..a0dce09fd95 --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/timeout.ts @@ -0,0 +1,64 @@ +import { tailStreamedSandboxOutput } from '@/lib/execution/remote-sandbox/output-limits' +import { SANDBOX_OUTPUT_DIR_SENTINEL } from '@/lib/execution/remote-sandbox/paths' +import type { SandboxDirectoryEntry } from '@/lib/execution/remote-sandbox/types' + +/** Output kept from a timed-out workbench run: its end says where the work stopped. */ +export const MAX_SANDBOX_TIMEOUT_OUTPUT_TAIL_BYTES = 64 * 1024 + +/** Output-directory files named in a timeout message; the rest are counted. */ +const MAX_NAMED_PERSISTED_FILES = 20 + +const TIMEOUT_OUTPUT_TRUNCATION_NOTE = '[earlier output omitted]\n' + +/** What a workbench run had produced when its own timeout stopped it. */ +export interface SandboxTimeoutProgress { + /** Tail of the combined output streams. */ + output: string + /** The timeout the code ran under, after which it was stopped. */ + stoppedAfterMs: number + /** Files already in the call's output directory; they persist with the workbench. */ + outputFiles: string[] +} + +/** + * A workbench process stopped by its own timeout. It is still the `timeout` abort every caller + * already handles (`isTimeoutAbortReason` accepts it), and additionally carries the run's progress + * so the caller can return it through its normal output path. + */ +export class SandboxExecutionTimeoutError extends DOMException { + constructor(readonly progress: SandboxTimeoutProgress) { + super('timeout', 'AbortError') + } +} + +export function sandboxTimeoutProgress( + streams: Array, + stoppedAfterMs: number, + listed: SandboxDirectoryEntry[] +): SandboxTimeoutProgress { + return { + output: tailStreamedSandboxOutput( + streams.filter(Boolean).join('\n'), + MAX_SANDBOX_TIMEOUT_OUTPUT_TAIL_BYTES, + TIMEOUT_OUTPUT_TRUNCATION_NOTE + ), + stoppedAfterMs, + outputFiles: listed + .filter( + (entry) => entry.kind === 'file' && entry.relativePath !== SANDBOX_OUTPUT_DIR_SENTINEL + ) + .map((entry) => entry.path) + .sort(), + } +} + +/** Appended to the route's timeout error so the next call resumes the work instead of redoing it. */ +export function describeSandboxTimeoutProgress(progress: SandboxTimeoutProgress): string { + const named = progress.outputFiles.slice(0, MAX_NAMED_PERSISTED_FILES) + const unnamed = progress.outputFiles.length - named.length + const files = named.length + ? ` Already in its output directory: ${named.join(', ')}${unnamed > 0 ? ` and ${unnamed} more` : ''}.` + : '' + const output = progress.output ? ' Its output so far is in stdout.' : '' + return ` The code was stopped after ${Math.round(progress.stoppedAfterMs / 1000)} s.${output} The workbench and the files the code wrote persist.${files} Rerun only the remaining work.` +} diff --git a/apps/sim/lib/function-execution/execute-request.test.ts b/apps/sim/lib/function-execution/execute-request.test.ts index eb9ebabc528..da0e15b0c4d 100644 --- a/apps/sim/lib/function-execution/execute-request.test.ts +++ b/apps/sim/lib/function-execution/execute-request.test.ts @@ -39,6 +39,7 @@ import { SandboxOutputFileError, SandboxOutputLimitError, } from '@/lib/execution/remote-sandbox/output-limits' +import { SandboxExecutionTimeoutError } from '@/lib/execution/remote-sandbox/timeout' const { mockExecuteInIsolatedVM, @@ -3457,6 +3458,39 @@ describe('Function execution request', () => { expect(directData.__resolvedSecretNames).toEqual(['API_KEY']) }) + it('returns a timed-out workbench run’s output with the secret names it may reveal', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteShellInSandbox.mockRejectedValueOnce( + new SandboxExecutionTimeoutError({ + output: 'secret-value\nrendered 2 of 3', + stoppedAfterMs: 290_000, + outputFiles: ['/tmp/sim/outputs/call-1/frame-1.png'], + }) + ) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'printf "%s\\n" "$API_KEY"; ./render', + language: 'shell', + envVars: { API_KEY: 'secret-value' }, + timeout: 300_000, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + const data = await response.json() + + expect(response.status).toBe(408) + expect(data.output.stdout).toBe('secret-value\nrendered 2 of 3') + expect(data.output.cost).toBeUndefined() + expect(data.__resolvedSecretNames).toEqual(['API_KEY']) + expect(data.error).toMatch(/^Function execution timed out after 300000ms\. /) + expect(data.error).toContain('stopped after 290 s') + expect(data.error).toContain('/tmp/sim/outputs/call-1/frame-1.png') + }) + it('returns nonzero shell stderr as a visible 422 error and diagnostic output', async () => { envFlagsMock.isRemoteSandboxEnabled = true const stderr = "error: unknown flag: --short\nSee 'kubectl version --help' for usage." diff --git a/apps/sim/lib/function-execution/execute-request.ts b/apps/sim/lib/function-execution/execute-request.ts index f0c4ba15b4d..7b50c729753 100644 --- a/apps/sim/lib/function-execution/execute-request.ts +++ b/apps/sim/lib/function-execution/execute-request.ts @@ -90,6 +90,10 @@ import { readTrustedSandboxOutputCost, } from '@/lib/execution/remote-sandbox/output-limits' import { MAX_BLOCK_MOUNTED_FILES, SANDBOX_OUTPUT_DIR } from '@/lib/execution/remote-sandbox/paths' +import { + describeSandboxTimeoutProgress, + SandboxExecutionTimeoutError, +} from '@/lib/execution/remote-sandbox/timeout' import type { SandboxCollectedFile, SandboxFile, @@ -3361,12 +3365,24 @@ export async function executeFunctionRequest( isTimeoutAbortReason(req.signal.reason) || isTimeoutAbortReason(error?.cause ?? error) || (executionDeadlineAt !== undefined && Date.now() >= executionDeadlineAt) + const timeoutMessage = `Function execution timed out${timeoutForError ? ` after ${timeoutForError}ms` : ''}` + // Only a route context carries the secret provenance that partial output must cross with. + const progress = + timedOut && routeContext && error instanceof SandboxExecutionTimeoutError + ? error.progress + : undefined const abortResponse = { success: false, - error: timedOut - ? `Function execution timed out${timeoutForError ? ` after ${timeoutForError}ms` : ''}` - : 'Function execution was cancelled', - output: { result: null, stdout: cleanStdout(stdout), executionTime }, + error: !timedOut + ? 'Function execution was cancelled' + : progress + ? `${timeoutMessage}.${describeSandboxTimeoutProgress(progress)}` + : timeoutMessage, + output: { + result: null, + stdout: cleanStdout(progress ? progress.output : stdout), + executionTime, + }, } logger.warn(`[${requestId}] Function execution ${timedOut ? 'timed out' : 'was cancelled'}`, { executionTime, From 75dcdbf5dccfa594e34cdb670c1b2cd9d4702620 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 2 Nov 2026 18:43:43 -0800 Subject: [PATCH 2/2] fix(sandbox): keep a cancellation during the timed-out output listing a cancellation --- .../sim/lib/execution/remote-sandbox/index.ts | 8 +++++- .../remote-sandbox/session-sandbox.test.ts | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index c9dd37350a2..4250472ca5f 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -279,6 +279,7 @@ async function throwIfRunTimedOut( lease: SandboxLease outputSandboxDir: string | undefined timeoutMs: number + signal: AbortSignal } ): Promise { if (!result.timedOut) return @@ -287,9 +288,12 @@ async function throwIfRunTimedOut( const listed = outputSandboxDir ? await withinDeadline( () => sandbox.listFiles(outputSandboxDir, { depth: SANDBOX_OUTPUT_DIR_MAX_DEPTH }), - Date.now() + TIMED_OUT_OUTPUT_LISTING_MS + Date.now() + TIMED_OUT_OUTPUT_LISTING_MS, + run.signal ).catch(() => []) : [] + // A cancellation that arrives while listing stays a cancellation. + throwIfAborted(run.signal) throw new SandboxExecutionTimeoutError(sandboxTimeoutProgress(streams, run.timeoutMs, listed)) } @@ -1025,6 +1029,7 @@ async function executeInSandboxWithinBudget( lease, outputSandboxDir: req.outputSandboxDir, timeoutMs: runTimeoutMs, + signal, }) if (execution.error) { @@ -1236,6 +1241,7 @@ async function executeShellInSandboxWithinBudget( lease, outputSandboxDir: req.outputSandboxDir, timeoutMs: runTimeoutMs, + signal, }) const stdout = [result.stdout, result.stderr].filter(Boolean).join('\n') diff --git a/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts b/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts index d5da31fc11f..90bc0e361a8 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts @@ -1038,6 +1038,34 @@ describe('session sandbox lease', () => { } }) + it('keeps a cancellation that arrives while listing the outputs a cancellation', async () => { + const { handle } = fakeSandbox('timeout-cancelled') + mockFindSessionSandbox.mockResolvedValue(handle) + const controller = new AbortController() + handle.runCommand = async () => ({ + stdout: 'step 1', + stderr: '', + exitCode: 124, + timedOut: true, + }) + handle.listFiles = async () => { + controller.abort(new Error('User Stop')) + return new Promise(() => {}) + } + const error = await executeShellInSandbox({ + ...CODE_REQUEST, + envs: {}, + session: { key: 'timeout-cancelled' }, + outputSandboxDir: OUTPUT_DIR, + signal: controller.signal, + }).then( + () => undefined, + (rejection: unknown) => rejection + ) + expect(error).not.toBeInstanceOf(SandboxExecutionTimeoutError) + expect(isTimeoutAbortReason(error)).toBe(false) + }) + it('withholds the output when the workbench secret provenance is unavailable', async () => { const error = await timedOutRun( 'shell',