From 6943fdebf8ea636e2664cf7fb537e993b52c3d64 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 2 Nov 2026 18:54:16 -0800 Subject: [PATCH] improvement(sandbox): verify the workbench CLI inside each command's launch Every warm workbench call ran a separate CLI integrity check command (ownership write, launch, stop receipt, settle) before the user's command, about a third of a warm call's fixed cost. - The session supervisor's PID 1 now runs the same check as the ordinary user, after recording itself and before starting the command; a mismatch starts nothing and is reported on the stop receipt the call already fetches - A refused launch is repaired under the session lock and relaunched once; a second refusal fails the call - The check runs with an isolated interpreter instead of the user's login shell, never follows links or blocks on non-regular files, and treats an unreadable artifact as stale so repair overwrites it - A reused machine runs no separate check command; a new machine still installs the CLI first - Gated live test against real E2B covers refusal, repair, hooks, FIFOs, Stop during a slow check, and the warm-call cost --- .../remote-sandbox/e2b-session.test.ts | 31 +++ apps/sim/lib/execution/remote-sandbox/e2b.ts | 55 +++- .../sim/lib/execution/remote-sandbox/index.ts | 111 ++++++-- .../execution/remote-sandbox/phase-timing.ts | 5 +- .../session-cli-launch.live.test.ts | 253 ++++++++++++++++++ .../execution/remote-sandbox/session-cli.ts | 116 ++++++-- .../remote-sandbox/session-process.ts | 57 +++- .../remote-sandbox/session-sandbox.test.ts | 92 +++++++ .../lib/execution/remote-sandbox/session.ts | 3 +- .../sim/lib/execution/remote-sandbox/types.ts | 17 ++ 10 files changed, 675 insertions(+), 65 deletions(-) create mode 100644 apps/sim/lib/execution/remote-sandbox/session-cli-launch.live.test.ts diff --git a/apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts b/apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts index 732cacb6fed..62e4c016f3a 100644 --- a/apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts @@ -51,6 +51,7 @@ import { stopE2BSessionProcess, } from '@/lib/execution/remote-sandbox/e2b' import { observeSandboxExecution } from '@/lib/execution/remote-sandbox/observer' +import { SessionCliStaleError } from '@/lib/execution/remote-sandbox/session-cli' setEnv({ E2B_API_KEY: 'test', MOTHERSHIP_E2B_TEMPLATE_ID: 'mothership-template' }) @@ -643,6 +644,36 @@ describe('E2B session recovery', () => { stderr: 'slow frame\n', }) }) + it.each([ + ['stale', SessionCliStaleError], + ['failed', Error], + ] as const)( + 'rejects a launch whose receipt reports a %s CLI, whatever its process returned', + async (cli, expected) => { + runCommand.mockResolvedValue({ + pid: 9, + wait: async () => ({ stdout: '', stderr: '', exitCode: 0 }), + }) + stopCommand.mockResolvedValue({ + stdout: JSON.stringify({ settled: true, cli }), + stderr: '', + exitCode: 0, + }) + const sandbox = await sessionSandbox('reconnected') + const launched = sandbox?.runCommand('touch /tmp/marker', { + timeoutMs: 1000, + verifyCli: { path: '/home/user/.sim-cli/r/cli.mjs', artifacts: { a: 'b' } }, + }) + await expect(launched).rejects.toBeInstanceOf(expected) + await expect(launched).rejects.toThrow(cli === 'stale' ? 'changed' : 'verification failed') + const [command] = runCommand.mock.calls.at(-1) ?? [] + expect(command).toContain( + Buffer.from( + JSON.stringify({ path: '/home/user/.sim-cli/r/cli.mjs', artifacts: { a: 'b' } }) + ).toString('base64') + ) + } + ) }) describe('E2B session lease', () => { diff --git a/apps/sim/lib/execution/remote-sandbox/e2b.ts b/apps/sim/lib/execution/remote-sandbox/e2b.ts index 425c2c70525..264308044ff 100644 --- a/apps/sim/lib/execution/remote-sandbox/e2b.ts +++ b/apps/sim/lib/execution/remote-sandbox/e2b.ts @@ -55,6 +55,10 @@ import { tailStreamedSandboxOutput, } from '@/lib/execution/remote-sandbox/output-limits' import { resolveSandboxDirectoryEntryPath } from '@/lib/execution/remote-sandbox/paths' +import { + SessionCliStaleError, + SessionCliVerificationError, +} from '@/lib/execution/remote-sandbox/session-cli' import { withSandboxSessionLock } from '@/lib/execution/remote-sandbox/session-lock' import { type SessionProcessIdentity, @@ -82,6 +86,7 @@ import type { SandboxImageMaterialization, SandboxKind, SandboxProvider, + SessionCliVerification, } from '@/lib/execution/remote-sandbox/types' const logger = createLogger('E2BSandboxProvider') @@ -319,11 +324,17 @@ async function listAllSandboxes(paginator: SandboxPaginator): Promise { +): Promise { signal.throwIfAborted() const result = await sandbox.commands.run(sessionProcessStopCommand(processId), { user: 'root', @@ -335,6 +346,7 @@ async function stopSessionCommand( if (result.exitCode !== 0 || !isRecordLike(receipt) || receipt.settled !== true) { throw new Error('Sandbox command namespace exit is unconfirmed') } + return receipt.cli === 'stale' || receipt.cli === 'failed' ? { cli: receipt.cli } : {} } /** Recovers the exact recorded workbench; expiry is absence, while lookup failure stays unresolved. */ @@ -413,6 +425,10 @@ class E2BSandboxHandle implements SandboxHandle { return this.sandbox.sandboxId } + get verifiesCliAtLaunch(): boolean { + return this.sessionKey !== undefined + } + outlives(lifetimeMs: number, fromMs: number): boolean { return ( this.sessionDeadlineAtMs !== undefined && @@ -441,6 +457,7 @@ class E2BSandboxHandle implements SandboxHandle { javascriptPreload?: string maxOutputBytes?: number signal?: AbortSignal + verifyCli?: SessionCliVerification } ): Promise { const runId = generateShortId(12) @@ -467,6 +484,7 @@ class E2BSandboxHandle implements SandboxHandle { maxOutputBytes: options.maxOutputBytes, signal: options.signal, atMostOnce: true, + verifyCli: options.verifyCli, envs: { ...options.envs, SIM_CODE_PATH: codePath, @@ -519,6 +537,25 @@ class E2BSandboxHandle implements SandboxHandle { command: string, options: RunCommandOptions, operation: 'code' | 'command' + ): Promise { + let refusal: SessionCommandReceipt['cli'] + const outcome = await this.runOwnedCommand(command, options, operation, (receipt) => { + refusal = receipt.cli + }).then( + (result) => ({ result }), + (error: unknown) => ({ error }) + ) + if (refusal === 'stale') throw new SessionCliStaleError() + if (refusal === 'failed') throw new SessionCliVerificationError() + if ('error' in outcome) throw outcome.error + return outcome.result + } + + private async runOwnedCommand( + command: string, + options: RunCommandOptions, + operation: 'code' | 'command', + onReceipt: (receipt: SessionCommandReceipt) => void ): Promise { if (this.sessionKey !== undefined) options.signal?.throwIfAborted() let commandDeadlineAtMs: number | undefined @@ -537,9 +574,9 @@ class E2BSandboxHandle implements SandboxHandle { let observedStderr = '' const processId = this.sessionKey !== undefined ? generateId() : undefined let processRecorded = false - let stopping: Promise | undefined - const stopCommand = (): Promise => { - if (!processId || !processRecorded) return Promise.resolve() + let stopping: Promise | undefined + const stopCommand = (): Promise => { + if (!processId || !processRecorded) return Promise.resolve(undefined) stopping ??= stopSessionCommand(this.sandbox, processId).catch((error: unknown) => { logger.warn('Failed to stop sandbox process', { sandboxId: this.sandboxId, @@ -595,7 +632,12 @@ class E2BSandboxHandle implements SandboxHandle { commandDeadlineAtMs = Date.now() + processOptions.timeoutMs started = await this.sandbox.commands.run( processId - ? sessionProcessCommand(processId, prepared.command, options.rootUser) + ? sessionProcessCommand( + processId, + prepared.command, + options.rootUser, + options.verifyCli + ) : prepared.command, { ...processOptions, @@ -753,11 +795,12 @@ class E2BSandboxHandle implements SandboxHandle { options.signal?.removeEventListener('abort', onAbort) /** The named SDK process can exit before its namespace; only the kernel receipt settles it. */ if (processId && processRecorded) { - await stopCommand().catch((error: unknown) => { + const receipt = await stopCommand().catch((error: unknown) => { reportUnsettledSandboxProcess(processId) throw error }) await settleSandboxProcess(processId) + if (receipt) onReceipt(receipt) } } } diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index 4250472ca5f..6de05b3acbd 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -60,7 +60,12 @@ import { ensureSessionSandbox, SESSION_SANDBOX_IDLE_MS, } from '@/lib/execution/remote-sandbox/session' -import { sessionCommandPath } from '@/lib/execution/remote-sandbox/session-cli' +import { + ensureSessionCli, + SessionCliStaleError, + sessionCliVerification, + sessionCommandPath, +} from '@/lib/execution/remote-sandbox/session-cli' import { readSessionSecretProvenance, recordSessionFileInput, @@ -87,6 +92,7 @@ import type { SandboxProviderId, SandboxSessionRequest, SandboxShellExecutionRequest, + SessionCliVerification, } from '@/lib/execution/remote-sandbox/types' export type { @@ -259,6 +265,42 @@ async function withSandboxExecutionBudget( } } +/** + * Launches a session command that verifies the deployment CLI as it starts. A launch that found + * the CLI changed started nothing, so the CLI is repaired under the session lock and the command + * launched once more; a second refusal means something keeps changing it, and the call fails. + */ +async function launchWithVerifiedCli( + sandbox: SandboxHandle, + lease: SandboxLease, + session: SandboxSessionRequest | undefined, + signal: AbortSignal, + phases: SandboxPhaseTimer, + launch: (verifyCli: SessionCliVerification | undefined) => Promise +): Promise { + const cli = lease.session && sandbox.verifiesCliAtLaunch ? session?.cli : undefined + if (!session || !cli) return launch(undefined) + const verifyCli = sessionCliVerification(cli) + try { + return await launch(verifyCli) + } catch (error) { + if (!(error instanceof SessionCliStaleError)) throw error + } + const endCli = phases.start('cli') + await withSandboxSessionLock(session.key, signal, (lockSignal) => + ensureSessionCli(sandbox, cli, lockSignal, remainingSandboxBudgetMs(signal)) + ) + endCli() + try { + return await launch(verifyCli) + } catch (error) { + if (error instanceof SessionCliStaleError) { + throw new Error('Workbench CLI installation could not be verified') + } + throw error + } +} + function throwIfSandboxTimedOut(result: { timedOut?: boolean }): void { if (result.timedOut) throw new DOMException('timeout', 'AbortError') } @@ -1008,16 +1050,27 @@ async function executeInSandboxWithinBudget( Object.keys(privateInputFiles.environment).length > 0 let execution: SandboxCodeResult - const runTimeoutMs = codeBudgetMs(req, signal) + let runTimeoutMs = 0 try { const endRun = phases.start('run') - execution = await sandbox.runCode(code, { - timeoutMs: runTimeoutMs, - javascriptPreload: buildJavaScriptRuntimeBindingsSource(req.runtimeBindings ?? []), - maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, + execution = await launchWithVerifiedCli( + sandbox, + lease, + req.session, signal, - ...(hasExecutionEnvironment ? { envs: executionEnvironment } : {}), - }) + phases, + (verifyCli) => { + runTimeoutMs = codeBudgetMs(req, signal) + return sandbox.runCode(code, { + timeoutMs: runTimeoutMs, + javascriptPreload: buildJavaScriptRuntimeBindingsSource(req.runtimeBindings ?? []), + maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, + signal, + ...(hasExecutionEnvironment ? { envs: executionEnvironment } : {}), + verifyCli, + }) + } + ) endRun() } catch (error) { throwIfAborted(signal) @@ -1212,24 +1265,36 @@ async function executeShellInSandboxWithinBudget( endPrepare() let result: SandboxCommandResult - const runTimeoutMs = codeBudgetMs(req, signal) + let runTimeoutMs = 0 try { const endRun = phases.start('run') - result = await sandbox.runCommand(code, { - envs: { - ...selected?.envs, - ...envs, - ...req.session?.envs, - PATH: sessionCommandPath(req.session, selected?.envs?.PATH ?? SANDBOX_SYSTEM_PATH), - ...privateInputFiles.environment, - ...(req.session && req.outputSandboxDir ? { SIM_OUTPUT_DIR: req.outputSandboxDir } : {}), - }, - timeoutMs: runTimeoutMs, - maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, + const commandEnvironment = { + ...selected?.envs, + ...envs, + ...req.session?.envs, + PATH: sessionCommandPath(req.session, selected?.envs?.PATH ?? SANDBOX_SYSTEM_PATH), + ...privateInputFiles.environment, + ...(req.session && req.outputSandboxDir ? { SIM_OUTPUT_DIR: req.outputSandboxDir } : {}), + } + result = await launchWithVerifiedCli( + sandbox, + lease, + req.session, signal, - rootUser: !lease.session, - atMostOnce: true, - }) + phases, + (verifyCli) => { + runTimeoutMs = codeBudgetMs(req, signal) + return sandbox.runCommand(code, { + envs: commandEnvironment, + timeoutMs: runTimeoutMs, + maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, + signal, + rootUser: !lease.session, + atMostOnce: true, + verifyCli, + }) + } + ) endRun() } catch (error) { throwIfAborted(signal) diff --git a/apps/sim/lib/execution/remote-sandbox/phase-timing.ts b/apps/sim/lib/execution/remote-sandbox/phase-timing.ts index ecc418075a2..6205e0e85a4 100644 --- a/apps/sim/lib/execution/remote-sandbox/phase-timing.ts +++ b/apps/sim/lib/execution/remote-sandbox/phase-timing.ts @@ -1,7 +1,8 @@ /** * Phases of one sandbox execution, in the order they run. `lease` spans the whole acquisition - * (session lock included); for a session, `lookup` and `cli` are the parts of it spent finding or - * creating the machine and verifying its deployment tooling. + * (session lock included); for a session, `lookup` is the part of it spent finding or creating the + * machine. `cli` is time spent installing or repairing deployment tooling: on a new machine, or + * after a launch refused a changed CLI. A reused machine verifies it inside `run`. */ type SandboxPhase = | 'resolve' diff --git a/apps/sim/lib/execution/remote-sandbox/session-cli-launch.live.test.ts b/apps/sim/lib/execution/remote-sandbox/session-cli-launch.live.test.ts new file mode 100644 index 00000000000..6baefa9725f --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/session-cli-launch.live.test.ts @@ -0,0 +1,253 @@ +/** + * Launch-time CLI verification against a real E2B workbench: a command starts only while the + * deployment CLI still matches, the check resists the user's shell and Python hooks, a refused + * launch starts nothing, and a repaired CLI launches again. + * + * Enable with `SANDBOX_SESSION_SMOKE=1`. Requires `E2B_API_KEY` and `MOTHERSHIP_E2B_TEMPLATE_ID`. + * Writes a JSON report of each check to `SESSION_CLI_LAUNCH_REPORT_PATH` when set. Creates one + * short-lived sandbox and always kills it. + */ +import { writeFile } from 'node:fs/promises' +import { getErrorMessage } from '@sim/utils/errors' +import { generateShortId } from '@sim/utils/id' +import { afterAll, describe, expect, it } from 'vitest' +import { e2bProvider } from '@/lib/execution/remote-sandbox/e2b' +import { + ensureSessionCli, + SESSION_CLI_CHECK_SCRIPT, + SessionCliStaleError, + sessionCliVerification, +} from '@/lib/execution/remote-sandbox/session-cli' +import type { SandboxHandle, SandboxSessionRequest } from '@/lib/execution/remote-sandbox/types' + +const smokeEnabled = process.env.SANDBOX_SESSION_SMOKE === '1' +const TIMEOUT_MS = 30_000 +const SAMPLES = 7 + +interface CheckReport { + name: string + status: 'passed' | 'failed' + durationMs: number + detail?: unknown +} +const report: CheckReport[] = [] + +async function check(name: string, run: () => Promise): Promise { + const startedAt = performance.now() + try { + const detail = await run() + report.push({ name, status: 'passed', durationMs: performance.now() - startedAt, detail }) + return detail + } catch (error) { + report.push({ + name, + status: 'failed', + durationMs: performance.now() - startedAt, + detail: getErrorMessage(error), + }) + throw error + } +} + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b) + return Math.round(sorted[Math.floor(sorted.length / 2)]) +} + +afterAll(async () => { + const path = process.env.SESSION_CLI_LAUNCH_REPORT_PATH + if (smokeEnabled && path) await writeFile(path, JSON.stringify(report, null, 2)) +}) + +describe.skipIf(!smokeEnabled)('session CLI verification at launch', () => { + it( + 'starts a command only while the deployment CLI matches its release', + async () => { + const directory = `/home/user/.sim-cli/live-${generateShortId(10)}` + const cli: NonNullable = { + path: `${directory}/cli.mjs`, + content: 'console.log("cli")\n', + runtime: { path: `${directory}/runtime.mjs`, content: 'export const runtime = 1\n' }, + } + const verifyCli = sessionCliVerification(cli) + const sandbox: SandboxHandle = await e2bProvider.create('mothership', { + sessionKey: `mothership-chat:live-cli-${generateShortId(10)}`, + lifetimeMs: 10 * 60_000, + }) + const run = (command: string, verify = true) => + sandbox.runCommand(command, { + timeoutMs: TIMEOUT_MS, + atMostOnce: true, + rootUser: false, + ...(verify ? { verifyCli } : {}), + }) + const ranMarker = async (marker: string) => + (await run(`test -e /tmp/${marker} && echo yes || echo no`, false)).stdout.trim() + const expectRefused = async (marker: string) => { + await expect(run(`touch /tmp/${marker}`)).rejects.toBeInstanceOf(SessionCliStaleError) + expect(await ranMarker(marker)).toBe('no') + } + try { + expect(sandbox.verifiesCliAtLaunch).toBe(true) + await check('install CLI', () => + ensureSessionCli(sandbox, cli, AbortSignal.timeout(TIMEOUT_MS), TIMEOUT_MS) + ) + + await check('verified launch runs the command as the user', async () => { + const result = await run(`id -un; test -L ${directory}/sim && echo linked`) + expect(result.exitCode).toBe(0) + expect(result.stdout).toContain('user') + expect(result.stdout).toContain('linked') + return result.stdout.trim() + }) + + await check('warm-call cost: separate check + launch vs verified launch', async () => { + const separate: number[] = [] + const merged: number[] = [] + const unverified: number[] = [] + for (let sample = 0; sample < SAMPLES; sample += 1) { + let startedAt = performance.now() + await run('true', false) + unverified.push(performance.now() - startedAt) + startedAt = performance.now() + await ensureSessionCli(sandbox, cli, AbortSignal.timeout(TIMEOUT_MS), TIMEOUT_MS) + await run('true', false) + separate.push(performance.now() - startedAt) + startedAt = performance.now() + await run('true') + merged.push(performance.now() - startedAt) + } + const result = { + separateMedianMs: median(separate), + mergedMedianMs: median(merged), + unverifiedLaunchMedianMs: median(unverified), + } + return result + }) + + await check('a modified CLI refuses the launch and starts nothing', async () => { + await run(`echo 'tampered' >> ${cli.path}`, false) + await expectRefused('ran-after-edit') + }) + + await check('the user shell and Python hooks cannot fake the check', async () => { + await run( + [ + 'cp -a ~/.profile /tmp/profile.bak 2>/dev/null || true', + `printf 'python3() { return 0; }\\n' >> ~/.profile`, + 'site=$(/usr/bin/env python3 -c "import site; print(site.getusersitepackages())")', + 'mkdir -p "$site"', + `printf 'import sys; sys.exit(0)\\n' > "$site/sim-live.pth"`, + 'echo "$site" > /tmp/sim-live-site', + ].join('\n'), + false + ) + try { + await expectRefused('ran-after-hooks') + const shellCheck = await sandbox.runCommand( + `python3 - <<'SIM_SESSION_CLI'\n${SESSION_CLI_CHECK_SCRIPT}SIM_SESSION_CLI`, + { + timeoutMs: TIMEOUT_MS, + atMostOnce: true, + rootUser: false, + envs: { + SIM_CLI_PATH: verifyCli.path, + SIM_CLI_ARTIFACTS: JSON.stringify(verifyCli.artifacts), + }, + } + ) + await expect( + ensureSessionCli(sandbox, cli, AbortSignal.timeout(TIMEOUT_MS), TIMEOUT_MS) + ).rejects.toThrow('could not be verified') + await expectRefused('ran-after-hooked-repair') + return { + loginShellCheck: + shellCheck.exitCode === 0 ? 'passed a tampered CLI' : 'refused a tampered CLI', + } + } finally { + await run( + 'rm -f "$(cat /tmp/sim-live-site)/sim-live.pth"; if [ -e /tmp/profile.bak ]; then cp -a /tmp/profile.bak ~/.profile; else rm -f ~/.profile; fi', + false + ) + } + }) + + await check('once the hooks are gone, repair restores the CLI', async () => { + await ensureSessionCli(sandbox, cli, AbortSignal.timeout(TIMEOUT_MS), TIMEOUT_MS) + expect((await run(`grep -c tampered ${cli.path} || true`, false)).stdout.trim()).toBe('0') + expect((await run('echo ok')).stdout.trim()).toBe('ok') + }) + + await check('a symlinked artifact refuses the launch', async () => { + await ensureSessionCli(sandbox, cli, AbortSignal.timeout(TIMEOUT_MS), TIMEOUT_MS) + await run(`rm -f ${cli.path} && ln -s /etc/hostname ${cli.path}`, false) + await expectRefused('ran-after-symlink') + }) + + await check('a missing runtime refuses the launch', async () => { + await run(`rm -f ${cli.path} ${cli.runtime?.path}`, false) + await expectRefused('ran-after-delete') + }) + + await check('an unreadable or non-regular artifact is refused and repaired', async () => { + await ensureSessionCli(sandbox, cli, AbortSignal.timeout(TIMEOUT_MS), TIMEOUT_MS) + await run(`chmod 000 ${cli.path}`, false) + await expectRefused('ran-after-chmod') + await ensureSessionCli(sandbox, cli, AbortSignal.timeout(TIMEOUT_MS), TIMEOUT_MS) + await run(`rm -f ${cli.runtime?.path} && mkfifo ${cli.runtime?.path}`, false) + const startedAt = performance.now() + await expectRefused('ran-after-fifo') + const refusedInMs = Math.round(performance.now() - startedAt) + await ensureSessionCli(sandbox, cli, AbortSignal.timeout(TIMEOUT_MS), TIMEOUT_MS) + expect((await run('echo ok')).stdout.trim()).toBe('ok') + return { fifoRefusedInMs: refusedInMs } + }) + + await check('a Stop during a slow check settles and starts nothing', async () => { + await run(`rm -f ${cli.runtime?.path} && truncate -s 6G ${cli.runtime?.path}`, false) + const controller = new AbortController() + const startedAt = performance.now() + setTimeout(() => controller.abort(new Error('User Stop')), 1_000) + // An aborted session command returns its killed process's result; the caller's own + // abort check turns it into the cancellation. + const stopped = await sandbox + .runCommand('touch /tmp/ran-after-stop', { + timeoutMs: TIMEOUT_MS, + atMostOnce: true, + rootUser: false, + verifyCli, + signal: controller.signal, + }) + .then( + (result) => `exit ${result.exitCode}`, + (error: unknown) => getErrorMessage(error) + ) + const settledInMs = Math.round(performance.now() - startedAt) + expect(settledInMs).toBeLessThan(10_000) + expect(await ranMarker('ran-after-stop')).toBe('no') + await ensureSessionCli(sandbox, cli, AbortSignal.timeout(TIMEOUT_MS), TIMEOUT_MS) + return { settledInMs, stopped } + }) + + await check('a repaired CLI launches again', async () => { + await run(`echo 'tampered' >> ${cli.path}`, false) + await expectRefused('ran-before-repair') + await ensureSessionCli(sandbox, cli, AbortSignal.timeout(TIMEOUT_MS), TIMEOUT_MS) + const result = await run('touch /tmp/ran-after-repair && echo done') + expect(result.stdout.trim()).toBe('done') + expect(await ranMarker('ran-after-repair')).toBe('yes') + }) + + await check('a verified launch keeps its exit code and streams', async () => { + const result = await run('echo out; echo err >&2; exit 3') + expect(result).toMatchObject({ exitCode: 3 }) + expect(result.stdout).toContain('out') + expect(result.stderr).toContain('err') + }) + } finally { + await sandbox.kill() + } + }, + 10 * 60_000 + ) +}) diff --git a/apps/sim/lib/execution/remote-sandbox/session-cli.ts b/apps/sim/lib/execution/remote-sandbox/session-cli.ts index b7bdfb0480a..d252ea6cb96 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-cli.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-cli.ts @@ -1,22 +1,30 @@ import { createHash } from 'node:crypto' import { posix } from 'node:path' import { withSandboxFilePublication } from '@/lib/execution/remote-sandbox/file-publication' -import type { SandboxHandle, SandboxSessionRequest } from '@/lib/execution/remote-sandbox/types' +import type { + SandboxHandle, + SandboxSessionRequest, + SessionCliVerification, +} from '@/lib/execution/remote-sandbox/types' -/** A valid release is reused; activation repairs only its own versioned launcher. */ -const CHECK_CLI_COMMAND = `python3 - <<'SIM_SESSION_CLI' -import hashlib, json, os, stat, sys, tempfile +/** + * Exits 0 when every artifact matches its digest, and 10 when one is missing, unreadable, not a + * regular file, a symlink, or changed, so repair overwrites it. Opening never follows a link or + * blocks on a FIFO. A valid release is reused; activation repairs only its own versioned launcher. + */ +export const SESSION_CLI_CHECK_SCRIPT = `import hashlib, json, os, stat, sys, tempfile path = os.environ['SIM_CLI_PATH'] for artifact, expected in json.loads(os.environ['SIM_CLI_ARTIFACTS']).items(): - if os.path.islink(artifact): - sys.exit(10) - digest = hashlib.sha256() try: - with open(artifact, 'rb') as source: - for chunk in iter(lambda: source.read(65536), b''): - digest.update(chunk) - except FileNotFoundError: + descriptor = os.open(artifact, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + except OSError: sys.exit(10) + digest = hashlib.sha256() + with os.fdopen(descriptor, 'rb') as source: + if not stat.S_ISREG(os.fstat(source.fileno()).st_mode): + sys.exit(10) + for chunk in iter(lambda: source.read(65536), b''): + digest.update(chunk) if digest.hexdigest() != expected: sys.exit(10) if stat.S_IMODE(os.stat(path).st_mode) != 0o755: @@ -27,7 +35,54 @@ if not os.path.islink(launcher) or os.readlink(launcher) != os.path.basename(pat staged = os.path.join(directory, 'sim') os.symlink(os.path.basename(path), staged) os.replace(staged, launcher) -SIM_SESSION_CLI` +` + +const CHECK_CLI_COMMAND = `python3 - <<'SIM_SESSION_CLI' +${SESSION_CLI_CHECK_SCRIPT}SIM_SESSION_CLI` + +/** + * A session command refused to start because the deployment CLI it would run with no longer + * matches this release. Nothing the caller asked for ran; repair the CLI and launch again. + */ +export class SessionCliStaleError extends Error { + constructor() { + super('Workbench CLI changed since it was verified') + this.name = 'SessionCliStaleError' + } +} + +/** A launch could not complete the CLI check (an unreadable artifact, a hung check), so it ran nothing. */ +export class SessionCliVerificationError extends Error { + constructor() { + super('Workbench CLI verification failed') + this.name = 'SessionCliVerificationError' + } +} + +const verifications = new WeakMap< + NonNullable, + SessionCliVerification +>() + +/** The digests a launch verifies before it starts the command, hashed once per CLI request. */ +export function sessionCliVerification( + cli: NonNullable +): SessionCliVerification { + let verification = verifications.get(cli) + if (!verification) { + verification = { + path: cli.path, + artifacts: Object.fromEntries( + [cli, ...(cli.runtime ? [cli.runtime] : [])].map(({ path, content }) => [ + path, + createHash('sha256').update(content).digest('hex'), + ]) + ), + } + verifications.set(cli, verification) + } + return verification +} /** Each running script keeps the CLI release chosen by the Sim instance that started it. */ export function sessionCommandPath( @@ -50,24 +105,37 @@ export async function ensureSessionCli( if (remaining <= 0) throw new DOMException('timeout', 'AbortError') return remaining } + const verification = sessionCliVerification(cli) + /** + * A handle that verifies at launch checks through its own supervisor, which the user's shell + * profile and Python site hooks cannot reach; the shell form sources the user's login profile. + */ const check = async (): Promise => { - const result = await sandbox.runCommand(CHECK_CLI_COMMAND, { - envs: { - SIM_CLI_PATH: cli.path, - SIM_CLI_ARTIFACTS: JSON.stringify( - Object.fromEntries( - [cli, ...(cli.runtime ? [cli.runtime] : [])].map(({ path, content }) => [ - path, - createHash('sha256').update(content).digest('hex'), - ]) - ) - ), - }, + const options = { timeoutMs: Math.min(30_000, remainingMs()), maxOutputBytes: 64 * 1024, rootUser: false, atMostOnce: true, signal, + } + if (sandbox.verifiesCliAtLaunch) { + try { + const result = await sandbox.runCommand('true', { ...options, verifyCli: verification }) + signal.throwIfAborted() + if (result.timedOut) throw new DOMException('timeout', 'AbortError') + if (result.exitCode !== 0) throw new SessionCliVerificationError() + return true + } catch (error) { + if (error instanceof SessionCliStaleError) return false + throw error + } + } + const result = await sandbox.runCommand(CHECK_CLI_COMMAND, { + ...options, + envs: { + SIM_CLI_PATH: verification.path, + SIM_CLI_ARTIFACTS: JSON.stringify(verification.artifacts), + }, }) signal.throwIfAborted() if (result.timedOut) throw new DOMException('timeout', 'AbortError') diff --git a/apps/sim/lib/execution/remote-sandbox/session-process.ts b/apps/sim/lib/execution/remote-sandbox/session-process.ts index 940cbc79277..313072ec171 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-process.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-process.ts @@ -1,3 +1,6 @@ +import { SESSION_CLI_CHECK_SCRIPT } from '@/lib/execution/remote-sandbox/session-cli' +import type { SessionCliVerification } from '@/lib/execution/remote-sandbox/types' + export interface SessionProcessIdentity { id: string sandboxId: string @@ -7,8 +10,15 @@ export interface SessionProcessIdentity { /** * PID 1 owns every descendant, including new sessions and double forks. A root-only * tombstone serializes Stop with launch; pidfds confirm exit without signalling a reused PID. + * + * With a CLI verification, PID 1 checks the deployment CLI as the ordinary user after recording + * itself and before it starts the command. The check runs outside the lock, so Stop can kill the + * namespace while it runs; the closed tombstone is re-read before the command starts. The isolated + * interpreter ignores the user's environment and site packages, which the user can edit. A + * mismatch starts nothing and is reported on the Stop receipt. */ const PROCESS_RUNTIME = `import base64, contextlib, fcntl, json, os, pathlib, pwd, re, select, signal, subprocess, sys, tempfile +CLI_CHECK = base64.b64decode('${Buffer.from(SESSION_CLI_CHECK_SCRIPT).toString('base64')}').decode() action, identity = sys.argv[1:3] if not re.fullmatch(r"[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}", identity): @@ -60,7 +70,7 @@ if action == 'run': elif action == 'init': if os.getpid() != 1: raise RuntimeError('Command requires its own PID namespace') - parent_proc = int(sys.argv[5]) + parent_proc = int(sys.argv[6]) with locked() as state: if state.get('closed'): sys.exit(125) @@ -71,14 +81,37 @@ elif action == 'init': namespace=os.readlink('self/ns/pid', dir_fd=parent_proc)) save(state) os.close(parent_proc) + def entering(account): + def enter(): + os.initgroups(account.pw_name, account.pw_gid) + os.setgid(account.pw_gid) + os.setuid(account.pw_uid) + return enter + refusal = None + if sys.argv[5] != '-': + expected = json.loads(base64.b64decode(sys.argv[5])) + checker = pwd.getpwnam('user') + try: + checked = subprocess.run([sys.executable, '-I', '-c', CLI_CHECK], cwd=checker.pw_dir, + env={'HOME': checker.pw_dir, 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'SIM_CLI_PATH': expected['path'], 'SIM_CLI_ARTIFACTS': json.dumps(expected['artifacts'])}, + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + preexec_fn=entering(checker), timeout=30).returncode + except subprocess.TimeoutExpired: + checked = None + if checked != 0: + refusal = 'stale' if checked == 10 else 'failed' + with locked() as state: + if state.get('closed'): + sys.exit(125) + if refusal: + state['cli'] = refusal + save(state) + sys.exit(125) owner = pwd.getpwnam(sys.argv[4]) - def enter_user(): - os.initgroups(owner.pw_name, owner.pw_gid) - os.setgid(owner.pw_gid) - os.setuid(owner.pw_uid) environment = dict(os.environ, HOME=owner.pw_dir, USER=owner.pw_name, LOGNAME=owner.pw_name) child = subprocess.Popen(['/bin/bash', '-lc', base64.b64decode(sys.argv[3]).decode()], - cwd=owner.pw_dir, env=environment, preexec_fn=enter_user) + cwd=owner.pw_dir, env=environment, preexec_fn=entering(owner)) while True: waited_pid, waited_status = os.wait() if waited_pid == child.pid: @@ -114,7 +147,7 @@ elif action == 'stop': raise RuntimeError('Command namespace exit is unconfirmed') finally: os.close(descriptor) - print(json.dumps({'settled': True})) + print(json.dumps({'settled': True, 'cli': state.get('cli')})) else: raise ValueError('Invalid command action') ` @@ -129,8 +162,14 @@ function commandIdentity(identity: string): string { } /** The caller invokes the supervisor as root; the requested command retains its ordinary user. */ -export function sessionProcessCommand(identity: string, command: string, rootUser = false): string { - return `${PROCESS_LAUNCHER} run ${commandIdentity(identity)} '${Buffer.from(command).toString('base64')}' ${rootUser ? 'root' : 'user'}` +export function sessionProcessCommand( + identity: string, + command: string, + rootUser = false, + verifyCli?: SessionCliVerification +): string { + const verification = verifyCli ? Buffer.from(JSON.stringify(verifyCli)).toString('base64') : '-' + return `${PROCESS_LAUNCHER} run ${commandIdentity(identity)} '${Buffer.from(command).toString('base64')}' ${rootUser ? 'root' : 'user'} '${verification}'` } /** Safe before launch as well as after a lost launch reply: the tombstone prevents future work. */ diff --git a/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts b/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts index 90bc0e361a8..93214c32c69 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts @@ -14,12 +14,14 @@ import { sleep } from '@sim/utils/helpers' import { beforeEach, describe, expect, it, vi } from 'vitest' import { CodeLanguage } from '@/lib/execution/languages' import { SANDBOX_OUTPUT_DIR_SENTINEL } from '@/lib/execution/remote-sandbox/paths' +import { SessionCliStaleError } from '@/lib/execution/remote-sandbox/session-cli' import type { SandboxCodeResult, SandboxCommandResult, SandboxHandle, SandboxProvider, SandboxSessionRequest, + SessionCliVerification, } from '@/lib/execution/remote-sandbox/types' const { mockCreate, mockFindSessionSandbox, mockResolveWorkspaceSandbox } = vi.hoisted(() => ({ @@ -166,6 +168,46 @@ function slowDown(handle: SandboxHandle): void { const CLI = { path: '/home/user/.sim-cli/release/cli.mjs', content: 'deployment-code' } +/** + * A session handle whose launches verify the CLI first, as E2B's supervisor does: a launch whose + * CLI no longer matches starts nothing and rejects. + */ +function launchVerifyingSandbox(id: string, options: { alwaysStale?: boolean } = {}) { + const { handle: base, calls } = fakeSandbox(id) + const ran: string[] = [] + const unverifiedLaunches: string[] = [] + const verify = async (launched: string, verifyCli: SessionCliVerification | undefined) => { + if (!verifyCli) { + unverifiedLaunches.push(launched) + return + } + const check = await base.runCommand('verify', { + timeoutMs: 1_000, + envs: { + SIM_CLI_PATH: verifyCli.path, + SIM_CLI_ARTIFACTS: JSON.stringify(verifyCli.artifacts), + }, + }) + calls.runCommand.pop() + if (options.alwaysStale || check.exitCode !== 0) throw new SessionCliStaleError() + } + const handle: SandboxHandle = { + ...base, + verifiesCliAtLaunch: true, + async runCommand(command, runOptions) { + await verify(command, runOptions.verifyCli) + ran.push(command) + return base.runCommand(command, runOptions) + }, + async runCode(code, runOptions) { + await verify(code, runOptions.verifyCli) + ran.push(code) + return base.runCode(code, runOptions) + }, + } + return { handle, calls, ran, unverifiedLaunches } +} + const CODE_REQUEST = { code: 'print(1)', language: CodeLanguage.Python, @@ -811,6 +853,56 @@ describe('session sandbox lease', () => { } }) + describe('a reused workbench that verifies its CLI at launch', () => { + it.each(['code', 'shell'] as const)( + 'runs %s with no separate CLI check command', + async (kind) => { + const { handle, calls, ran, unverifiedLaunches } = launchVerifyingSandbox(`launch-${kind}`) + mockFindSessionSandbox.mockResolvedValue(handle) + const request = { ...CODE_REQUEST, session: { key: `launch-${kind}`, cli: CLI } } + const execute = () => + kind === 'code' + ? executeInSandbox(request) + : executeShellInSandbox({ ...request, envs: {} }) + await execute() + calls.runCommand.length = 0 + ran.length = 0 + await execute() + expect(calls.runCommand.filter(({ envs }) => envs?.SIM_CLI_PATH)).toEqual([]) + expect(ran).toEqual([CODE_REQUEST.code]) + expect(unverifiedLaunches).not.toContain(CODE_REQUEST.code) + } + ) + + it.each(['code', 'shell'] as const)( + 'repairs a changed CLI and runs the %s exactly once', + async (kind) => { + const { handle, calls, ran, unverifiedLaunches } = launchVerifyingSandbox(`repair-${kind}`) + mockFindSessionSandbox.mockResolvedValue(handle) + const request = { ...CODE_REQUEST, session: { key: `repair-${kind}`, cli: CLI } } + await (kind === 'code' + ? executeInSandbox(request) + : executeShellInSandbox({ ...request, envs: {} })) + expect(ran.filter((launched) => launched === CODE_REQUEST.code)).toHaveLength(1) + expect(calls.runCommand.some(({ envs }) => envs?.SIM_FILE_TARGET === CLI.path)).toBe(true) + expect(unverifiedLaunches).not.toContain(CODE_REQUEST.code) + } + ) + + it('fails without running the code when the CLI cannot be kept verified', async () => { + const { handle, ran } = launchVerifyingSandbox('stale', { alwaysStale: true }) + mockFindSessionSandbox.mockResolvedValue(handle) + await expect( + executeShellInSandbox({ + ...CODE_REQUEST, + envs: {}, + session: { key: 'stale', cli: CLI }, + }) + ).rejects.toThrow('Workbench CLI installation could not be verified') + expect(ran).not.toContain(CODE_REQUEST.code) + }) + }) + it('does not rewrite an unchanged executable while earlier code can still use it', async () => { const { handle } = fakeSandbox('unchanged-tooling') mockFindSessionSandbox.mockResolvedValue(handle) diff --git a/apps/sim/lib/execution/remote-sandbox/session.ts b/apps/sim/lib/execution/remote-sandbox/session.ts index cdd9cb01b10..1fc3ffe628c 100644 --- a/apps/sim/lib/execution/remote-sandbox/session.ts +++ b/apps/sim/lib/execution/remote-sandbox/session.ts @@ -99,7 +99,8 @@ export async function ensureSessionSandbox(args: { if (!created.sandbox.outlives?.(lifetimeMs, requestedAtMs)) await created.sandbox.extendLifetime?.(lifetimeMs) signal.throwIfAborted() - if (session.cli) { + // A reused machine's commands verify the CLI as they launch, so only a new one installs it here. + if (session.cli && !(existing && created.sandbox.verifiesCliAtLaunch)) { const endCli = args.phases?.start('cli') await ensureSessionCli(created.sandbox, session.cli, signal, args.bootstrapTimeoutMs) endCli?.() diff --git a/apps/sim/lib/execution/remote-sandbox/types.ts b/apps/sim/lib/execution/remote-sandbox/types.ts index 254b513d784..9c4131116da 100644 --- a/apps/sim/lib/execution/remote-sandbox/types.ts +++ b/apps/sim/lib/execution/remote-sandbox/types.ts @@ -236,6 +236,12 @@ export interface SandboxCodeResult { providerFailure?: 'provider_limit' } +/** Expected digests of a session's deployment CLI artifacts, keyed by sandbox path. */ +export interface SessionCliVerification { + path: string + artifacts: Record +} + export interface RunCommandOptions { envs?: Record timeoutMs: number @@ -247,6 +253,11 @@ export interface RunCommandOptions { rootUser?: boolean /** Never relaunch this command when provider acknowledgement is ambiguous. */ atMostOnce?: boolean + /** + * Session commands only: start the command only if these artifacts still match. A mismatch + * rejects with `SessionCliStaleError` and runs nothing. + */ + verifyCli?: SessionCliVerification onStdout?: (chunk: string) => void onStderr?: (chunk: string) => void } @@ -257,6 +268,11 @@ export interface RunCommandOptions { */ export interface SandboxHandle { readonly sandboxId: string + /** + * Every command this session handle launches honors `verifyCli` inside its own owned process, + * so a reused session needs no separate verification command. + */ + readonly verifiesCliAtLaunch?: boolean /** * Runs code in the language fixed at {@link SandboxProvider.create} time. * Language is bound at creation rather than per call because Daytona applies it @@ -273,6 +289,7 @@ export interface SandboxHandle { javascriptPreload?: string maxOutputBytes?: number signal?: AbortSignal + verifyCli?: SessionCliVerification } ): Promise runCommand(command: string, options: RunCommandOptions): Promise