improvement(sandbox): verify the workbench CLI inside each command's launch

apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts

@@ -51,6 +51,7 @@ import {
   stopE2BSessionProcess,
 } from '@/lib/execution/remote-sandbox/e2b'
 import { observeSandboxExecution } from '@/lib/execution/remote-sandbox/observer'
+import { SessionCliStaleError } from '@/lib/execution/remote-sandbox/session-cli'
 
 setEnv({ E2B_API_KEY: 'test', MOTHERSHIP_E2B_TEMPLATE_ID: 'mothership-template' })
 
@@ -643,6 +644,36 @@ describe('E2B session recovery', () => {
       stderr: 'slow frame\n',
     })
   })
+  it.each([
+    ['stale', SessionCliStaleError],
+    ['failed', Error],
+  ] as const)(
+    'rejects a launch whose receipt reports a %s CLI, whatever its process returned',
+    async (cli, expected) => {
+      runCommand.mockResolvedValue({
+        pid: 9,
+        wait: async () => ({ stdout: '', stderr: '', exitCode: 0 }),
+      })
+      stopCommand.mockResolvedValue({
+        stdout: JSON.stringify({ settled: true, cli }),
+        stderr: '',
+        exitCode: 0,
+      })
+      const sandbox = await sessionSandbox('reconnected')
+      const launched = sandbox?.runCommand('touch /tmp/marker', {
+        timeoutMs: 1000,
+        verifyCli: { path: '/home/user/.sim-cli/r/cli.mjs', artifacts: { a: 'b' } },
+      })
+      await expect(launched).rejects.toBeInstanceOf(expected)
+      await expect(launched).rejects.toThrow(cli === 'stale' ? 'changed' : 'verification failed')
+      const [command] = runCommand.mock.calls.at(-1) ?? []
+      expect(command).toContain(
+        Buffer.from(
+          JSON.stringify({ path: '/home/user/.sim-cli/r/cli.mjs', artifacts: { a: 'b' } })
+        ).toString('base64')
+      )
+    }
+  )
 })
 
 describe('E2B session lease', () => {

apps/sim/lib/execution/remote-sandbox/e2b.ts

@@ -55,6 +55,10 @@ import {
   tailStreamedSandboxOutput,
 } from '@/lib/execution/remote-sandbox/output-limits'
 import { resolveSandboxDirectoryEntryPath } from '@/lib/execution/remote-sandbox/paths'
+import {
+  SessionCliStaleError,
+  SessionCliVerificationError,
+} from '@/lib/execution/remote-sandbox/session-cli'
 import { withSandboxSessionLock } from '@/lib/execution/remote-sandbox/session-lock'
 import {
   type SessionProcessIdentity,
@@ -82,6 +86,7 @@ import type {
   SandboxImageMaterialization,
   SandboxKind,
   SandboxProvider,
+  SessionCliVerification,
 } from '@/lib/execution/remote-sandbox/types'
 
 const logger = createLogger('E2BSandboxProvider')
@@ -319,11 +324,17 @@ async function listAllSandboxes(paginator: SandboxPaginator): Promise<SandboxInf
   return sandboxes
 }
 
+/** What a session command's launch found when it verified the deployment CLI; absent when it matched. */
+interface SessionCommandReceipt {
+  cli?: 'stale' | 'failed'
+}
+
+/** Settles the command; the receipt is the only proof that a refused launch started nothing. */
 async function stopSessionCommand(
   sandbox: E2BSandbox,
   processId: string,
   signal = AbortSignal.timeout(10_000)
-): Promise<void> {
+): Promise<SessionCommandReceipt> {
   signal.throwIfAborted()
   const result = await sandbox.commands.run(sessionProcessStopCommand(processId), {
     user: 'root',
@@ -335,6 +346,7 @@ async function stopSessionCommand(
   if (result.exitCode !== 0 || !isRecordLike(receipt) || receipt.settled !== true) {
     throw new Error('Sandbox command namespace exit is unconfirmed')
   }
+  return receipt.cli === 'stale' || receipt.cli === 'failed' ? { cli: receipt.cli } : {}
 }
 
 /** Recovers the exact recorded workbench; expiry is absence, while lookup failure stays unresolved. */
@@ -413,6 +425,10 @@ class E2BSandboxHandle implements SandboxHandle {
     return this.sandbox.sandboxId
   }
 
+  get verifiesCliAtLaunch(): boolean {
+    return this.sessionKey !== undefined
+  }
+
   outlives(lifetimeMs: number, fromMs: number): boolean {
     return (
       this.sessionDeadlineAtMs !== undefined &&
@@ -441,6 +457,7 @@ class E2BSandboxHandle implements SandboxHandle {
       javascriptPreload?: string
       maxOutputBytes?: number
       signal?: AbortSignal
+      verifyCli?: SessionCliVerification
     }
   ): Promise<SandboxCodeResult> {
     const runId = generateShortId(12)
@@ -467,6 +484,7 @@ class E2BSandboxHandle implements SandboxHandle {
           maxOutputBytes: options.maxOutputBytes,
           signal: options.signal,
           atMostOnce: true,
+          verifyCli: options.verifyCli,
           envs: {
             ...options.envs,
             SIM_CODE_PATH: codePath,
@@ -519,6 +537,25 @@ class E2BSandboxHandle implements SandboxHandle {
     command: string,
     options: RunCommandOptions,
     operation: 'code' | 'command'
+  ): Promise<SandboxCommandResult> {
+    let refusal: SessionCommandReceipt['cli']
+    const outcome = await this.runOwnedCommand(command, options, operation, (receipt) => {
+      refusal = receipt.cli
+    }).then(
+      (result) => ({ result }),
+      (error: unknown) => ({ error })
+    )
+    if (refusal === 'stale') throw new SessionCliStaleError()
+    if (refusal === 'failed') throw new SessionCliVerificationError()
+    if ('error' in outcome) throw outcome.error
+    return outcome.result
+  }
+
+  private async runOwnedCommand(
+    command: string,
+    options: RunCommandOptions,
+    operation: 'code' | 'command',
+    onReceipt: (receipt: SessionCommandReceipt) => void
   ): Promise<SandboxCommandResult> {
     if (this.sessionKey !== undefined) options.signal?.throwIfAborted()
     let commandDeadlineAtMs: number | undefined
@@ -537,9 +574,9 @@ class E2BSandboxHandle implements SandboxHandle {
     let observedStderr = ''
     const processId = this.sessionKey !== undefined ? generateId() : undefined
     let processRecorded = false
-    let stopping: Promise<void> | undefined
-    const stopCommand = (): Promise<void> => {
-      if (!processId || !processRecorded) return Promise.resolve()
+    let stopping: Promise<SessionCommandReceipt> | undefined
+    const stopCommand = (): Promise<SessionCommandReceipt | undefined> => {
+      if (!processId || !processRecorded) return Promise.resolve(undefined)
       stopping ??= stopSessionCommand(this.sandbox, processId).catch((error: unknown) => {
         logger.warn('Failed to stop sandbox process', {
           sandboxId: this.sandboxId,
@@ -595,7 +632,12 @@ class E2BSandboxHandle implements SandboxHandle {
         commandDeadlineAtMs = Date.now() + processOptions.timeoutMs
         started = await this.sandbox.commands.run(
           processId
-            ? sessionProcessCommand(processId, prepared.command, options.rootUser)
+            ? sessionProcessCommand(
+                processId,
+                prepared.command,
+                options.rootUser,
+                options.verifyCli
+              )
             : prepared.command,
           {
             ...processOptions,
@@ -753,11 +795,12 @@ class E2BSandboxHandle implements SandboxHandle {
       options.signal?.removeEventListener('abort', onAbort)
       /** The named SDK process can exit before its namespace; only the kernel receipt settles it. */
       if (processId && processRecorded) {
-        await stopCommand().catch((error: unknown) => {
+        const receipt = await stopCommand().catch((error: unknown) => {
           reportUnsettledSandboxProcess(processId)
           throw error
         })
         await settleSandboxProcess(processId)
+        if (receipt) onReceipt(receipt)
       }
     }
   }

apps/sim/lib/execution/remote-sandbox/index.ts

@@ -60,7 +60,12 @@ import {
   ensureSessionSandbox,
   SESSION_SANDBOX_IDLE_MS,
 } from '@/lib/execution/remote-sandbox/session'
-import { sessionCommandPath } from '@/lib/execution/remote-sandbox/session-cli'
+import {
+  ensureSessionCli,
+  SessionCliStaleError,
+  sessionCliVerification,
+  sessionCommandPath,
+} from '@/lib/execution/remote-sandbox/session-cli'
 import {
   readSessionSecretProvenance,
   recordSessionFileInput,
@@ -87,6 +92,7 @@ import type {
   SandboxProviderId,
   SandboxSessionRequest,
   SandboxShellExecutionRequest,
+  SessionCliVerification,
 } from '@/lib/execution/remote-sandbox/types'
 
 export type {
@@ -259,6 +265,42 @@ async function withSandboxExecutionBudget<T>(
   }
 }
 
+/**
+ * Launches a session command that verifies the deployment CLI as it starts. A launch that found
+ * the CLI changed started nothing, so the CLI is repaired under the session lock and the command
+ * launched once more; a second refusal means something keeps changing it, and the call fails.
+ */
+async function launchWithVerifiedCli<T>(
+  sandbox: SandboxHandle,
+  lease: SandboxLease,
+  session: SandboxSessionRequest | undefined,
+  signal: AbortSignal,
+  phases: SandboxPhaseTimer,
+  launch: (verifyCli: SessionCliVerification | undefined) => Promise<T>
+): Promise<T> {
+  const cli = lease.session && sandbox.verifiesCliAtLaunch ? session?.cli : undefined
+  if (!session || !cli) return launch(undefined)
+  const verifyCli = sessionCliVerification(cli)
+  try {
+    return await launch(verifyCli)
+  } catch (error) {
+    if (!(error instanceof SessionCliStaleError)) throw error
+  }
+  const endCli = phases.start('cli')
+  await withSandboxSessionLock(session.key, signal, (lockSignal) =>
+    ensureSessionCli(sandbox, cli, lockSignal, remainingSandboxBudgetMs(signal))
+  )
+  endCli()
+  try {
+    return await launch(verifyCli)
+  } catch (error) {
+    if (error instanceof SessionCliStaleError) {
+      throw new Error('Workbench CLI installation could not be verified')
+    }
+    throw error
+  }
+}
+
 function throwIfSandboxTimedOut(result: { timedOut?: boolean }): void {
   if (result.timedOut) throw new DOMException('timeout', 'AbortError')
 }
@@ -1008,16 +1050,27 @@ async function executeInSandboxWithinBudget(
       Object.keys(privateInputFiles.environment).length > 0
 
     let execution: SandboxCodeResult
-    const runTimeoutMs = codeBudgetMs(req, signal)
+    let runTimeoutMs = 0
     try {
       const endRun = phases.start('run')
-      execution = await sandbox.runCode(code, {
-        timeoutMs: runTimeoutMs,
-        javascriptPreload: buildJavaScriptRuntimeBindingsSource(req.runtimeBindings ?? []),
-        maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
+      execution = await launchWithVerifiedCli(
+        sandbox,
+        lease,
+        req.session,
         signal,
-        ...(hasExecutionEnvironment ? { envs: executionEnvironment } : {}),
-      })
+        phases,
+        (verifyCli) => {
+          runTimeoutMs = codeBudgetMs(req, signal)
+          return sandbox.runCode(code, {
+            timeoutMs: runTimeoutMs,
+            javascriptPreload: buildJavaScriptRuntimeBindingsSource(req.runtimeBindings ?? []),
+            maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
+            signal,
+            ...(hasExecutionEnvironment ? { envs: executionEnvironment } : {}),
+            verifyCli,
+          })
+        }
+      )
       endRun()
     } catch (error) {
       throwIfAborted(signal)
@@ -1212,24 +1265,36 @@ async function executeShellInSandboxWithinBudget(
     endPrepare()
 
     let result: SandboxCommandResult
-    const runTimeoutMs = codeBudgetMs(req, signal)
+    let runTimeoutMs = 0
     try {
       const endRun = phases.start('run')
-      result = await sandbox.runCommand(code, {
-        envs: {
-          ...selected?.envs,
-          ...envs,
-          ...req.session?.envs,
-          PATH: sessionCommandPath(req.session, selected?.envs?.PATH ?? SANDBOX_SYSTEM_PATH),
-          ...privateInputFiles.environment,
-          ...(req.session && req.outputSandboxDir ? { SIM_OUTPUT_DIR: req.outputSandboxDir } : {}),
-        },
-        timeoutMs: runTimeoutMs,
-        maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
+      const commandEnvironment = {
+        ...selected?.envs,
+        ...envs,
+        ...req.session?.envs,
+        PATH: sessionCommandPath(req.session, selected?.envs?.PATH ?? SANDBOX_SYSTEM_PATH),
+        ...privateInputFiles.environment,
+        ...(req.session && req.outputSandboxDir ? { SIM_OUTPUT_DIR: req.outputSandboxDir } : {}),
+      }
+      result = await launchWithVerifiedCli(
+        sandbox,
+        lease,
+        req.session,
         signal,
-        rootUser: !lease.session,
-        atMostOnce: true,
-      })
+        phases,
+        (verifyCli) => {
+          runTimeoutMs = codeBudgetMs(req, signal)
+          return sandbox.runCommand(code, {
+            envs: commandEnvironment,
+            timeoutMs: runTimeoutMs,
+            maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
+            signal,
+            rootUser: !lease.session,
+            atMostOnce: true,
+            verifyCli,
+          })
+        }
+      )
       endRun()
     } catch (error) {
       throwIfAborted(signal)

apps/sim/lib/execution/remote-sandbox/phase-timing.ts

@@ -1,7 +1,8 @@
 /**
  * Phases of one sandbox execution, in the order they run. `lease` spans the whole acquisition
- * (session lock included); for a session, `lookup` and `cli` are the parts of it spent finding or
- * creating the machine and verifying its deployment tooling.
+ * (session lock included); for a session, `lookup` is the part of it spent finding or creating the
+ * machine. `cli` is time spent installing or repairing deployment tooling: on a new machine, or
+ * after a launch refused a changed CLI. A reused machine verifies it inside `run`.
  */
 type SandboxPhase =
   | 'resolve'