From 8c8efbc54d51defd2ba321cd289a7a0ed532f5d5 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Tue, 27 Oct 2026 22:47:32 +0000 Subject: [PATCH 1/2] Use stagehand-middleware for Bell browser tools --- agent/agent.ts | 1 + docs/browser-runtime.md | 13 +- docs/provider-tests.md | 18 +++ docs/repository.md | 5 +- next.config.ts | 2 +- package.json | 1 + pnpm-lock.yaml | 14 ++ server/browser/browser-session.ts | 141 ++++++++---------- tests/browser/browser-security.test.ts | 194 +++++++++++++++++++++++-- tests/browser/captcha-recovery.test.ts | 12 +- tests/browser/middleware-real.test.ts | 168 +++++++++++++++++++++ tests/browser/stagehand-fixture.ts | 23 +++ tests/workspace-runtime.test.ts | 25 +++- 13 files changed, 516 insertions(+), 101 deletions(-) create mode 100644 tests/browser/middleware-real.test.ts create mode 100644 tests/browser/stagehand-fixture.ts diff --git a/agent/agent.ts b/agent/agent.ts index 05bd7cc..bc45d0e 100644 --- a/agent/agent.ts +++ b/agent/agent.ts @@ -8,6 +8,7 @@ export default defineAgent({ "@1password/sdk", "@browserbasehq/sdk", "@browserbasehq/stagehand", + "stagehand-middleware", "playwright-core", "pg", ], diff --git a/docs/browser-runtime.md b/docs/browser-runtime.md index f0c57bf..e04e612 100644 --- a/docs/browser-runtime.md +++ b/docs/browser-runtime.md @@ -1,7 +1,8 @@ # Browser runtime Bell's Eve by Vercel agent owns conversation history and tool execution. Browser tools share one in-process -Stagehand runtime per Eve session. Follow-up turns reuse the browser; saved Browserbase session +Stagehand runtime per Eve session, with browser execution provided by the published +[`stagehand-middleware`](https://www.npmjs.com/package/stagehand-middleware) package. Follow-up turns reuse the browser; saved Browserbase session metadata supports reconnecting after a worker restart. Hosted launches request managed proxies, Verified Browsers, and keep-alive on every attempt, including retries. @@ -10,6 +11,16 @@ Verified Browsers, and keep-alive on every attempt, including retries. embedded credentials. A destination URL cannot change the browser provider. Local fixtures require `STAGEHAND_BROWSER=local`. +Bell keeps its typed `run({ actions })` interface: callers can mix CSS selectors and IDs from +`snapshot` in one batch. The runtime translates these actions into one middleware `run` call, +with entered values encoded as string literals. Snapshot capture retains the tree, target map, +URL, and title together; screenshots use the middleware's PNG output. Model-authored JavaScript +is not accepted by Bell's `run` tool. + +Bell owns the browser's launch, reconnect, CAPTCHA gates, privacy policy, and release lifecycle. +The middleware wraps that existing Stagehand client and does not launch a second browser or +keeper tab. Protected login and hosted integrations continue under Bell's operation lock. + ## Configuration Copy the root [`.env.example`](../.env.example) for Bell and integration settings. diff --git a/docs/provider-tests.md b/docs/provider-tests.md index da82281..a13583c 100644 --- a/docs/provider-tests.md +++ b/docs/provider-tests.md @@ -4,6 +4,24 @@ Run `pnpm check` for the normal credential-free checks. Live provider tests are opt-in and use real service resources. Keep credentials in ignored environment files. Browserbase must use `verified: true` and managed proxies, including after launch retries. +## Browser middleware + +The middleware regression uses a synthetic form to exercise navigation, CSS selectors, +snapshot IDs, quoted input, keyboard actions, screenshots, browser reuse, and cleanup. +Run it on local Chrome: + +```sh +BELL_LIVE_MIDDLEWARE=1 STAGEHAND_BROWSER=local pnpm exec vitest run tests/browser/middleware-real.test.ts +``` + +With `BROWSERBASE_API_KEY` available in the process environment, run the same check hosted: + +```sh +BELL_LIVE_MIDDLEWARE=1 STAGEHAND_BROWSER=browserbase pnpm exec vitest run tests/browser/middleware-real.test.ts +``` + +The hosted fixture is served over CDP and the test releases its browser when finished. + ## 1Password login and OTP Use a dedicated test vault with a Login item titled exactly diff --git a/docs/repository.md b/docs/repository.md index f77d120..437aa13 100644 --- a/docs/repository.md +++ b/docs/repository.md @@ -60,8 +60,9 @@ See [browser configuration and credential handling](browser-runtime.md). ## License and ownership -Bell maintains its browser implementation directly in this repository. There is no submodule -or separately maintained application to install. The root [MIT license](../LICENSE) preserves +Bell maintains its browser lifecycle and tool adapters in this repository and uses the published +`stagehand-middleware` package for browser execution. There is no submodule or separate application +to install. The root [MIT license](../LICENSE) preserves the Browserbase copyright and license terms covering the browser source. ## Browser retirement diff --git a/next.config.ts b/next.config.ts index 9182661..78c832a 100644 --- a/next.config.ts +++ b/next.config.ts @@ -4,7 +4,7 @@ import type { NextConfig } from "next"; const config: NextConfig = { agentRules: false, allowedDevOrigins: ["127.0.0.1", "localhost"], - serverExternalPackages: ["@browserbasehq/sdk", "@browserbasehq/stagehand", "playwright-core", "pg"], + serverExternalPackages: ["@browserbasehq/sdk", "@browserbasehq/stagehand", "stagehand-middleware", "playwright-core", "pg"], turbopack: { root: process.cwd() }, }; export default withEve(config, { eveBuildCommand: "pnpm build:agent" }); diff --git a/package.json b/package.json index 9d283d3..d2ef8c0 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "playwright-core": "1.62.1", "react": "19.3.0", "react-dom": "19.3.0", + "stagehand-middleware": "0.1.1", "zod": "4.6.7" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e71bd2c..c439ddb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -49,6 +49,9 @@ importers: react-dom: specifier: 19.3.0 version: 19.3.0(react@19.3.0) + stagehand-middleware: + specifier: 0.1.1 + version: 0.1.1(@browserbasehq/stagehand@4.3.0) zod: specifier: 4.6.7 version: 4.6.7 @@ -2434,6 +2437,12 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + stagehand-middleware@0.1.1: + resolution: {integrity: sha512-q9tZ8Gay5dksY1uczUiq4o3bMxXmg2l6SZhvSDC9pI72vN9JQWfswzkNWTVGNj9yUN+OBfqNyMN8tu78+8I7mQ==} + engines: {node: '>=22.18.0'} + peerDependencies: + '@browserbasehq/stagehand': '>=4.3.0 <5' + std-env@4.3.0: resolution: {integrity: sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==} @@ -5125,6 +5134,11 @@ snapshots: stackback@0.0.2: {} + stagehand-middleware@0.1.1(@browserbasehq/stagehand@4.3.0): + dependencies: + '@browserbasehq/stagehand': 4.3.0 + zod: 4.6.7 + std-env@4.3.0: {} stop-iteration-iterator@1.1.0: diff --git a/server/browser/browser-session.ts b/server/browser/browser-session.ts index bf4b94c..ee8ba8e 100644 --- a/server/browser/browser-session.ts +++ b/server/browser/browser-session.ts @@ -1,5 +1,6 @@ import { personalBrowserEnabled, requireReusableContexts } from "../onepassword/browser-policy"; import Browserbase from "@browserbasehq/sdk"; +import { StagehandFacadeTools } from "stagehand-middleware"; import { localBrowser, Stagehand, @@ -41,6 +42,7 @@ const DISCOVERY_HOSTS = ["google.", "bing.com", "duckduckgo.com", "search.brave. export class BrowserSession { private stagehand?: Stagehand; + private facade?: StagehandFacadeTools; private browser?: StagehandBrowser; private browserbase?: Browserbase; private remoteSessionId?: string; @@ -212,14 +214,7 @@ export class BrowserSession { await this.start(); await this.gateCaptcha(); const started = performance.now(); - const result = await this.requireStagehand().experimentalBatch(async ({ page }) => { - const snapshot = await page.snapshot({ includeIframes: true }); - return { - snapshot, - url: await page.url(), - title: await page.title(), - }; - }); + const result = await this.captureSnapshot(); await this.gateCaptcha(); this.xpathMap = result.snapshot.xpathMap; const url = result.url; @@ -288,40 +283,8 @@ export class BrowserSession { ? { ...item, target: this.resolveTarget(item.target) } : item, ); - const result = await this.requireStagehand().experimentalBatch( - async ({ page }, batchActions) => { - for (const item of batchActions) { - switch (item.action) { - case "goto": - await page.goto(item.url); - await page.waitForLoadState("domcontentloaded", 15_000); - break; - case "click": - await page.locator(item.target).click(); - break; - case "fill": - await page.locator(item.target).fill(item.value); - break; - case "type": - await page.locator(item.target).type(item.value); - break; - case "press": - if (item.target) await page.locator(item.target).click(); - await page.keyPress(item.key); - break; - case "select": - await page.locator(item.target).selectOption(item.value); - break; - case "wait": - await page.waitForTimeout(Math.min(item.milliseconds, 10_000)); - break; - } - } - return { url: await page.url(), title: await page.title() }; - }, - resolvedActions, - { timeout: 60_000 }, - ); + const result = await this.requireFacade().run(`${actionCode(resolvedActions)} +return { url: await page.evaluate(() => location.href), title: await page.title() };`) as { url: string; title: string }; await this.gateCaptcha(); this.xpathMap = {}; @@ -460,13 +423,13 @@ export class BrowserSession { await this.start(); await this.gateCaptcha(); const started = performance.now(); - const result = await this.requireStagehand().experimentalBatch(async ({ page }) => ({ - data: await page.screenshot({ type: "png" }), - url: await page.url(), - title: await page.title(), - })); + const facade = this.requireFacade(); + const image = await facade.screenshot({ type: "png" }); + const result = await facade.run( + "return { url: await page.evaluate(() => location.href), title: await page.title() };", + ) as { url: string; title: string }; await this.gateCaptcha(); - const buffer = toBuffer(result.data); + const buffer = Buffer.from(image.data, "base64"); this.state.screenshot = buffer; this.updateBrowserState(result.url, result.title); const url = result.url; @@ -491,6 +454,8 @@ export class BrowserSession { const stagehand = this.stagehand; const browser = this.browser; this.stagehand = undefined; + this.facade = undefined; + this.xpathMap = {}; this.browser = undefined; await stagehand?.close().catch(() => undefined); await browser?.close().catch(() => undefined); @@ -548,7 +513,7 @@ export class BrowserSession { private resolveTarget(target: string): string { const raw = target.replace(/^\[/, "").replace(/\]$/, ""); const xpath = this.xpathMap[target] ?? this.xpathMap[raw]; - if (!xpath && /^\[?\d+\]?$/.test(target)) { + if (!xpath && /^\[?\d+(?:-\d+)*\]?$/.test(target)) { throw new Error("This snapshot ID is not in the latest snapshot. Call snapshot again."); } if (!xpath) return target; @@ -560,6 +525,30 @@ export class BrowserSession { return this.stagehand; } + private requireFacade(): StagehandFacadeTools { + return this.facade ??= new StagehandFacadeTools(this.requireStagehand(), { + keeperPage: false, + onCloseRequested: () => this.close(), + }); + } + + private async captureSnapshot(): Promise<{ + snapshot: { formattedTree: string; xpathMap: Record }; + url: string; + title: string; + }> { + // Bell accepts both CSS selectors and snapshot IDs in one action batch, so + // retain the map and page metadata from the same capture. + return await this.requireFacade().run(` +const snapshot = await page.accessibility.snapshot({ includeIframes: true }); +return { snapshot, url: await page.evaluate(() => location.href), title: await page.title() }; +`) as { + snapshot: { formattedTree: string; xpathMap: Record }; + url: string; + title: string; + }; + } + private async acquireOperation(): Promise<() => void> { let release!: () => void; const gate = new Promise((resolve) => { @@ -605,6 +594,8 @@ export class BrowserSession { await this.stagehand?.close().catch(() => undefined); await this.browser?.close().catch(() => undefined); this.stagehand = undefined; + this.facade = undefined; + this.xpathMap = {}; this.browser = undefined; if (attempt < 2) await new Promise((resolve) => setTimeout(resolve, 500)); } @@ -641,6 +632,8 @@ export class BrowserSession { await this.stagehand?.close().catch(() => undefined); await launched?.close().catch(() => undefined); this.stagehand = undefined; + this.facade = undefined; + this.xpathMap = {}; this.browser = undefined; if (launched?.sessionId) { this.remoteSessionId = launched.sessionId; @@ -671,6 +664,8 @@ export class BrowserSession { await this.clearCaptchaSubscriptions(); this.browser = browser; this.stagehand = stagehand; + this.facade = undefined; + this.xpathMap = {}; await this.ensureCaptchaSubscriptions(); } catch (error) { await browser.close().catch(() => undefined); @@ -771,11 +766,7 @@ export class BrowserSession { throw new CaptchaHandoffError(this.captchaStatus); if (this.captchaNeedsObservation) { this.captchaNeedsObservation = false; - const result = await this.requireStagehand().experimentalBatch(async ({ page }) => ({ - snapshot: await page.snapshot({ includeIframes: true }), - url: await page.url(), - title: await page.title(), - })); + const result = await this.captureSnapshot(); this.xpathMap = result.snapshot.xpathMap; this.updateBrowserState(result.url, result.title); addTrace( @@ -836,27 +827,40 @@ export class BrowserSession { } export function formatRunCode(actions: RunAction[]): string { + return actionCode(actions.map((item): RunAction => { + switch (item.action) { + case "goto": return { ...item, url: safeActivityUrl(item.url) }; + case "fill": + case "type": + case "select": return { ...item, value: "[value hidden]" }; + case "press": return { ...item, key: "[key hidden]" }; + default: return item; + } + })); +} + +function actionCode(actions: RunAction[]): string { return actions .flatMap((item) => { switch (item.action) { case "goto": return [ - `await page.goto(${JSON.stringify(safeActivityUrl(item.url))});`, - 'await page.waitForLoadState("domcontentloaded", 15_000);', + `await page.goto(${JSON.stringify(item.url)});`, + 'await page.waitForLoadState("domcontentloaded", { timeout: 15_000 });', ]; case "click": return `await page.locator(${JSON.stringify(item.target)}).click();`; case "fill": - return `await page.locator(${JSON.stringify(item.target)}).fill("[value hidden]");`; + return `await page.locator(${JSON.stringify(item.target)}).fill(${JSON.stringify(item.value)});`; case "type": - return `await page.locator(${JSON.stringify(item.target)}).type("[value hidden]");`; + return `await page.locator(${JSON.stringify(item.target)}).type(${JSON.stringify(item.value)});`; case "press": return [ ...(item.target ? [`await page.locator(${JSON.stringify(item.target)}).click();`] : []), - 'await page.keyPress("[key hidden]");', + `await page.keyboard.press(${JSON.stringify(item.key)});`, ]; case "select": - return `await page.locator(${JSON.stringify(item.target)}).selectOption("[value hidden]");`; + return `await page.locator(${JSON.stringify(item.target)}).selectOption(${JSON.stringify(item.value)});`; case "wait": return `await page.waitForTimeout(${Math.min(item.milliseconds, 10_000)});`; } @@ -971,20 +975,3 @@ function isLiveSourceUrl(url: string): boolean { if (host.includes("google.")) return /^\/maps\/place\//.test(parsed.pathname); return !DISCOVERY_HOSTS.some((item) => host.includes(item)); } - -function toBuffer(value: unknown): Buffer { - if (Buffer.isBuffer(value)) return value; - if (value instanceof Uint8Array) return Buffer.from(value); - if (Array.isArray(value) && value.every((item) => typeof item === "number")) { - return Buffer.from(value); - } - if (value && typeof value === "object") { - const record = value as Record; - if (Array.isArray(record.data)) return Buffer.from(record.data as number[]); - const numericValues = Object.values(record); - if (numericValues.length && numericValues.every((item) => typeof item === "number")) { - return Buffer.from(numericValues as number[]); - } - } - throw new Error("Stagehand returned a screenshot in an unsupported byte format."); -} diff --git a/tests/browser/browser-security.test.ts b/tests/browser/browser-security.test.ts index 6b25ebd..821ff59 100644 --- a/tests/browser/browser-security.test.ts +++ b/tests/browser/browser-security.test.ts @@ -1,7 +1,9 @@ +import { Stagehand, type StagehandBrowser } from "@browserbasehq/stagehand"; import { afterEach, describe, expect, it, vi } from "vitest"; import { BrowserSession } from "../../server/browser/browser-session"; import type { ConversationState } from "../../server/browser/types"; +import { stagehandFixture } from "./stagehand-fixture"; const login = { allowedHosts: ["shop.example"], @@ -26,30 +28,64 @@ function fixture() { let url = "https://shop.example/login"; const fill = vi.fn<(target: string, value: string) => Promise>().mockResolvedValue(); const click = vi.fn<(target: string) => Promise>().mockResolvedValue(); + const type = vi.fn<(target: string, value: string) => Promise>().mockResolvedValue(); + const select = vi.fn<(target: string, value: unknown) => Promise>().mockResolvedValue(); + const taggedTargets = new Map(); const page = { + pageId: "fixture-page", url: async () => url, title: async () => "Test shop", + // The middleware evaluates locator queries, then tags the matched element + // before dispatching native input. Preserve the original selector for assertions. + evaluate: vi.fn(async (expression: string | (() => unknown)) => { + if (typeof expression === "function") return url; + const match = expression.match(/return await execute\((.+)\);/); + if (!match) return { width: 1360, height: 900 }; + const query = JSON.parse(match[1]) as { + operation: string; + plan: Array<{ kind: string; value?: string }>; + token?: string; + }; + if (query.operation === "tag" && query.token) { + const target = query.plan.find(step => step.kind === "selector")?.value; + if (!target) throw new Error("The fixture requires a concrete selector."); + taggedTargets.set('[data-stagehand-pw-compat="' + query.token + '"]', target); + } + return { count: 1, visible: true }; + }), goto: vi.fn(async (next: string) => { url = next; }), waitForLoadState: vi.fn(async () => {}), waitForTimeout: vi.fn(async () => {}), + keyPress: vi.fn<(key: string) => Promise>().mockResolvedValue(), + snapshot: vi.fn(async () => ({ + formattedTree: "[0-42] textbox: Email\n[7] button: Submit", + xpathMap: { + "0-42": "/html/body/input", + "7": "xpath=/html/body/button", + "0-99": "/html/body/div", + }, + })), + screenshot: vi.fn(async () => Uint8Array.from([137, 80, 78, 71])), + on: vi.fn(async () => ({ unsubscribe: async () => {} })), locator: (target: string) => ({ - fill: (value: string) => fill(target, value), - click: () => click(target), + fill: (value: string) => fill(taggedTargets.get(target) ?? target, value), + click: () => click(taggedTargets.get(target) ?? target), + type: (value: string) => type(taggedTargets.get(target) ?? target, value), + selectOption: (value: unknown) => select(taggedTargets.get(target) ?? target, value), count: async () => 1, }), }; - const batch = vi.fn(async ( - callback: (context: { page: typeof page }, args: A) => Promise, - args: A, - ) => callback({ page }, args)); + const { stagehand, browser, batch, context } = stagehandFixture(page); const start = vi.spyOn(session, "start").mockResolvedValue(); vi.spyOn(session as unknown as { gateCaptcha(): Promise }, "gateCaptcha") .mockResolvedValue(); Object.assign(session, { - stagehand: { experimentalBatch: batch }, + stagehand, + browser, browserbase: {}, }); - return { session, state, start, batch, page, fill, click, setUrl: (next: string) => { url = next; } }; + return { session, state, start, batch, context, page, fill, click, type, select, + setUrl: (next: string) => { url = next; } }; } afterEach(() => { @@ -133,11 +169,10 @@ describe("browser navigation boundary", () => { }); it("rejects operations queued behind a browser that has been closed", async () => { - const { session, batch } = fixture(); - Object.assign(session, { stagehand: { experimentalBatch: batch, close: vi.fn(async () => {}) } }); + const { session, batch, fill } = fixture(); let resume!: () => void; const gate = new Promise(resolve => { resume = resolve; }); - batch.mockImplementationOnce(async () => { await gate; return { url: "https://shop.example", title: "Fixture" }; }); + fill.mockImplementationOnce(async () => { await gate; }); const first = session.run([{ action: "fill", target: "#first", value: "one" }]); await vi.waitFor(() => expect(batch).toHaveBeenCalledOnce()); const second = session.run([{ action: "fill", target: "#second", value: "two" }]); @@ -175,6 +210,141 @@ describe("browser navigation boundary", () => { const { session, start, page } = fixture(); await session.run([{ action: "goto", url: "http://127.0.0.1:4318/" }]); expect(start).toHaveBeenCalledWith(); - expect(page.goto).toHaveBeenCalledWith("http://127.0.0.1:4318/"); + expect(page.goto.mock.calls[0]?.[0]).toBe("http://127.0.0.1:4318/"); + }); +}); + + +describe("middleware-backed browser tools", () => { + it("preserves snapshot targets and runs mixed typed actions in one batch", async () => { + const { session, batch, context, page, fill, click, type, select } = fixture(); + const snapshot = await session.snapshot(); + expect(snapshot).toMatchObject({ + tree: "[0-42] textbox: Email\n[7] button: Submit", + url: "https://shop.example/login", + // Count the complete map, including nodes omitted from the displayed tree. + snapshotIdCount: 3, + }); + expect(page.snapshot).toHaveBeenCalledWith({ includeIframes: true }); + + const result = await session.run([ + { action: "fill", target: "[0-42]", value: "synthetic-email" }, + { action: "type", target: "#notes", value: "synthetic-notes" }, + { action: "press", target: "[7]", key: "Enter" }, + { action: "select", target: "#choice", value: "second" }, + { action: "wait", milliseconds: 5 }, + ]); + expect(result).toEqual({ completed: 5, url: "https://shop.example/login" }); + expect(fill).toHaveBeenCalledWith("xpath=/html/body/input", "synthetic-email"); + expect(type).toHaveBeenCalledWith("#notes", "synthetic-notes"); + expect(click).toHaveBeenCalledWith("xpath=/html/body/button"); + expect(page.keyPress.mock.calls[0]?.[0]).toBe("Enter"); + expect(select).toHaveBeenCalledWith("#choice", "second"); + expect(page.waitForTimeout).toHaveBeenCalledWith(5); + expect(batch).toHaveBeenCalledTimes(2); + expect(context.newPage).not.toHaveBeenCalled(); + + for (const target of ["[7]", "[0-42]"]) { + await expect(session.run([{ action: "click", target }])) + .rejects.toThrow("Call snapshot again"); + } + expect(batch).toHaveBeenCalledTimes(2); + }); + + it("keeps quotes, backslashes and newlines as literal action inputs", async () => { + const { session, fill, page, state } = fixture(); + const target = '#field"); throw new Error("injected selector"); //'; + const value = '"; throw new Error("injected value"); //\\\nsecond line'; + await expect(session.run([{ action: "fill", target, value }])) + .resolves.toMatchObject({ completed: 1 }); + expect(fill).toHaveBeenCalledExactlyOnceWith(target, value); + expect(page.goto).not.toHaveBeenCalled(); + expect(JSON.stringify(state.traces)).not.toContain("injected value"); + }); + + it("preserves navigation waits and untargeted keyboard input", async () => { + const { session, page, batch } = fixture(); + await session.run([ + { action: "goto", url: "https://shop.example/account" }, + { action: "press", key: "Tab" }, + ]); + expect(page.goto.mock.calls[0]?.[0]).toBe("https://shop.example/account"); + expect(page.waitForLoadState).toHaveBeenCalledWith("domcontentloaded", 15_000); + expect(page.keyPress.mock.calls[0]?.[0]).toBe("Tab"); + expect(batch).toHaveBeenCalledOnce(); + }); + + it("reports live metadata when keyboard input navigates the page", async () => { + const { session, page, state, setUrl } = fixture(); + const destination = "https://shop.example/search?q=fixture"; + page.keyPress.mockImplementationOnce(async () => { setUrl(destination); }); + + await expect(session.run([{ action: "press", key: "Enter" }])) + .resolves.toEqual({ completed: 1, url: destination }); + expect(state.browser.url).toBe(destination); + }); + + it("reports the current URL when navigation finishes during snapshot capture", async () => { + const { session, page, state, setUrl } = fixture(); + const capture = page.snapshot.getMockImplementation()!; + const destination = "https://shop.example/account"; + page.snapshot.mockImplementationOnce(async () => { + setUrl(destination); + return capture(); + }); + + await expect(session.snapshot()).resolves.toMatchObject({ url: destination }); + expect(state.browser.url).toBe(destination); + }); + + it("never replays an action batch after a partial failure", async () => { + const { session, fill, click, batch } = fixture(); + fill.mockRejectedValueOnce(new Error("Synthetic fill failure")); + await expect(session.run([ + { action: "click", target: "#first" }, + { action: "fill", target: "#second", value: "fixture" }, + { action: "click", target: "#third" }, + ])).rejects.toThrow("Synthetic fill failure"); + expect(click.mock.calls).toEqual([["#first"]]); + expect(batch).toHaveBeenCalledOnce(); + await expect(session.run([{ action: "click", target: "#next" }])) + .resolves.toMatchObject({ completed: 1 }); + expect(click.mock.calls).toEqual([["#first"], ["#next"]]); + }); + + it("rebinds middleware to the new Stagehand client after reattachment", async () => { + const current = fixture(); + const replacement = fixture(); + const native = stagehandFixture(replacement.page); + await current.session.snapshot(); + vi.spyOn(Stagehand, "create").mockResolvedValue(native.stagehand); + + await (current.session as unknown as { + attachStagehand(browser: StagehandBrowser): Promise; + }).attachStagehand(native.browser as unknown as StagehandBrowser); + + await current.session.snapshot(); + expect(current.page.snapshot).toHaveBeenCalledOnce(); + expect(replacement.page.snapshot).toHaveBeenCalledOnce(); + expect(native.batch).toHaveBeenCalledOnce(); + expect(native.context.newPage).not.toHaveBeenCalled(); + }); + + it("preserves PNG bytes for the viewer and model image output", async () => { + const { session, state, page, context } = fixture(); + const image = await session.screenshot(); + expect(image).toEqual({ + mediaType: "image/png", + data: Buffer.from([137, 80, 78, 71]).toString("base64"), + url: "https://shop.example/login", + }); + expect(state.screenshot).toEqual(Buffer.from(image.data, "base64")); + expect(page.screenshot).toHaveBeenCalledWith({ type: "png" }); + expect(state.browser).toMatchObject({ + url: image.url, + title: "Test shop", + status: "active", + }); + expect(context.newPage).not.toHaveBeenCalled(); }); }); diff --git a/tests/browser/captcha-recovery.test.ts b/tests/browser/captcha-recovery.test.ts index 5188695..3d9d87f 100644 --- a/tests/browser/captcha-recovery.test.ts +++ b/tests/browser/captcha-recovery.test.ts @@ -5,6 +5,7 @@ import { closeBrowserRuntime, runBrowserOperation } from "../../agent/lib/browse import { applyCaptchaTransition, closeTaskBrowser, taskState } from "../../agent/lib/task-state"; import { BrowserSession, CaptchaHandoffError } from "../../server/browser/browser-session"; import { ContextContainer, contextStorage } from "../../node_modules/eve/dist/src/context/container.js"; +import { stagehandFixture } from "./stagehand-fixture"; afterEach(() => vi.restoreAllMocks()); @@ -18,17 +19,22 @@ describe("resuming a browser after CAPTCHA handoff", () => { captcha: { status, attemptCount: 1, journal: [] }, })); const page = { + pageId: "captcha-page", snapshot: vi.fn(async () => ({ formattedTree: "Account home", xpathMap: {} })), url: async () => "https://shop.example/account", title: async () => "Account", + evaluate: async (expression: string | (() => unknown)) => + typeof expression === "function" + ? "https://shop.example/account" + : { width: 1360, height: 900 }, + on: vi.fn(async () => ({ unsubscribe: async () => {} })), }; + const native = stagehandFixture(page); const close = vi.fn(async () => {}); await runBrowserOperation(id, async browser => { vi.spyOn(browser, "start").mockResolvedValue(); vi.spyOn(browser, "close").mockImplementation(close); - Object.assign(browser, { stagehand: { - experimentalBatch: (operation: (context: { page: typeof page }) => Promise) => operation({ page }), - } }); + Object.assign(browser, { stagehand: native.stagehand, browser: native.browser }); }); try { // Before the reply, the restored challenge must still gate all browser work. diff --git a/tests/browser/middleware-real.test.ts b/tests/browser/middleware-real.test.ts new file mode 100644 index 0000000..1adfa98 --- /dev/null +++ b/tests/browser/middleware-real.test.ts @@ -0,0 +1,168 @@ +import { createServer, type Server } from "node:http"; + +import { chromium, type Browser } from "playwright-core"; +import { describe, expect, it } from "vitest"; + +import { BrowserSession, browserProvider } from "../../server/browser/browser-session.js"; +import type { ConversationState } from "../../server/browser/types.js"; + +const live = process.env.BELL_LIVE_MIDDLEWARE === "1"; +const literal = '"); await page.locator("#literal").fill("injected"); // "quoted" \\ ${literal}'; +const typed = 'typed "quotes" and \\ ${expression}'; +const option = 'option "quoted"; throw new Error("injected")'; +const fixtureHtml = ` +Bell middleware fixture +

Browser middleware regression

+ + + + +

Waiting for actions

+`; + +async function startLocalFixture(): Promise<{ server: Server; url: string }> { + const server = createServer((_request, response) => { + response.writeHead(200, { "content-type": "text/html" }); + response.end(fixtureHtml); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Fixture did not bind a TCP port."); + return { server, url: `http://127.0.0.1:${address.port}/middleware` }; +} + +describe("opt-in real middleware browser regression", () => { + it.runIf(live)( + "preserves all typed actions, snapshot IDs, images, and browser lifetime", + async () => { + const provider = browserProvider(); + if (provider === "browserbase" && !process.env.BROWSERBASE_API_KEY) + throw new Error("BROWSERBASE_API_KEY is required for the hosted middleware regression."); + const state: ConversationState = { + id: "middleware-live-fixture", + traces: [], + browser: { provider: "not-started", status: "idle" }, + }; + const session = new BrowserSession(state); + let fixtureBrowser: Browser | undefined; + let fixtureServer: Server | undefined; + let browser: { closed: boolean } | undefined; + try { + let fixtureUrl: string; + if (provider === "browserbase") { + fixtureUrl = "https://bell-fixture.invalid/middleware"; + fixtureBrowser = await session.withHostedPage( + new AbortController().signal, + (_page, connectUrl) => chromium.connectOverCDP(connectUrl), + ); + await fixtureBrowser.contexts()[0].route(fixtureUrl, route => route.fulfill({ + contentType: "text/html", body: fixtureHtml, + })); + } else { + const fixture = await startLocalFixture(); + fixtureServer = fixture.server; + fixtureUrl = fixture.url; + } + + expect(await session.run([{ action: "goto", url: fixtureUrl }])).toEqual({ + completed: 1, url: fixtureUrl, + }); + browser = (session as unknown as { browser: { closed: boolean } }).browser; + const sessionId = state.browser.sessionId; + const snapshot = await session.snapshot(); + expect(snapshot.url).toBe(fixtureUrl); + expect(snapshot.pageStatus).toBe("live"); + expect(snapshot.snapshotIdCount).toBeGreaterThanOrEqual(4); + expect(snapshot.recovery.visitedOrigins).toContain(new URL(fixtureUrl).origin); + expect(state.browser.title).toBe("Bell middleware fixture"); + const recordLine = snapshot.tree.split("\n").find(line => line.includes("Record result")); + const recordId = recordLine?.match(/\[([0-9-]+)\]/)?.[1]; + expect(recordId, "The fixture button should have a snapshot ID.").toBeTruthy(); + if (!recordId) throw new Error("The fixture button has no snapshot ID."); + + const result = await session.run([ + { action: "fill", target: "#literal", value: literal }, + { action: "type", target: "#typed", value: typed }, + { action: "select", target: "#choice", value: option }, + { action: "click", target: `[${recordId}]` }, + { action: "press", target: "#typed", key: "Enter" }, + { action: "press", key: "Enter" }, + { action: "wait", milliseconds: 20 }, + ]); + expect(result).toEqual({ completed: 7, url: fixtureUrl }); + await expect(session.run([{ action: "click", target: recordId }])).rejects.toThrow( + "This snapshot ID is not in the latest snapshot. Call snapshot again.", + ); + const after = await session.snapshot(); + for (const expected of [ + "Literal preserved: yes", "Typed preserved: yes", "Option preserved: yes", "Clicks: 1", "Keys: 2", + ]) expect(after.tree).toContain(expected); + expect(after.snapshotIdCount).toBeGreaterThanOrEqual(4); + expect(after.url).toBe(fixtureUrl); + + const navigatedUrl = `${fixtureUrl}#keyboard-navigation`; + expect(await session.run([{ action: "press", target: "#typed", key: "Escape" }])).toEqual({ + completed: 1, url: navigatedUrl, + }); + expect(state.browser.url).toBe(navigatedUrl); + expect((await session.snapshot()).url).toBe(navigatedUrl); + + const screenshot = await session.screenshot(); + const png = Buffer.from(screenshot.data, "base64"); + expect(screenshot.mediaType).toBe("image/png"); + expect(screenshot.url).toBe(navigatedUrl); + expect(png.subarray(0, 8).toString("hex")).toBe("89504e470d0a1a0a"); + expect(png.byteLength).toBeGreaterThan(100); + expect(state.screenshot?.equals(png)).toBe(true); + expect((session as unknown as { browser: unknown }).browser === browser).toBe(true); + expect(state.browser.sessionId === sessionId).toBe(true); + expect(state.traces.filter(event => event.name === "browser.start")).toHaveLength(1); + expect(session.hasStarted()).toBe(true); + } finally { + try { + await fixtureBrowser?.close(); + } finally { + try { + await session.close(); + } finally { + await new Promise((resolve, reject) => { + if (!fixtureServer) return resolve(); + fixtureServer.close(error => error ? reject(error) : resolve()); + }); + } + } + } + expect(browser?.closed).toBe(true); + expect(session.hasStarted()).toBe(false); + expect(state.browser.status).toBe("idle"); + await expect(session.start()).rejects.toMatchObject({ name: "AbortError" }); + }, + 120_000, + ); +}); diff --git a/tests/browser/stagehand-fixture.ts b/tests/browser/stagehand-fixture.ts new file mode 100644 index 0000000..4de7cf1 --- /dev/null +++ b/tests/browser/stagehand-fixture.ts @@ -0,0 +1,23 @@ +import type { Stagehand } from "@browserbasehq/stagehand"; +import { vi } from "vitest"; + +/** Execute real middleware callbacks against a native-page fixture. */ +export function stagehandFixture(page: Page) { + const context = { + activePage: vi.fn(async () => page), + pages: vi.fn(async () => [page]), + newPage: vi.fn(async () => page), + setActivePage: vi.fn(async () => {}), + }; + const browser = { context, close: vi.fn(async () => {}) }; + const batch = vi.fn(async ( + callback: (runtime: { page: Page; context: typeof context }, input: Input) => Promise, + input: Input, + ) => callback({ page, context }, input)); + const stagehand = { + browser, + experimentalBatch: batch, + close: vi.fn(async () => {}), + } as unknown as Stagehand; + return { stagehand, browser, batch, context }; +} diff --git a/tests/workspace-runtime.test.ts b/tests/workspace-runtime.test.ts index 687e650..23b639b 100644 --- a/tests/workspace-runtime.test.ts +++ b/tests/workspace-runtime.test.ts @@ -10,6 +10,7 @@ import lifecycle from "../agent/hooks/browser-lifecycle"; import run from "../agent/tools/run"; import snapshot from "../agent/tools/snapshot"; import type { ConversationState } from "../server/browser/types"; +import { stagehandFixture } from "./browser/stagehand-fixture"; import { ContextContainer, contextStorage, @@ -36,17 +37,31 @@ async function withEveContext(check: () => Promise) { describe("Bell's shared browser runtime", () => { it("redacts form values and preserves the browser identity through Bell's actual import", () => withEveContext(async () => { vi.stubEnv("BELL_HANDOFF_SECRET", "synthetic-shared-handoff-secret-at-least-32-bytes"); - const fill = vi.fn(async () => {}); - const page = { locator: () => ({ fill }), url: async () => "https://shop.example/", title: async () => "Fixture" }; + const fill = vi.fn<(value: string) => Promise>().mockResolvedValue(); + const page = { + pageId: "workspace-page", + locator: () => ({ fill: (value: string) => fill(value) }), + url: async () => "https://shop.example/", + title: async () => "Fixture", + evaluate: async (expression: string | (() => unknown)) => { + if (typeof expression === "function") return "https://shop.example/"; + return expression.includes("return await execute(") + ? { count: 1, visible: true } + : { width: 1360, height: 900 }; + }, + }; + const native = stagehandFixture(page); await runBrowserOperation(sessionId, async browser => { const internal = browser as unknown as { state: ConversationState; gateCaptcha(): Promise }; internal.state.browser = { provider: "browserbase", status: "active", sessionId: "synthetic-hosted-session" }; vi.spyOn(browser, "start").mockResolvedValue(); vi.spyOn(browser, "close").mockResolvedValue(); vi.spyOn(internal, "gateCaptcha").mockResolvedValue(); - Object.assign(browser, { browserbase: {}, stagehand: { - experimentalBatch: async (callback: (context: { page: typeof page }, args: A) => Promise, args: A) => callback({ page }, args), - } }); + Object.assign(browser, { + browserbase: {}, + stagehand: native.stagehand, + browser: native.browser, + }); }); const value = "synthetic-private-form-value"; const output = await run.execute({ actions: [{ action: "fill", target: "#field", value }] }, context()); From 8cbe0a01e876f698c4dd8d16ba2286b27b4436ef Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Tue, 27 Oct 2026 23:50:13 +0000 Subject: [PATCH 2/2] Remove browser integration documentation additions --- docs/browser-runtime.md | 13 +------------ docs/provider-tests.md | 18 ------------------ docs/repository.md | 5 ++--- 3 files changed, 3 insertions(+), 33 deletions(-) diff --git a/docs/browser-runtime.md b/docs/browser-runtime.md index e04e612..f0c57bf 100644 --- a/docs/browser-runtime.md +++ b/docs/browser-runtime.md @@ -1,8 +1,7 @@ # Browser runtime Bell's Eve by Vercel agent owns conversation history and tool execution. Browser tools share one in-process -Stagehand runtime per Eve session, with browser execution provided by the published -[`stagehand-middleware`](https://www.npmjs.com/package/stagehand-middleware) package. Follow-up turns reuse the browser; saved Browserbase session +Stagehand runtime per Eve session. Follow-up turns reuse the browser; saved Browserbase session metadata supports reconnecting after a worker restart. Hosted launches request managed proxies, Verified Browsers, and keep-alive on every attempt, including retries. @@ -11,16 +10,6 @@ Verified Browsers, and keep-alive on every attempt, including retries. embedded credentials. A destination URL cannot change the browser provider. Local fixtures require `STAGEHAND_BROWSER=local`. -Bell keeps its typed `run({ actions })` interface: callers can mix CSS selectors and IDs from -`snapshot` in one batch. The runtime translates these actions into one middleware `run` call, -with entered values encoded as string literals. Snapshot capture retains the tree, target map, -URL, and title together; screenshots use the middleware's PNG output. Model-authored JavaScript -is not accepted by Bell's `run` tool. - -Bell owns the browser's launch, reconnect, CAPTCHA gates, privacy policy, and release lifecycle. -The middleware wraps that existing Stagehand client and does not launch a second browser or -keeper tab. Protected login and hosted integrations continue under Bell's operation lock. - ## Configuration Copy the root [`.env.example`](../.env.example) for Bell and integration settings. diff --git a/docs/provider-tests.md b/docs/provider-tests.md index a13583c..da82281 100644 --- a/docs/provider-tests.md +++ b/docs/provider-tests.md @@ -4,24 +4,6 @@ Run `pnpm check` for the normal credential-free checks. Live provider tests are opt-in and use real service resources. Keep credentials in ignored environment files. Browserbase must use `verified: true` and managed proxies, including after launch retries. -## Browser middleware - -The middleware regression uses a synthetic form to exercise navigation, CSS selectors, -snapshot IDs, quoted input, keyboard actions, screenshots, browser reuse, and cleanup. -Run it on local Chrome: - -```sh -BELL_LIVE_MIDDLEWARE=1 STAGEHAND_BROWSER=local pnpm exec vitest run tests/browser/middleware-real.test.ts -``` - -With `BROWSERBASE_API_KEY` available in the process environment, run the same check hosted: - -```sh -BELL_LIVE_MIDDLEWARE=1 STAGEHAND_BROWSER=browserbase pnpm exec vitest run tests/browser/middleware-real.test.ts -``` - -The hosted fixture is served over CDP and the test releases its browser when finished. - ## 1Password login and OTP Use a dedicated test vault with a Login item titled exactly diff --git a/docs/repository.md b/docs/repository.md index 437aa13..f77d120 100644 --- a/docs/repository.md +++ b/docs/repository.md @@ -60,9 +60,8 @@ See [browser configuration and credential handling](browser-runtime.md). ## License and ownership -Bell maintains its browser lifecycle and tool adapters in this repository and uses the published -`stagehand-middleware` package for browser execution. There is no submodule or separate application -to install. The root [MIT license](../LICENSE) preserves +Bell maintains its browser implementation directly in this repository. There is no submodule +or separately maintained application to install. The root [MIT license](../LICENSE) preserves the Browserbase copyright and license terms covering the browser source. ## Browser retirement