diff --git a/apps/desktop/e2e/browser-page-parity.spec.ts b/apps/desktop/e2e/browser-page-parity.spec.ts new file mode 100644 index 00000000000..63063e18abe --- /dev/null +++ b/apps/desktop/e2e/browser-page-parity.spec.ts @@ -0,0 +1,445 @@ +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs' +import { createServer, type ServerResponse } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { + type ElectronApplication, + _electron as electron, + expect, + type Page, + test, +} from '@playwright/test' +import type { BrowserPageDialog, BrowserToolName } from '@sim/browser-protocol' +import type { SimDesktopApi } from '@sim/desktop-bridge' +import { getErrorMessage } from '@sim/utils/errors' +import { recordCheck } from './check-report' + +/** + * The embedded browser against what a user expects of Chrome: the page menu, + * history and tab keys, Escape on a slow load, and a video's fullscreen + * button. Each test drives the real shell and native tab views with trusted + * input, and lands in a JSON report at BROWSER_PAGE_PARITY_REPORT_PATH + * (test-results/browser-page-parity-report.json by default) as it finishes. + */ + +const DESKTOP_DIR = fileURLToPath(new URL('..', import.meta.url)) +const SCOPE = 'browser-page-parity-e2e' +const IS_MAC = process.platform === 'darwin' +const SHELL_FIXTURE = 'Sim fixture

Browser parity fixture

' +/** A 2x2 PNG, so a saved copy is checked byte for byte. */ +const PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFklEQVR4nGP8z8DwnwEIGBkZGRgYAAAhBAEB/0bFxgAAAABJRU5ErkJggg==', + 'base64' +) +const PAGES: Record = { + '/form': `form +`, + '/image': + 'image', + '/links': 'linksLanding', + '/fullscreen': `fullscreen + +`, +} + +type Bridge = typeof globalThis & { + simDesktop: SimDesktopApi + pageDialog?: BrowserPageDialog | null + boundsTimer?: number +} +/** What the suite installs in the main process. */ +type MainFixture = typeof globalThis & { + agentMenus?: (Electron.MenuItemConstructorOptions | Electron.MenuItem)[][] + visibleView?: () => Electron.WebContentsView | undefined +} + +const reportPath = () => + process.env.BROWSER_PAGE_PARITY_REPORT_PATH ?? + join(test.info().project.outputDir, 'browser-page-parity-report.json') + +/** Runs one check and adds its outcome to the report as soon as it is known. */ +async function check(name: string, body: () => Promise): Promise { + const startedAt = Date.now() + const record = (status: 'passed' | 'failed', error?: unknown) => + recordCheck(reportPath(), 'browser-page-parity', { + name, + status, + durationMs: Date.now() - startedAt, + retry: test.info().retry, + ...(error === undefined ? {} : { error: getErrorMessage(error) }), + }) + try { + await body() + record('passed') + } catch (error) { + record('failed', error) + throw error + } +} + +/** A test whose outcome is also one check in the suite's report. */ +function checkedTest(name: string, body: () => Promise): void { + test(name, () => check(name, body)) +} + +test.describe('browser page parity with Chrome', () => { + let app: ElectronApplication | undefined + let shell: Page + let site = '' + let userData = '' + let downloads = '' + /** Held responses for the slow page, released when the suite ends. */ + const held = new Set() + /** The Referer each load of the link target arrived with. */ + const referrers: (string | undefined)[] = [] + const calls = new Map() + const server = createServer(async (request, response) => { + const path = new URL(request.url ?? '/', 'http://localhost').pathname + if (path === '/api/desktop/tool/authorize') { + let body = '' + for await (const chunk of request) body += chunk.toString() + const authorization = calls.get(JSON.parse(body).toolCallId) + response.writeHead(authorization ? 200 : 403, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify(authorization ?? {})) + return + } + if (path === '/pixel.png') { + response.writeHead(200, { 'Content-Type': 'image/png' }) + response.end(PNG) + return + } + if (path === '/landing') referrers.push(request.headers.referer) + if (path === '/slow') { + held.add(response) + return + } + // The app origin serves the shell; the same server on localhost is the web. + const isSite = request.headers.host?.startsWith('localhost') === true + response.writeHead(200, { + 'Content-Type': 'text/html', + ...(isSite ? {} : { 'Set-Cookie': 'better-auth.session_token=fixture; HttpOnly; Path=/' }), + }) + response.end( + isSite + ? (PAGES[path] ?? `${path.slice(1)}

${path}

`) + : SHELL_FIXTURE + ) + }) + + test.beforeAll(async () => { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('Missing fixture address') + const origin = `http://127.0.0.1:${address.port}` + site = origin.replace('127.0.0.1', 'localhost') + userData = mkdtempSync(join(tmpdir(), 'sim-browser-parity-e2e-')) + downloads = mkdtempSync(join(tmpdir(), 'sim-browser-parity-downloads-')) + app = await electron.launch({ + args: [process.env.SIM_DESKTOP_E2E_MAIN ?? '.'], + cwd: DESKTOP_DIR, + env: { ...process.env, SIM_DESKTOP_ORIGIN: origin, SIM_DESKTOP_USER_DATA: userData }, + }) + // A dialog listener stops Playwright auto-dismissing a page's leave-site + // question, so the desktop's own handling decides it as in production. + const leaveDialogsToDesktop = (page: Page) => page.on('dialog', () => {}) + app.context().pages().forEach(leaveDialogsToDesktop) + app.context().on('page', leaveDialogsToDesktop) + shell = await app.firstWindow() + await app.evaluate(({ app, BrowserWindow, Menu, WebContentsView }, downloads) => { + // Downloads default to the OS folder; keep the suite's files out of the user's. + app.setPath('downloads', downloads) + const host = BrowserWindow.getAllWindows()[0] + host.webContents.setBackgroundThrottling(false) + app.focus({ steal: true }) + host.focus() + // A native menu blocks the main process while it is open; keep each page + // menu's items so a check can choose one the way a click would. + const capture = globalThis as MainFixture + capture.visibleView = () => + BrowserWindow.getAllWindows()[0].contentView.children.find( + (child): child is Electron.WebContentsView => + child instanceof WebContentsView && child.getVisible() + ) + capture.agentMenus = [] + const build = Menu.buildFromTemplate.bind(Menu) + Menu.buildFromTemplate = (template) => { + capture.agentMenus?.push(template) + const menu = build(template) + menu.popup = () => {} + return menu + } + }, downloads) + await expect(shell.getByRole('heading')).toHaveText('Browser parity fixture') + await shell.evaluate(async (scope) => { + const bridge = globalThis as Bridge + const api = bridge.simDesktop.browserAgent + await api.activateScope(scope) + api.onPageState((state) => { + bridge.pageDialog = state.dialog ?? null + }) + const updateBounds = () => + api.setPanelBounds( + { x: 0, y: 80, width: innerWidth, height: innerHeight - 80 }, + null, + scope + ) + updateBounds() + bridge.boundsTimer = window.setInterval(updateBounds, 200) + }, SCOPE) + await execute('browser_open_url', { url: `${site}/start` }) + await panelAction({ action: 'switch-tab', tabId: '1' }) + await panelAction({ action: 'enable-page-dialogs' }) + }) + + test.afterAll(async () => { + for (const response of held) response.destroy() + await app?.close() + server.closeAllConnections() + await new Promise((resolve) => server.close(() => resolve())) + rmSync(userData, { recursive: true, force: true }) + rmSync(downloads, { recursive: true, force: true }) + }) + + let callCount = 0 + async function execute(tool: BrowserToolName, args: Record) { + const callId = `browser-parity-${++callCount}` + calls.set(callId, { chatId: SCOPE, toolName: tool, args }) + const result = await shell.evaluate( + ({ callId, tool, args, scope }) => + (globalThis as Bridge).simDesktop.browserAgent.executeTool(callId, tool, args, scope), + { callId, tool, args, scope: SCOPE } + ) + expect(result.ok, result.error).toBe(true) + } + + /** Browser-chrome actions are user gestures, so each follows a real click in Sim. */ + async function panelAction(action: Record) { + await shell.getByRole('heading').click() + await shell.evaluate( + ({ action, scope }) => + (globalThis as Bridge).simDesktop.browserAgent.panelAction( + action as unknown as Parameters[0], + scope + ), + { action, scope: SCOPE } + ) + } + + /** Reads one property of the page on screen from the main process. */ + function visiblePage(read: string): Promise { + if (!app) throw new Error('App not launched') + return app.evaluate(({ BrowserWindow }, read) => { + const view = (globalThis as MainFixture).visibleView?.() + if (!view) return null + const contents = view.webContents + const values: Record = { + url: contents.getURL(), + title: contents.getTitle(), + loading: contents.isLoadingMainFrame(), + zoom: contents.getZoomFactor(), + bounds: view.getBounds(), + windowFullScreen: BrowserWindow.getAllWindows()[0].isFullScreen(), + } + return values[read] + }, read) as Promise + } + + const pageUrl = () => visiblePage('url') + const pageTitle = () => visiblePage('title') + const pageDialog = () => shell.evaluate(() => (globalThis as Bridge).pageDialog ?? null) + + /** Trusted input into the page on screen, the way the user's own mouse and keys arrive. */ + async function inPage( + action: + | { kind: 'click' | 'right-click'; selector: string; offsetX?: number } + | { kind: 'type'; text: string } + | { kind: 'key'; keyCode: string; modifiers?: string[] } + | { kind: 'script'; script: string } + ): Promise { + if (!app) throw new Error('App not launched') + return app.evaluate(async (_electron, action) => { + const view = (globalThis as MainFixture).visibleView?.() + if (!view) throw new Error('No visible page') + const contents = view.webContents + if (action.kind === 'script') return contents.executeJavaScript(action.script) + if (action.kind === 'type') { + for (const character of action.text) { + contents.sendInputEvent({ type: 'char', keyCode: character }) + } + return null + } + if (action.kind === 'key') { + const input = { + keyCode: action.keyCode, + modifiers: action.modifiers ?? [], + } as Electron.KeyboardInputEvent + contents.focus() + contents.sendInputEvent({ ...input, type: 'keyDown' }) + contents.sendInputEvent({ ...input, type: 'keyUp' }) + return null + } + const rect: { x: number; y: number; height: number } = await contents.executeJavaScript( + `(() => { const { x, y, height } = document.querySelector(${JSON.stringify(action.selector)}).getBoundingClientRect(); return { x, y, height } })()` + ) + const point = { + x: Math.round(rect.x + (action.offsetX ?? 10)), + y: Math.round(rect.y + Math.min(rect.height / 2, 20)), + } + const button = action.kind === 'click' ? 'left' : 'right' + contents.sendInputEvent({ type: 'mouseDown', ...point, button, clickCount: 1 }) + contents.sendInputEvent({ type: 'mouseUp', ...point, button, clickCount: 1 }) + return null + }, action) + } + + /** Right-clicks `selector` and returns the page menu's item labels. */ + async function openPageMenu(selector: string, offsetX?: number): Promise { + if (!app) throw new Error('App not launched') + const menus = () => + app?.evaluate(() => (globalThis as MainFixture).agentMenus?.length ?? 0) ?? 0 + const before = await menus() + await inPage({ kind: 'right-click', selector, offsetX }) + await expect.poll(menus).toBeGreaterThan(before) + return app.evaluate(() => + ((globalThis as MainFixture).agentMenus?.at(-1) ?? []) + .filter((item) => item.type !== 'separator') + .map((item) => item.label ?? '') + ) + } + + /** Chooses an item from the page menu opened last. */ + async function choosePageMenuItem(label: string): Promise { + if (!app) throw new Error('App not launched') + await app.evaluate((_electron, label) => { + const item = (globalThis as MainFixture).agentMenus + ?.at(-1) + ?.find((entry) => entry.label === label) + if (!item?.click) throw new Error(`The page menu has no "${label}" item`) + ;(item.click as () => void)() + }, label) + } + + async function navigate(path: string): Promise { + await panelAction({ action: 'navigate', url: `${site}${path}` }) + await expect.poll(pageUrl).toBe(`${site}${path}`) + await expect.poll(() => visiblePage('loading')).toBe(false) + } + + checkedTest("the page menu's Back asks before leaving unsaved changes", async () => { + await navigate('/next') + await navigate('/form') + await inPage({ kind: 'click', selector: '#draft' }) + await inPage({ kind: 'type', text: 'draft' }) + await expect.poll(() => inPage({ kind: 'script', script: 'draft.value' })).toBe('draft') + + await openPageMenu('#draft', 120) + await choosePageMenuItem('Back') + await expect.poll(pageDialog).toMatchObject({ kind: 'beforeunload' }) + const stay = await pageDialog() + await panelAction({ action: 'respond-dialog', requestId: stay?.requestId, allowed: false }) + await expect.poll(pageDialog).toBeNull() + expect(await pageUrl()).toBe(`${site}/form`) + expect(await inPage({ kind: 'script', script: 'draft.value' })).toBe('draft') + + await openPageMenu('#draft', 120) + await choosePageMenuItem('Back') + await expect.poll(pageDialog).toMatchObject({ kind: 'beforeunload' }) + const leave = await pageDialog() + await panelAction({ action: 'respond-dialog', requestId: leave?.requestId, allowed: true }) + await expect.poll(pageUrl).toBe(`${site}/next`) + }) + + checkedTest("the page menu's Save Image lands through the browser's downloads", async () => { + for (const file of readdirSync(downloads)) rmSync(join(downloads, file), { force: true }) + await navigate('/image') + await expect + .poll(() => inPage({ kind: 'script', script: 'document.getElementById("pic").complete' })) + .toBe(true) + const labels = await openPageMenu('#pic') + expect(labels).toEqual( + expect.arrayContaining(['Open Image in New Tab', 'Copy Image', 'Copy Image Address']) + ) + await choosePageMenuItem('Save Image') + await expect.poll(() => readdirSync(downloads)).toEqual(['pixel.png']) + expect(readFileSync(join(downloads, 'pixel.png')).equals(PNG)).toBe(true) + }) + + checkedTest("the page menu's Open Link in New Tab sends the page as referrer", async () => { + referrers.length = 0 + await navigate('/links') + await openPageMenu('#link') + await choosePageMenuItem('Open Link in New Tab') + await expect.poll(() => referrers).toEqual([`${site}/links`]) + }) + + checkedTest('tab keys, and history keys off macOS, typed in the page follow Chrome', async () => { + await navigate('/one') + await execute('browser_open_tab', { url: `${site}/two` }) + await execute('browser_open_tab', { url: `${site}/three` }) + const tabs = await shell.evaluate( + async (scope) => + (await (globalThis as Bridge).simDesktop.browserAgent.activateScope(scope)).tabs, + SCOPE + ) + // Earlier checks may have left tabs open; the keys move to the neighbours in strip order. + const urls = tabs.map((tab) => tab.url) + const index = urls.indexOf(`${site}/one`) + const next = urls[(index + 1) % urls.length] + const previous = urls[(index - 1 + urls.length) % urls.length] + expect(urls.length).toBeGreaterThan(2) + await panelAction({ action: 'switch-tab', tabId: tabs[index]?.tabId }) + await expect.poll(pageUrl).toBe(`${site}/one`) + + if (IS_MAC) { + await inPage({ kind: 'key', keyCode: 'Right', modifiers: ['meta', 'alt'] }) + await expect.poll(pageUrl).toBe(next) + await inPage({ kind: 'key', keyCode: '[', modifiers: ['meta', 'shift'] }) + await expect.poll(pageUrl).toBe(`${site}/one`) + await inPage({ kind: 'key', keyCode: '[', modifiers: ['meta', 'shift'] }) + await expect.poll(pageUrl).toBe(previous) + await inPage({ kind: 'key', keyCode: ']', modifiers: ['meta', 'shift'] }) + await expect.poll(pageUrl).toBe(`${site}/one`) + await inPage({ kind: 'key', keyCode: 'Left', modifiers: ['meta', 'alt'] }) + await expect.poll(pageUrl).toBe(previous) + } else { + await inPage({ kind: 'key', keyCode: 'PageDown', modifiers: ['control'] }) + await expect.poll(pageUrl).toBe(next) + await inPage({ kind: 'key', keyCode: 'PageUp', modifiers: ['control'] }) + await expect.poll(pageUrl).toBe(`${site}/one`) + await navigate('/one-later') + await inPage({ kind: 'key', keyCode: 'Left', modifiers: ['alt'] }) + await expect.poll(pageUrl).toBe(`${site}/one`) + await inPage({ kind: 'key', keyCode: 'Right', modifiers: ['alt'] }) + await expect.poll(pageUrl).toBe(`${site}/one-later`) + } + }) + + checkedTest('Escape stops a page that is still loading', async () => { + await navigate('/before-slow') + await panelAction({ action: 'navigate', url: `${site}/slow` }) + await expect.poll(() => visiblePage('loading')).toBe(true) + await inPage({ kind: 'key', keyCode: 'Escape' }) + await expect.poll(() => visiblePage('loading'), { timeout: 3_000 }).toBe(false) + }) + + checkedTest("a page's fullscreen fills the browser panel and Escape leaves it", async () => { + await navigate('/fullscreen') + const bounds = await visiblePage('bounds') + await inPage({ kind: 'click', selector: '#go' }) + await expect.poll(pageTitle).toBe('entered') + expect(await visiblePage('windowFullScreen')).toBe(false) + expect(await visiblePage('bounds')).toEqual(bounds) + await inPage({ kind: 'key', keyCode: 'Escape' }) + await expect.poll(pageTitle).toBe('exited') + expect(await visiblePage('windowFullScreen')).toBe(false) + }) +}) diff --git a/apps/desktop/src/main/browser-agent/context-menu.test.ts b/apps/desktop/src/main/browser-agent/context-menu.test.ts index 44058879f89..113f1738f05 100644 --- a/apps/desktop/src/main/browser-agent/context-menu.test.ts +++ b/apps/desktop/src/main/browser-agent/context-menu.test.ts @@ -29,7 +29,18 @@ type Page = Parameters[1] type Handlers = Parameters[2] function params(overrides: Partial = {}): Params { - return { selectionText: '', linkURL: '', isEditable: false, editFlags: EDIT_FLAGS, ...overrides } + return { + selectionText: '', + linkURL: '', + isEditable: false, + editFlags: EDIT_FLAGS, + misspelledWord: '', + dictionarySuggestions: [], + mediaType: 'none', + srcURL: '', + hasImageContents: false, + ...overrides, + } } function page(overrides: Partial = {}): Page { @@ -46,13 +57,20 @@ function page(overrides: Partial = {}): Page { function handlers(): Handlers { return { addToChat: vi.fn(), + replaceMisspelling: vi.fn(), + addToDictionary: vi.fn(), + undo: vi.fn(), + redo: vi.fn(), + cut: vi.fn(), copy: vi.fn(), paste: vi.fn(), - back: vi.fn(), - forward: vi.fn(), - reload: vi.fn(), + pasteAndMatchStyle: vi.fn(), + selectAll: vi.fn(), + navigate: vi.fn(), openTab: vi.fn(), copyLink: vi.fn(), + copyImage: vi.fn(), + saveImage: vi.fn(), setZoomFactor: vi.fn(), } } @@ -82,11 +100,16 @@ describe('buildAgentContextMenuTemplate', () => { }) describe('attachAgentContextMenu', () => { + const navigationHost = { + history: () => ({ canGoBack: false, canGoForward: false }), + navigate: vi.fn(), + } type ContextMenuListener = (event: unknown, params: Params) => void it('suppresses one agent context menu and immediately allows the next menu', async () => { const contents = new WebContentsView().webContents attachAgentContextMenu(contents, { + ...navigationHost, addToChat: vi.fn(), openTab: vi.fn(), defaultZoomFactor: () => BASE_ZOOM_FACTOR, @@ -110,6 +133,7 @@ describe('attachAgentContextMenu', () => { async (inputEvent) => { const contents = new WebContentsView().webContents attachAgentContextMenu(contents, { + ...navigationHost, addToChat: vi.fn(), openTab: vi.fn(), defaultZoomFactor: () => BASE_ZOOM_FACTOR, diff --git a/apps/desktop/src/main/browser-agent/context-menu.ts b/apps/desktop/src/main/browser-agent/context-menu.ts index de9688e6592..d4da63e1b40 100644 --- a/apps/desktop/src/main/browser-agent/context-menu.ts +++ b/apps/desktop/src/main/browser-agent/context-menu.ts @@ -10,18 +10,20 @@ * while the page keeps rendering, so none of that applies. * * It is also what a browser's page menu is everywhere else, and unlike the - * terminal's hidden textarea, the roles here act on a real page: `copy` and + * terminal's hidden textarea, the edit items act on a real page: `copy` and * `paste` go to the frame that was clicked. */ import { resolveDesktopZoom } from '@sim/desktop-bridge' -import type { ContextMenuParams, MenuItemConstructorOptions, WebContents } from 'electron' +import type { ContextMenuParams, MenuItemConstructorOptions, Referrer, WebContents } from 'electron' import { clipboard, Menu } from 'electron' import { clearAgentContextMenu, consumeAgentContextMenu, isDispatchingAgentInput, } from '@/main/browser-agent/cdp' +import { spellingItems } from '@/main/context-menu' +import { parseHttpUrl } from '@/main/navigation' /** * Page-zoom ladder for the embedded browser, in Chromium's absolute zoom @@ -78,7 +80,15 @@ export function steppedZoomFactor(current: number, direction: 1 | -1): number { /** The parts of a right-click the menu acts on. */ type AgentContextMenuParams = Pick< ContextMenuParams, - 'selectionText' | 'linkURL' | 'isEditable' | 'editFlags' + | 'selectionText' + | 'linkURL' + | 'isEditable' + | 'editFlags' + | 'misspelledWord' + | 'dictionarySuggestions' + | 'mediaType' + | 'srcURL' + | 'hasImageContents' > /** What the page can currently do, read at the moment of the click. */ @@ -89,34 +99,89 @@ interface AgentPageContext { defaultZoomFactor: number } +/** A history move or reload started from browser chrome rather than by the page. */ +export type PageNavigation = 'back' | 'forward' | 'reload' + +/** + * Edit actions are explicit calls on this page rather than menu roles: a role + * acts on whichever contents holds focus, and a right-click need not focus the page. + */ interface AgentContextMenuHandlers { addToChat(text: string): void + replaceMisspelling(word: string): void + addToDictionary(word: string): void + undo(): void + redo(): void + cut(): void copy(): void paste(): void - back(): void - forward(): void - reload(): void + pasteAndMatchStyle(): void + selectAll(): void + navigate(kind: PageNavigation): void openTab(url: string): void copyLink(url: string): void + copyImage(): void + saveImage(url: string): void setZoomFactor(factor: number): void } export interface AgentContextMenuHost { /** Attaches selected page text to the chat that owns this browser tab. */ addToChat(text: string): void - /** Opens a link from the page in another tab of the same browser. */ - openTab(url: string): void + /** Opens a URL from the page in another tab of the same browser. */ + openTab(url: string, referrer: Referrer): void /** Returns the device's current default page zoom factor. */ defaultZoomFactor(): number + history(): Pick + /** Runs through the user-navigation path, so a page's leave-site question reaches the user. */ + navigate(kind: PageNavigation): void +} + +const isWebUrl = (url: string) => parseHttpUrl(url) !== null + +const REFERRER_POLICIES: ReadonlySet = new Set([ + 'default', + 'unsafe-url', + 'no-referrer-when-downgrade', + 'no-referrer', + 'origin', + 'strict-origin-when-cross-origin', + 'same-origin', + 'strict-origin', +]) + +function isReferrerPolicy(value: unknown): value is Referrer['policy'] { + return REFERRER_POLICIES.has(value) +} + +/** Electron types `referrerPolicy` as a Referrer, but it arrives as the bare policy string. */ +function menuReferrer( + params: Pick +): Referrer { + const policy: unknown = params.referrerPolicy + const url = params.frameURL || params.pageURL + return { + url: isWebUrl(url) ? url : '', + policy: isReferrerPolicy(policy) ? policy : 'default', + } +} + +/** Ends the template with one separator, never two in a row or one at the top. */ +function endGroup(template: MenuItemConstructorOptions[]): void { + if (template.length > 0 && template[template.length - 1].type !== 'separator') { + template.push({ type: 'separator' }) + } } /** * Builds the page menu. Unlike the main window's menu this is never empty — - * navigation and zoom always apply, and only the clipboard and link items - * depend on what was clicked. + * navigation and zoom always apply, and only the spelling, clipboard, link, and + * image items depend on what was clicked. * - * Only http(s) links get link items: the actions open a browser tab or copy an - * address, and neither means anything for a `javascript:` or `mailto:` target. + * Only http(s) links and image sources get address items: the actions open a + * browser tab, download, or copy an address, and none of them means anything + * for a `javascript:`, `data:`, or `blob:` target. Copy Image needs no address, + * so it is offered for any loaded image. */ export function buildAgentContextMenuTemplate( params: AgentContextMenuParams, @@ -124,8 +189,13 @@ export function buildAgentContextMenuTemplate( handlers: AgentContextMenuHandlers ): MenuItemConstructorOptions[] { const template: MenuItemConstructorOptions[] = [] - const linkUrl = /^https?:\/\//i.test(params.linkURL) ? params.linkURL : '' + const linkUrl = isWebUrl(params.linkURL) ? params.linkURL : '' + const imageUrl = isWebUrl(params.srcURL) ? params.srcURL : '' const selectionText = params.selectionText + const flags = params.editFlags + + template.push(...spellingItems(params, handlers)) + endGroup(template) if (selectionText.trim()) { template.push( @@ -142,20 +212,47 @@ export function buildAgentContextMenuTemplate( ) } - if (selectionText.trim()) { - template.push({ label: 'Copy', click: () => handlers.copy() }) - } - if (params.isEditable && params.editFlags.canPaste) { - template.push({ label: 'Paste', click: () => handlers.paste() }) + if (params.mediaType === 'image') { + const copyImage: MenuItemConstructorOptions[] = params.hasImageContents + ? [{ label: 'Copy Image', click: () => handlers.copyImage() }] + : [] + template.push( + ...(imageUrl + ? [ + { label: 'Open Image in New Tab', click: () => handlers.openTab(imageUrl) }, + { label: 'Save Image', click: () => handlers.saveImage(imageUrl) }, + ...copyImage, + { label: 'Copy Image Address', click: () => handlers.copyLink(imageUrl) }, + ] + : copyImage) + ) + endGroup(template) } - if (template.length > 0 && template[template.length - 1].type !== 'separator') { - template.push({ type: 'separator' }) + + if (params.isEditable) { + template.push( + { label: 'Undo', enabled: flags.canUndo, click: () => handlers.undo() }, + { label: 'Redo', enabled: flags.canRedo, click: () => handlers.redo() }, + { type: 'separator' }, + { label: 'Cut', enabled: flags.canCut, click: () => handlers.cut() }, + { label: 'Copy', enabled: flags.canCopy, click: () => handlers.copy() }, + { label: 'Paste', enabled: flags.canPaste, click: () => handlers.paste() }, + { + label: 'Paste and Match Style', + enabled: flags.canPaste, + click: () => handlers.pasteAndMatchStyle(), + }, + { label: 'Select All', enabled: flags.canSelectAll, click: () => handlers.selectAll() } + ) + } else if (selectionText.trim()) { + template.push({ label: 'Copy', click: () => handlers.copy() }) } + endGroup(template) template.push( - { label: 'Back', enabled: page.canGoBack, click: () => handlers.back() }, - { label: 'Forward', enabled: page.canGoForward, click: () => handlers.forward() }, - { label: 'Reload', click: () => handlers.reload() }, + { label: 'Back', enabled: page.canGoBack, click: () => handlers.navigate('back') }, + { label: 'Forward', enabled: page.canGoForward, click: () => handlers.navigate('forward') }, + { label: 'Reload', click: () => handlers.navigate('reload') }, { type: 'separator' } ) @@ -198,20 +295,27 @@ export function attachAgentContextMenu(contents: WebContents, host: AgentContext const template = buildAgentContextMenuTemplate( params, { - canGoBack: contents.navigationHistory.canGoBack(), - canGoForward: contents.navigationHistory.canGoForward(), + ...host.history(), zoomFactor: contents.getZoomFactor(), defaultZoomFactor: host.defaultZoomFactor(), }, { addToChat: (text) => host.addToChat(text), + replaceMisspelling: (word) => contents.replaceMisspelling(word), + addToDictionary: (word) => contents.session.addWordToSpellCheckerDictionary(word), + undo: () => contents.undo(), + redo: () => contents.redo(), + cut: () => contents.cut(), copy: () => contents.copy(), paste: () => contents.paste(), - back: () => contents.navigationHistory.goBack(), - forward: () => contents.navigationHistory.goForward(), - reload: () => contents.reload(), - openTab: (url) => host.openTab(url), + pasteAndMatchStyle: () => contents.pasteAndMatchStyle(), + selectAll: () => contents.selectAll(), + navigate: (kind) => host.navigate(kind), + openTab: (url) => host.openTab(url, menuReferrer(params)), copyLink: (url) => clipboard.writeText(url), + copyImage: () => contents.copyImageAt(params.x, params.y), + // A download, so the browser's download limits and request guards apply. + saveImage: (url) => contents.downloadURL(url), setZoomFactor: (factor) => contents.setZoomFactor(factor), } ) diff --git a/apps/desktop/src/main/browser-agent/session.test.ts b/apps/desktop/src/main/browser-agent/session.test.ts index d95cc7709ba..8f04b61520c 100644 --- a/apps/desktop/src/main/browser-agent/session.test.ts +++ b/apps/desktop/src/main/browser-agent/session.test.ts @@ -352,6 +352,13 @@ describe('browser-agent session', () => { session = freshSession(win) }) + it('ignores browser shortcuts and mouse buttons before any chat has opened a browser', () => { + session.disposeBrowserScope('chat-test') + + expect(session.handleFocusedShortcut('back', win)).toBe(false) + expect(session.handleFocusedShortcut('close-tab', win)).toBe(false) + }) + it('invalidates only top-frame starts and identifies same-document commits', () => { const onTabNavigated = vi.fn() session = freshSession(win, { onTabNavigated }) diff --git a/apps/desktop/src/main/browser-agent/session.ts b/apps/desktop/src/main/browser-agent/session.ts index 0ac0c5d7c41..f96d0645f04 100644 --- a/apps/desktop/src/main/browser-agent/session.ts +++ b/apps/desktop/src/main/browser-agent/session.ts @@ -52,6 +52,7 @@ import { isDispatchingAgentInput } from '@/main/browser-agent/cdp' import { attachAgentContextMenu, BASE_ZOOM_FACTOR, + type PageNavigation, steppedZoomFactor, } from '@/main/browser-agent/context-menu' import type { BrowserCookieSignal } from '@/main/browser-agent/known-sessions' @@ -81,10 +82,11 @@ import { import { clearHostVerdictCache } from '@/main/browser-agent/url-guard' import type { BrowserSessionSnapshot } from '@/main/desktop-chat-session-store' import { suggestedFilename, uniqueDownloadPath } from '@/main/downloads' -import { isAppOrigin, isBlankPopupUrl } from '@/main/navigation' +import { isAppOrigin, isBlankPopupUrl, parseHttpUrl } from '@/main/navigation' import { type FocusedResourceShortcut, isResourceTabSelectionShortcut, + type ResourceTabSelectionShortcut, resourceTabTargetIndex, zoomActionForShortcut, } from '@/main/resource-shortcuts' @@ -232,39 +234,73 @@ export type BrowserShortcut = | 'find' | 'reload' | 'hard-reload' + | UserOnlyShortcut + +/** Shortcuts that move the user's history, tabs, or load; keystrokes the agent sends never trigger them. */ +const USER_ONLY_SHORTCUTS = ['back', 'forward', 'previous-tab', 'next-tab', 'stop'] as const +type UserOnlyShortcut = (typeof USER_ONLY_SHORTCUTS)[number] + +function isUserOnlyShortcut(shortcut: BrowserShortcut): shortcut is UserOnlyShortcut { + return USER_ONLY_SHORTCUTS.some((candidate) => candidate === shortcut) +} type BrowserShortcutInput = Pick< Input, - 'type' | 'key' | 'isAutoRepeat' | 'isComposing' | 'shift' | 'control' | 'alt' | 'meta' + 'type' | 'key' | 'code' | 'isAutoRepeat' | 'isComposing' | 'shift' | 'control' | 'alt' | 'meta' > /** * Resolves browser-level shortcuts using Command on macOS and Control - * elsewhere. Alt, composing, and repeated keystrokes stay with the page. + * elsewhere, following Chrome's bindings. Composing and repeated keystrokes, + * and every other Alt chord, stay with the page. * * Reload resolves here, from the page's own keystroke, rather than through the * application menu: the menu can only guess which surface owns focus, and a * wrong guess reloads all of Sim instead of the page. + * + * `stop` is any bare Escape; only a loading page acts on it, and the page + * still receives the key. */ export function browserShortcutForInput( input: BrowserShortcutInput, platform: NodeJS.Platform = process.platform ): BrowserShortcut | null { - if (input.type !== 'keyDown' || input.isAutoRepeat || input.isComposing || input.alt) { - return null + if (input.type !== 'keyDown' || input.isAutoRepeat || input.isComposing) return null + const isMac = platform === 'darwin' + const primaryModifier = isMac ? input.meta : input.control + const otherModifier = isMac ? input.control : input.meta + + if (input.alt) { + if (input.shift || otherModifier) return null + const arrow = input.key === 'ArrowLeft' ? 'left' : input.key === 'ArrowRight' ? 'right' : null + if (!arrow) return null + // As in Chrome, these win even inside a text field. + if (isMac) { + if (!input.meta) return null + return arrow === 'left' ? 'previous-tab' : 'next-tab' + } + if (input.control) return null + return arrow === 'left' ? 'back' : 'forward' } - const primaryModifier = platform === 'darwin' ? input.meta : input.control - const otherModifier = platform === 'darwin' ? input.control : input.meta - + if (input.key === 'Escape' && !input.shift && !input.control && !input.meta) return 'stop' if (input.key === 'F5' && !otherModifier) { if (input.shift || input.control) return 'hard-reload' return input.meta ? null : 'reload' } if (!primaryModifier || otherModifier) return null + if (!isMac && !input.shift && (input.key === 'PageUp' || input.key === 'PageDown')) { + return input.key === 'PageUp' ? 'previous-tab' : 'next-tab' + } const key = input.key.toLowerCase() if (key === 'r') return input.shift ? 'hard-reload' : 'reload' - if (input.shift) return null + if (input.shift) { + if (!isMac) return null + // Physical key: Shift turns the bracket into a brace on most layouts. + if (input.code === 'BracketLeft') return 'previous-tab' + if (input.code === 'BracketRight') return 'next-tab' + return null + } switch (key) { case 'l': return 'focus-omnibox' @@ -1605,6 +1641,12 @@ const browserPermissions: BrowserPermissionHandlers = { withBrowserScope(scoped.scopeId, () => publishPageIssue(scoped.tab)) return } + // Blink asks only after a click or key, and the view keeps the window as it + // is; Electron never settles a refused request, blocking every later one. + if (permission === 'fullscreen') { + callback(Boolean(scopedTabForContents(contents))) + return + } callback(ALLOWED_SITE_PERMISSIONS.has(permission)) }, check: (contents, permission, requestingOrigin, details) => { @@ -2177,6 +2219,38 @@ function hardReloadPage(contents: WebContents): void { contents.reloadIgnoringCache() } +/** A browser-chrome history move or reload, held for the page's leave-site question. */ +function userNavigate(contents: WebContents, kind: PageNavigation): void { + navigateForUser(contents, () => + kind === 'back' + ? goBack(contents) + : kind === 'forward' + ? goForward(contents) + : reloadPage(contents) + ) +} + +/** Moves one tab's history, or selects a tab relative to it in the strip. */ +function runTabShortcut( + tab: AgentTab, + shortcut: 'back' | 'forward' | ResourceTabSelectionShortcut +): void { + if (shortcut === 'back' || shortcut === 'forward') { + userNavigate(tab.view.webContents, shortcut) + return + } + const targetIndex = resourceTabTargetIndex( + shortcut, + tabs.length, + tabs.findIndex((entry) => entry.id === tab.id) + ) + const target = targetIndex === null ? null : tabs[targetIndex] + if (target) { + switchTab(target.id) + target.view.webContents.focus() + } +} + /** Hands one page selection to the exact app window and chat hosting its tab. */ function addPageSelectionToChat(contents: WebContents, text: string): void { if (!text.trim() || getBrowserScopeId() !== getActiveBrowserScopeId()) return @@ -2362,7 +2436,7 @@ function openTabWithUrl( referrer, }: { agentOwned: boolean; background?: boolean; referrer?: Referrer } ): void { - if (!/^https?:\/\//i.test(url)) return + if (!parseHttpUrl(url)) return try { const tab = openLinkTab(url, { agentOwned, background }) void tab.view.webContents.loadURL(url, { httpReferrer: referrer }).catch(() => {}) @@ -2417,6 +2491,19 @@ function createTabView(url?: string, popup?: AdoptedPopup): WebContentsView { } } +/** + * Preferences every tab needs, popups Chromium creates included. + * `focusOnNavigation`: Electron focuses a page on every main-frame commit, even + * a hidden agent tab, which steals the caret from chat; focus follows explicit + * user actions instead (see focusPageForUser). + * `disableHtmlFullscreenWindowResize`: a page's fullscreen fills the browser + * panel, never the Sim window. + */ +const AGENT_TAB_WEB_PREFERENCES = { + focusOnNavigation: false, + disableHtmlFullscreenWindowResize: true, +} as const + function createFreshTabView(appSession: BrowserAppSession | undefined): WebContentsView { return new WebContentsView({ webPreferences: { @@ -2437,11 +2524,9 @@ function createFreshTabView(appSession: BrowserAppSession | undefined): WebConte // the active tab while a tool waits on it, applied explicitly by // applyAutomationTabPolicy — never blanket across every tab. backgroundThrottling: true, - // Electron focuses a page on every main-frame commit, even a hidden - // agent tab, which steals the caret from chat. Focus follows explicit - // user actions instead (see focusPageForUser). - focusOnNavigation: false, - spellcheck: false, + ...AGENT_TAB_WEB_PREFERENCES, + // Off elsewhere: Chromium would download Hunspell dictionaries. + spellcheck: process.platform === 'darwin', // The default every origin this tab visits starts at; a per-origin zoom // the user sets from the page menu still wins and still persists. zoomFactor: getBrowserDefaultZoomFactor(), @@ -2485,10 +2570,27 @@ function initializeTabView( registerAgentNavigation(contents, routeNavigation) attachAgentContextMenu(contents, { addToChat: (text) => withBrowserScope(scopeId, () => addPageSelectionToChat(contents, text)), - openTab: (url) => - withBrowserScope(scopeId, () => openTabWithUrl(url, { agentOwned: false, background: true })), + openTab: (url, referrer) => + withBrowserScope(scopeId, () => + openTabWithUrl(url, { agentOwned: false, background: true, referrer }) + ), defaultZoomFactor: getBrowserDefaultZoomFactor, + history: () => + withBrowserScope(scopeId, () => ({ + canGoBack: canGoBack(contents), + canGoForward: canGoForward(contents), + })), + navigate: (kind) => withBrowserScope(scopeId, () => userNavigate(contents, kind)), }) + // Chromium leaves Control-wheel zoom to the embedder; step the page menu's ladder. + contents.on( + 'zoom-changed', + bindToBrowserScope(scopeId, (_event, direction) => { + contents.setZoomFactor( + steppedZoomFactor(contents.getZoomFactor(), direction === 'in' ? 1 : -1) + ) + }) + ) contents.on( 'focus', @@ -2591,9 +2693,8 @@ function initializeTabView( return { action: 'allow', outlivesOpener: true, - // A popup's contents are created by Chromium from these preferences, not - // from createFreshTabView, so they must opt out of navigation focus here. - overrideBrowserWindowOptions: { webPreferences: { focusOnNavigation: false } }, + // Chromium creates a popup's contents from these, not from createFreshTabView. + overrideBrowserWindowOptions: { webPreferences: AGENT_TAB_WEB_PREFERENCES }, createWindow: (options) => withBrowserScope(scopeId, () => adoptPopupTab(contents, details.url, options, { @@ -2675,20 +2776,31 @@ function initializeTabView( 'before-input-event', bindToBrowserScope(scopeId, (event, input) => { const tab = tabs.find((entry) => entry.view === view) + const agentInput = isDispatchingAgentInput(contents) if ( - !isDispatchingAgentInput(contents) && + !agentInput && input.type === 'keyDown' && tab && tab.id === currentScope.automationTabId ) { noteUserIntervention(tab.id) } - if (!isDispatchingAgentInput(contents) && tab?.id === currentScope.activeTabId) { + if (!agentInput && tab?.id === currentScope.activeTabId) { currentScope.visibleTabUserSelected = true if (input.type === 'keyDown' && !input.isAutoRepeat) tab.lastRealUserGestureAt = Date.now() } const shortcut = browserShortcutForInput(input) if (!shortcut) return + if (isUserOnlyShortcut(shortcut)) { + if (agentInput || !tab) return + if (shortcut === 'stop') { + if (contents.isLoadingMainFrame()) contents.stop() + return + } + event.preventDefault() + runTabShortcut(tab, shortcut) + return + } event.preventDefault() if (shortcut === 'focus-omnibox') { @@ -2705,7 +2817,7 @@ function initializeTabView( return } if (shortcut === 'reload') { - navigateForUser(contents, () => reloadPage(contents)) + userNavigate(contents, 'reload') return } if (shortcut === 'hard-reload') { @@ -3697,6 +3809,8 @@ export function handleFocusedShortcut( shortcut: FocusedResourceShortcut, ownerWindow?: BrowserWindow | null ): boolean { + // Accelerators and mouse buttons arrive before any chat has opened a browser. + if (!getActiveBrowserScopeId()) return false // Tab-management accelerators belong to the visible Browser even after // focus moves into Sim chrome. Otherwise Cmd-T/L/Shift-T silently stop // behaving like browser shortcuts, and Cmd-W becomes especially unsafe. @@ -3710,17 +3824,8 @@ export function handleFocusedShortcut( (visibleBrowserShortcut ? visibleActiveTab(ownerWindow) : null) if (!shortcutTab) return false - if (isResourceTabSelectionShortcut(shortcut)) { - const targetIndex = resourceTabTargetIndex( - shortcut, - tabs.length, - tabs.findIndex((tab) => tab.id === shortcutTab.id) - ) - const target = targetIndex === null ? null : tabs[targetIndex] - if (target) { - switchTab(target.id) - target.view.webContents.focus() - } + if (isResourceTabSelectionShortcut(shortcut) || shortcut === 'back' || shortcut === 'forward') { + runTabShortcut(shortcutTab, shortcut) return true } @@ -3740,26 +3845,14 @@ export function handleFocusedShortcut( case 'focus-omnibox': focusRendererOmnibox('select') return true - case 'reload-or-clear': { - const contents = shortcutTab.view.webContents - navigateForUser(contents, () => reloadPage(contents)) + case 'reload-or-clear': + userNavigate(shortcutTab.view.webContents, 'reload') return true - } case 'hard-reload': { const contents = shortcutTab.view.webContents navigateForUser(contents, () => hardReloadPage(contents)) return true } - case 'back': { - const contents = shortcutTab.view.webContents - navigateForUser(contents, () => goBack(contents)) - return true - } - case 'forward': { - const contents = shortcutTab.view.webContents - navigateForUser(contents, () => goForward(contents)) - return true - } } const zoomAction = zoomActionForShortcut(shortcut) diff --git a/apps/desktop/src/main/context-menu.ts b/apps/desktop/src/main/context-menu.ts index 689da904371..d0c88501507 100644 --- a/apps/desktop/src/main/context-menu.ts +++ b/apps/desktop/src/main/context-menu.ts @@ -9,14 +9,37 @@ export interface ContextMenuDeps { allowHttpLocalhost: boolean } -interface TemplateHandlers { +interface SpellingHandlers { replaceMisspelling(word: string): void addToDictionary(word: string): void +} + +interface TemplateHandlers extends SpellingHandlers { openLink(url: string): void copyLink(url: string): void inspect(x: number, y: number): void } +/** Corrections for the misspelled word under the cursor, then Add to Dictionary; empty when none. */ +export function spellingItems( + params: Pick, + handlers: SpellingHandlers +): MenuItemConstructorOptions[] { + if (!params.misspelledWord) return [] + const suggestions = params.dictionarySuggestions.slice(0, MAX_SUGGESTIONS) + return [ + ...(suggestions.length > 0 + ? suggestions.map( + (suggestion): MenuItemConstructorOptions => ({ + label: suggestion, + click: () => handlers.replaceMisspelling(suggestion), + }) + ) + : [{ label: 'No Guesses Found', enabled: false }]), + { label: 'Add to Dictionary', click: () => handlers.addToDictionary(params.misspelledWord) }, + ] +} + /** * Builds the native right-click menu for a given context. Returns an empty * template when there is nothing editable, selected, or linked — which leaves @@ -38,18 +61,8 @@ export function buildContextMenuTemplate( ): MenuItemConstructorOptions[] { const template: MenuItemConstructorOptions[] = [] - if (params.misspelledWord) { - for (const suggestion of params.dictionarySuggestions.slice(0, MAX_SUGGESTIONS)) { - template.push({ label: suggestion, click: () => handlers.replaceMisspelling(suggestion) }) - } - if (params.dictionarySuggestions.length === 0) { - template.push({ label: 'No Guesses Found', enabled: false }) - } - template.push( - { label: 'Add to Dictionary', click: () => handlers.addToDictionary(params.misspelledWord) }, - { type: 'separator' } - ) - } + const spelling = spellingItems(params, handlers) + if (spelling.length > 0) template.push(...spelling, { type: 'separator' }) if (params.isEditable) { template.push( diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 56c4261fe10..152e5827147 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -531,6 +531,12 @@ function main(): void { win.on('focus', () => { lastActiveWindow = win }) + // Windows and Linux report a mouse's back and forward buttons as app + // commands. They move the Browser tab the user is in, never the Sim shell. + win.on('app-command', (_event, command) => { + if (command === 'browser-backward') handleFocusedBrowserShortcut('back', win) + else if (command === 'browser-forward') handleFocusedBrowserShortcut('forward', win) + }) // A fresh document (reload, origin change, crash recovery) has no browser // panel mounted yet — hide the embedded agent-browser view immediately // rather than letting it linger over the loading page.