Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions .claude/skills/project-guardrails/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,16 @@ genuinely correct, and reaching for it means it's time to stop and ask.
entry the user opens and asks for a task, past dictations included. The
third is the Rewrite Selection key's (`Rewrite/SelectionRewriter.swift`,
through `GatewayConnection.send`): highlighted text rewritten as a spoken
instruction asks, and pasted over the selection — the one gateway output
that is pasted, owner-directed (2026-11-03). It never touches a dictation's
words; the instruction it is given is a hand-back press's, never pasted.
instruction asks, and pasted over the selection — owner-directed
(2026-11-03). It never touches a dictation's words; the instruction it is
given is a hand-back press's, never pasted. Its sibling, the Compose From
Instructions key's (`Rewrite/MessageComposer.swift`, also through
`GatewayConnection.send`), writes the text a hand-back press's spoken
instructions describe ("a Slack reply to Dan saying…") for the app in front,
pasted at the cursor — owner-directed (2026-11-06). Those two are the only
gateway outputs that are pasted. Neither is a door back to a cleanup pass:
the words they are given are instructions to work _from_, never the text
pasted tidied up.
The fourth is the Control Your Mac key's (`Command/VoiceCommandPlanner.swift`,
through `GatewayConnection.send`): a spoken command turned into steps from a
closed list (`VoiceCommandStep`) that Blurt carries out itself — the one
Expand Down
80 changes: 48 additions & 32 deletions AGENTS.md

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion App/Blurt/Blurt/AppCoordinator.swift
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ final class AppCoordinator {
let router =
triggerKeySeams.map {
SelectionSpeechRouter(
session: session, speaker: $0.speaker, rewriter: $0.rewriter, planner: $0.planner,
session: session, speaker: $0.speaker, rewriter: $0.rewriter, composer: $0.composer, planner: $0.planner,
performer: $0.performer, readSelection: $0.readSelection)
}
?? SelectionSpeechRouter(
Expand Down
5 changes: 3 additions & 2 deletions App/Blurt/Blurt/AppDelegate.swift
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,9 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
triggerKeys: TriggerKeySeams(
speaker: UITestReadAloudSpeaker(),
readSelection: { [selection = UITestIdentifiers.cannedSelection] in selection },
// Rewrite and command keys offline too: no gateway, nothing acted on.
rewriter: UITestRewriter(), planner: UITestPlanner(), performer: UITestPerformer()),
// Rewrite, compose and command keys offline too: no gateway, nothing acted on.
rewriter: UITestRewriter(), composer: UITestComposer(), planner: UITestPlanner(),
performer: UITestPerformer()),
// A fresh history per run — never the user's saved transcripts —
// seeded with known entries when the suite asks, and the canned
// transport behind transcription and the LLM.
Expand Down
3 changes: 3 additions & 0 deletions App/Blurt/Blurt/Overlay/OverlayView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,9 @@ struct OverlayView: View {
case .rewriting:
waitingContent("Rewriting")
.transition(.opacity)
case .composing:
waitingContent("Writing")
.transition(.opacity)
case .commanding:
waitingContent("Working")
.transition(.opacity)
Expand Down
4 changes: 2 additions & 2 deletions App/Blurt/Blurt/SoundPackCatalog+ReadAloud.swift
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@ extension SoundPackStore {

/// The setting a key with no voice of its own follows: its action's. A read
/// key sounds like a read and a command key like a command, and any other
/// key — dictate, ask, rewrite, each a recording — like a dictation.
/// key — dictate, ask, rewrite, compose, each a recording — like a dictation.
static func fallback(for action: TriggerKeyAction) -> SoundPackStore {
switch action {
case .readAloud: .readAloud
case .command: .command
case .off, .dictate, .ask, .rewrite: .dictation
case .off, .dictate, .ask, .rewrite, .compose: .dictation
}
}
}
Expand Down
4 changes: 3 additions & 1 deletion App/Blurt/Blurt/TriggerKeySeams.swift
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
import BlurtEngine

/// Stand-ins for the trigger keys' own jobs — read-aloud's speaker and
/// selection read, Rewrite Selection's rewriter, Control Your Mac's planner
/// selection read, Rewrite Selection's rewriter, Compose From Instructions'
/// composer, Control Your Mac's planner
/// and performer — so a host can run them with no network, audio or
/// Accessibility (Blurt's UI tests).
struct TriggerKeySeams {
let speaker: any ReadAloudSpeaker
let readSelection: @Sendable () async -> String?
let rewriter: any SelectionRewriting
let composer: any MessageComposing
let planner: any VoiceCommandPlanning
let performer: any VoiceCommandPerforming
}
10 changes: 9 additions & 1 deletion App/Blurt/Blurt/UITestCommandDoubles.swift
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import BlurtEngine
import Foundation

// The rewrite and command keys' stand-ins under UI testing, beside the
// The rewrite, compose and command keys' stand-ins under UI testing, beside the
// pipeline doubles in UITestSupport.swift: no gateway, nothing acted on.

/// Stub rewriter: the rewrite a spoken instruction asks for, as a marked
Expand All @@ -13,6 +13,14 @@
}
}

/// Stub composer: the instructions as a marked echo, so a test sees them
/// reach it and the result pasted.
nonisolated struct UITestComposer: MessageComposing {
func compose(_ instructions: String, for app: String?) async throws -> String {
"Composed (\(instructions))"
}
}

/// Stub planner: a few canned commands, each a fixed plan, so the router's
/// whole command path runs with no gateway. "Open X" opens X; "type X"
/// types it; "mute" mutes; anything else is a command Blurt can't do.
Expand Down
2 changes: 2 additions & 0 deletions App/Blurt/Blurt/UITestSupport.swift
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,8 @@
.accessibilityIdentifier(UITestIdentifiers.harnessSetCommand)
Button("Rewrite") { coordinator?.setKeyChoiceForTesting(.rewrite, for: state.selectedKey) }
.accessibilityIdentifier(UITestIdentifiers.harnessSetRewrite)
Button("Compose") { coordinator?.setKeyChoiceForTesting(.compose, for: state.selectedKey) }
.accessibilityIdentifier(UITestIdentifiers.harnessSetCompose)
Button("Tap Mode") { coordinator?.setActivationForTesting(.tap) }
.accessibilityIdentifier(UITestIdentifiers.harnessSetTapMode)
}
Expand Down
1 change: 1 addition & 0 deletions App/Blurt/Blurt/Wizard/KeyCap.swift
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ extension TriggerKeyAction {
case .readAloud: LibraryKind.readAloud.systemImage
case .ask: LibraryKind.ask.systemImage
case .rewrite: "pencil.line"
case .compose: "square.and.pencil"
case .command: "command"
case .dictate, .off: "keyboard"
}
Expand Down
1 change: 1 addition & 0 deletions App/Blurt/Blurt/Wizard/KeysSettingsTab.swift
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ struct KeysSettingsTab: View {
ActionRow(choice: .readAloud, title: KeyActionName.readAloud),
ActionRow(choice: .ask, title: KeyActionName.ask),
ActionRow(choice: .rewrite, title: KeyActionName.rewrite),
ActionRow(choice: .compose, title: KeyActionName.compose),
ActionRow(choice: .command, title: KeyActionName.command),
]
}
Expand Down
1 change: 1 addition & 0 deletions App/Blurt/Blurt/Wizard/ReadyView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ struct ReadyView: View {
case .readAloud: LibraryKind.readAloud.name
case .ask: LibraryKind.ask.name
case .rewrite: KeyActionName.rewrite
case .compose: KeyActionName.compose
case .command: KeyActionName.command
case .dictate(TriggerKeyChoice.mainWindowStyle), .off: "Dictate"
case .dictate(let style):
Expand Down
1 change: 1 addition & 0 deletions App/Blurt/Blurt/Wizard/SoundsSettingsTab.swift
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ struct SoundsSettingsTab: View {
case .readAloud: KeyActionName.readAloud
case .ask: KeyActionName.ask
case .rewrite: KeyActionName.rewrite
case .compose: KeyActionName.compose
case .command: KeyActionName.command
}
}
Expand Down
10 changes: 10 additions & 0 deletions App/Blurt/BlurtUITests/CommandPipelineUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -116,4 +116,14 @@ final class CommandPipelineUITests: BlurtUITestCase {
waitForText(
pastedLabel(in: harness), containing: "Rewritten (Make it shorter)", timeout: 15, "The rewrite should paste")
}

/// A compose key: no selection needed; what the spoken instructions describe is pasted at the cursor.
func testComposeKeyPastesTheComposition() {
let harness = readyHarness()
bind("left ⌃", with: UITestIdentifiers.harnessSetCompose, in: harness)
command("Tell Dan we should ship it", in: harness)
waitForText(
pastedLabel(in: harness), containing: "Composed (Tell Dan we should ship it)", timeout: 15,
"The composition should paste")
}
}
5 changes: 3 additions & 2 deletions App/Blurt/BlurtUITests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,13 @@ observable UI rather than mocking views.
`DictationSession` press/release — via both the real key tap and the direct
Start/Stop seam — and asserts the live status walks to `recording` and back to
`idle`, including two back-to-back runs to prove the pipeline re-arms.
- **Control Your Mac and Rewrite Selection** — `CommandPipelineUITests`: a command key held
- **Control Your Mac, Rewrite Selection and Compose From Instructions** — `CommandPipelineUITests`: a command key held
through the real key tap, recorded and transcribed by the session, planned and carried out by the
router with the stub planner and performer (`UITestCommandDoubles.swift` — no gateway, nothing
acted on), and the pill: Working for as long as the command runs, then Done; a typed step pasted
through the session; a complaint noted, never planned; overheard speech refused; a command it can't
do failing with the key still working after; and a rewrite pasted over the canned selection.
do failing with the key still working after; a rewrite pasted over the canned selection; and a
composition pasted from spoken instructions, no selection needed.
- **Pasting into an app** — `DictationPipelineUITests`: the stub injector records
what it would paste, with the harness window standing in for the target app.

Expand Down
2 changes: 1 addition & 1 deletion App/Blurt/BlurtUITests/SettingsUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ final class SettingsUITests: BlurtUITestCase {

let actions = [
"Dictate", "Dictate: No custom style", KeyActionName.readAloud, KeyActionName.ask,
KeyActionName.rewrite, KeyActionName.command,
KeyActionName.rewrite, KeyActionName.compose, KeyActionName.command,
]
for action in actions {
XCTAssertTrue(
Expand Down
3 changes: 2 additions & 1 deletion App/Blurt/Shared/KeyActionName.swift
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
/// The names the Keys tab gives its read, ask, rewrite and command actions. The tab's menu, the
/// The names the Keys tab gives its read, ask, rewrite, compose and command actions. The tab's menu, the
/// ready screen's key rows, the Read Aloud tab's footers and the UI tests all
/// quote them, so they're spelled once — here in `Shared/`, where the UI tests
/// can reach them too.
enum KeyActionName {
static let readAloud = "Read Selection Aloud"
static let ask = "Ask About Selection"
static let rewrite = "Rewrite Selection"
static let compose = "Compose From Instructions"
static let command = "Control Your Mac"
}
1 change: 1 addition & 0 deletions App/Blurt/Shared/UITestIdentifiers.swift
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ enum UITestIdentifiers {
static func keySoundPicker(_ keyCode: Int) -> String { "settings.keySound.picker.\(keyCode)" }
static let harnessSetCommand = "harness.key.setCommand"
static let harnessSetRewrite = "harness.key.setRewrite"
static let harnessSetCompose = "harness.key.setCompose"
/// What the stub planner was asked to plan, and what the stub performer did.
static let plannedLabel = "harness.planned"
static let performedLabel = "harness.performed"
Expand Down
8 changes: 4 additions & 4 deletions App/BlurtiOS/BlurtiOSCore/VoiceState.swift
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,8 @@ extension PhaseSnapshot.State {
nonisolated package init(_ overlay: OverlayUIState) {
self =
switch overlay {
// Reading aloud, rewriting and commands are the Mac's alone, so the iPhone never sees them.
case .idle, .speaking, .rewriting, .commanding, .done, .noted: .idle
// Reading aloud, rewriting, composing and commands are the Mac's alone, so the iPhone never sees them.
case .idle, .speaking, .rewriting, .composing, .commanding, .done, .noted: .idle
case .connecting: .connecting
case .recording: .recording
case .processing: .processing
Expand All @@ -140,8 +140,8 @@ extension PhaseSnapshot {
nonisolated package static func message(for overlay: OverlayUIState) -> String? {
switch overlay {
case .error(let message, _): message
case .idle, .connecting, .recording, .processing, .speaking, .rewriting, .commanding, .pasted, .noTarget,
.done, .noted:
case .idle, .connecting, .recording, .processing, .speaking, .rewriting, .composing, .commanding, .pasted,
.noTarget, .done, .noted:
nil
}
}
Expand Down
56 changes: 41 additions & 15 deletions Sources/BlurtEngine/Hotkey/SelectionSpeechRouter.swift
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ import Foundation
/// set to read aloud reads the selected text; a press of a key set to ask
/// records a spoken request about it and reads the answer; a press of a key
/// set to rewrite records a spoken instruction and pastes the selection,
/// rewritten that way, over it; a press of a key set to command records a
/// rewritten that way, over it; a press of a key set to compose records
/// spoken instructions and pastes what they describe at the cursor; a press
/// of a key set to command records a
/// spoken command and carries it out; a dictating key's
/// commands go straight to `DictationSession.submit` (see `TriggerKeyBindings`
/// for which key does what).
Expand All @@ -26,6 +28,7 @@ import Foundation
private let session: DictationSession
private let speaker: any ReadAloudSpeaker
private let rewriter: any SelectionRewriting
private let composer: any MessageComposing
private let planner: any VoiceCommandPlanning
private let performer: any VoiceCommandPerforming
private let readSelection: @Sendable () async -> String?
Expand All @@ -35,8 +38,8 @@ import Foundation
private let clock: any Clock<Duration>
private var routing = SelectionSpeechRouting()
private var speech: Task<Void, Never>?
/// The job in flight — a rewrite until it pastes, a command until its
/// last step. Cancelled by any press.
/// The job in flight — a rewrite or a composition until it pastes, a
/// command until its last step. Cancelled by any press.
private var job: Task<Void, Never>?
/// The screen read a command's release started (`.readScreen`), running
/// while the command transcribes; `.runCommand` plans with its result.
Expand All @@ -63,7 +66,7 @@ import Foundation
/// A read, answer or voice sample failed (it's already logged): the app
/// beeps, since the pill just goes away.
public var speechFailed: (any Error) -> Void = { _ in }
/// What the pill shows for a job: `.rewriting` or `.commanding` as it
/// What the pill shows for a job: `.rewriting`, `.composing` or `.commanding` as it
/// starts, then its outcome — Pasted (Done, for a command), Copied, or the
/// error — or nil when a press abandoned it, and the pill goes back to
/// what the session is doing.
Expand All @@ -81,14 +84,15 @@ import Foundation
session: session, speaker: SelectionSpeaker(clips: clips), readSelection: SelectionSpeaker.focusedSelection)
}

/// The seams: what speaks, what rewrites, what plans and carries out a
/// The seams: what speaks, what rewrites, what composes, what plans and carries out a
/// command, how the selection is read, where the
/// settings each press reads come from, and the clock its timers run on.
/// Public so a host can run the whole read-aloud path offline (Blurt's UI
/// tests do).
public init(
session: DictationSession, speaker: any ReadAloudSpeaker,
rewriter: any SelectionRewriting = SelectionRewriter(),
composer: any MessageComposing = MessageComposer(),
planner: any VoiceCommandPlanning = VoiceCommandPlanner(),
performer: any VoiceCommandPerforming = SystemVoiceCommandPerformer(),
readSelection: @escaping @Sendable () async -> String?, defaults: UserDefaults = .standard,
Expand All @@ -97,6 +101,7 @@ import Foundation
self.session = session
self.speaker = speaker
self.rewriter = rewriter
self.composer = composer
self.planner = planner
self.performer = performer
self.readSelection = readSelection
Expand Down Expand Up @@ -163,13 +168,13 @@ import Foundation
case .startHoldTimer: startHoldTimer()
case .syncGateAfterSession: syncGateAfterSession()
case .resetGate: resetGate()
case .speak, .answer, .stopSpeech, .rewrite, .runCommand, .stopJob: performTask(action)
case .speak, .answer, .stopSpeech, .rewrite, .compose, .runCommand, .stopJob: performTask(action)
}
}
}

/// The actions that start or stop a task of this router's own: a read, an
/// answer, a rewrite, a command.
/// answer, a rewrite, a composition, a command.
private func performTask(_ action: SelectionSpeechRouting.Action) {
switch action {
case .speak(let text):
Expand All @@ -190,11 +195,12 @@ import Foundation
// by this read's late end.
endSpeech()
case .rewrite(let request, let selection): rewrite(selection, as: request)
case .compose(let request): compose(request)
case .runCommand(let command): run(command)
case .stopJob:
// A stopped command's own end never runs, so stop its screen read and
// undo its preparation here — before it ran, too; for a rewrite
// there's neither, and this does nothing.
// undo its preparation here — before it ran, too; for a rewrite or a
// composition there's neither, and this does nothing.
screenRead?.cancel()
screenRead = nil
Task { [performer] in await performer.commandEnded() }
Expand Down Expand Up @@ -227,17 +233,37 @@ import Foundation
/// rewrite this router stopped reports nothing more: `.stopJob` already
/// ended it.
private func rewrite(_ selection: String, as request: String) {
jobPill(.rewriting)
job = Task { [weak self, rewriter, session] in
paste(as: .rewriting, named: "rewrite") { [rewriter] in try await rewriter.rewrite(selection, as: request) }
}

/// Writes what the spoken `instructions` describe and pastes it at the
/// cursor as a rewrite is pasted (`replaceSelection`: the hand-back
/// press's target, no separating space). Written for that same target
/// ("Slack" gets a chat message, "Mail" an email) — the app the press
/// pinned, not whatever the user switched to while it transcribed.
private func compose(_ instructions: String) {
paste(as: .composing, named: "compose") { [composer, session] in
try await composer.compose(instructions, for: await session.pinnedAppName)
}
}

/// One rewrite or composition as the router's job: `pill` while `write`
/// runs, then its text pasted through the session's injector and the
/// outcome on the pill.
private func paste(
as pill: OverlayUIState, named name: String, _ write: @escaping @Sendable () async throws -> String
) {
jobPill(pill)
job = Task { [weak self, session] in
let outcome: OverlayUIState
do {
let text = try await rewriter.rewrite(selection, as: request)
let text = try await write()
try Task.checkCancellation()
outcome = try await session.replaceSelection(with: text).overlayState
} catch {
// A cancelled request answers `URLError.cancelled`: abandoned, not failed.
if Self.isCancellation(error) { return }
Self.logger.error("rewrite failed: \(error.localizedDescription, privacy: .public)")
Self.logger.error("\(name, privacy: .public) failed: \(error.localizedDescription, privacy: .public)")
outcome = Self.failure(error)
}
guard !Task.isCancelled, let self else { return }
Expand Down Expand Up @@ -286,13 +312,13 @@ import Foundation
}
}

/// Whether a read, answer, rewrite or command is wearing the pill — what
/// Whether a read, answer, rewrite, composition or command is wearing the pill — what
/// `PipelinePhase.overlayState(whileSpeaking:)` asks, so a session phase
/// arriving under any of them can't cover it.
public var ownsPill: Bool { activeGeneration != nil || job != nil }

/// What the session's transcription shows as when it's a job's spoken
/// words — "Rewriting", "Working" — or nil for a dictation's "Transcribing":
/// words — "Rewriting", "Writing", "Working" — or nil for a dictation's "Transcribing":
/// what `PipelinePhase.overlayState(whileSpeaking:processingAs:)` asks.
public var processingState: OverlayUIState? { routing.processingState }

Expand Down
Loading
Loading