From 50a041c35a7e3bb8346fdd6ce352721d322b29c1 Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 17:18:32 +0700 Subject: [PATCH 01/15] feat(speech): add speech preferences and capability detection Refs #844 --- src/lib/speech-capabilities.test.ts | 231 ++++++++++++++++++++++++++++ src/lib/speech-capabilities.ts | 162 +++++++++++++++++++ src/lib/speech-prefs.test.ts | 112 ++++++++++++++ src/lib/speech-prefs.ts | 162 +++++++++++++++++++ 4 files changed, 667 insertions(+) create mode 100644 src/lib/speech-capabilities.test.ts create mode 100644 src/lib/speech-capabilities.ts create mode 100644 src/lib/speech-prefs.test.ts create mode 100644 src/lib/speech-prefs.ts diff --git a/src/lib/speech-capabilities.test.ts b/src/lib/speech-capabilities.test.ts new file mode 100644 index 0000000000..81c92406ae --- /dev/null +++ b/src/lib/speech-capabilities.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, it } from "vitest" + +import { + detectSpeechCapabilities, + resolveInputEngine, + resolveSpeechLanguage, + type SpeechCapabilities, +} from "./speech-capabilities" + +describe("speech capabilities detection", () => { + it("detects browser STT and media capture in a standard browser", () => { + const caps = detectSpeechCapabilities({ + isDesktop: false, + platform: "macos", + hasSpeechRecognition: true, + hasMediaDevices: true, + isSecureContext: true, + }) + expect(caps).toEqual({ + browserStt: true, + mediaCapture: true, + secureContext: true, + }) + }) + + it("supports webkitSpeechRecognition prefix", () => { + const caps = detectSpeechCapabilities({ + isDesktop: false, + platform: "macos", + webkitSpeechRecognition: function () {}, + hasMediaDevices: true, + isSecureContext: true, + }) + expect(caps.browserStt).toBe(true) + }) + + it("permits browser STT on desktop macOS", () => { + const caps = detectSpeechCapabilities({ + isDesktop: true, + platform: "macos", + hasSpeechRecognition: true, + hasMediaDevices: true, + isSecureContext: true, + }) + expect(caps.browserStt).toBe(true) + }) + + it("excludes browser STT on desktop Windows due to WebView2 limitations", () => { + const caps = detectSpeechCapabilities({ + isDesktop: true, + platform: "windows", + hasSpeechRecognition: true, + hasMediaDevices: true, + isSecureContext: true, + }) + expect(caps.browserStt).toBe(false) + }) + + it("excludes browser STT on desktop Linux due to WebKitGTK limitations", () => { + const caps = detectSpeechCapabilities({ + isDesktop: true, + platform: "linux", + hasSpeechRecognition: true, + hasMediaDevices: true, + isSecureContext: true, + }) + expect(caps.browserStt).toBe(false) + }) + + it("disables mediaCapture when not in a secure context", () => { + const caps = detectSpeechCapabilities({ + isDesktop: false, + platform: "macos", + hasSpeechRecognition: true, + hasMediaDevices: true, + isSecureContext: false, + }) + expect(caps.mediaCapture).toBe(false) + expect(caps.secureContext).toBe(false) + }) + + it("disables mediaCapture when getUserMedia is missing", () => { + const caps = detectSpeechCapabilities({ + isDesktop: false, + platform: "macos", + hasSpeechRecognition: true, + hasMediaDevices: false, + isSecureContext: true, + }) + expect(caps.mediaCapture).toBe(false) + expect(caps.secureContext).toBe(true) + }) +}) + +describe("resolveInputEngine decision table", () => { + const fullCaps: SpeechCapabilities = { + browserStt: true, + mediaCapture: true, + secureContext: true, + } + + const noBrowserCaps: SpeechCapabilities = { + browserStt: false, + mediaCapture: true, + secureContext: true, + } + + const noMicCaps: SpeechCapabilities = { + browserStt: false, + mediaCapture: false, + secureContext: true, + } + + const insecureCaps: SpeechCapabilities = { + browserStt: false, + mediaCapture: false, + secureContext: false, + } + + it("resolves explicit browser preference when available", () => { + expect(resolveInputEngine({ engine: "browser" }, fullCaps, false)).toEqual({ + engine: "browser", + }) + }) + + it("returns no-engine when explicit browser preference is unavailable", () => { + expect( + resolveInputEngine({ engine: "browser" }, noBrowserCaps, true) + ).toEqual({ + engine: null, + reason: "no-engine", + }) + }) + + it("resolves explicit cloud preference when media capture and cloud config exist", () => { + expect( + resolveInputEngine({ engine: "cloud" }, noBrowserCaps, true) + ).toEqual({ + engine: "cloud", + }) + }) + + it("returns insecure-context for explicit cloud preference when insecure", () => { + expect(resolveInputEngine({ engine: "cloud" }, insecureCaps, true)).toEqual( + { + engine: null, + reason: "insecure-context", + } + ) + }) + + it("returns no-mic for explicit cloud preference when mic is missing", () => { + expect(resolveInputEngine({ engine: "cloud" }, noMicCaps, true)).toEqual({ + engine: null, + reason: "no-mic", + }) + }) + + it("returns cloud-not-configured for explicit cloud preference when unconfigured", () => { + expect( + resolveInputEngine({ engine: "cloud" }, noBrowserCaps, false) + ).toEqual({ + engine: null, + reason: "cloud-not-configured", + }) + }) + + it("resolves auto preference to browser when browser STT is available", () => { + expect(resolveInputEngine({ engine: "auto" }, fullCaps, false)).toEqual({ + engine: "browser", + }) + }) + + it("resolves auto preference to cloud when browser is unavailable but cloud is ready", () => { + expect(resolveInputEngine({ engine: "auto" }, noBrowserCaps, true)).toEqual( + { + engine: "cloud", + } + ) + }) + + it("prioritizes insecure-context for auto fallback", () => { + expect(resolveInputEngine({ engine: "auto" }, insecureCaps, false)).toEqual( + { + engine: null, + reason: "insecure-context", + } + ) + }) + + it("prioritizes cloud-not-configured over no-engine for auto fallback", () => { + expect( + resolveInputEngine({ engine: "auto" }, noBrowserCaps, false) + ).toEqual({ + engine: null, + reason: "cloud-not-configured", + }) + }) + + it("returns no-mic for auto fallback when cloud is configured but mic missing", () => { + expect(resolveInputEngine({ engine: "auto" }, noMicCaps, true)).toEqual({ + engine: null, + reason: "no-mic", + }) + }) +}) + +describe("resolveSpeechLanguage mapping", () => { + it("uses custom preference language when non-empty", () => { + expect(resolveSpeechLanguage({ language: "fr-CA" }, "en")).toBe("fr-CA") + expect(resolveSpeechLanguage("de-AT", "zh-CN")).toBe("de-AT") + }) + + it("maps next-intl UI locales to BCP-47 tags when preference language is empty", () => { + expect(resolveSpeechLanguage({ language: "" }, "en")).toBe("en-US") + expect(resolveSpeechLanguage("", "zh-CN")).toBe("zh-CN") + expect(resolveSpeechLanguage("", "zh-TW")).toBe("zh-TW") + expect(resolveSpeechLanguage("", "ja")).toBe("ja-JP") + expect(resolveSpeechLanguage("", "ko")).toBe("ko-KR") + expect(resolveSpeechLanguage("", "es")).toBe("es-ES") + expect(resolveSpeechLanguage("", "de")).toBe("de-DE") + expect(resolveSpeechLanguage("", "fr")).toBe("fr-FR") + expect(resolveSpeechLanguage("", "pt")).toBe("pt-BR") + expect(resolveSpeechLanguage("", "ar")).toBe("ar-SA") + }) + + it("falls back to raw non-empty locale or en-US for unmapped or empty locales", () => { + expect(resolveSpeechLanguage("", "it")).toBe("it") + expect(resolveSpeechLanguage("", "")).toBe("en-US") + }) +}) diff --git a/src/lib/speech-capabilities.ts b/src/lib/speech-capabilities.ts new file mode 100644 index 0000000000..51bfb90685 --- /dev/null +++ b/src/lib/speech-capabilities.ts @@ -0,0 +1,162 @@ +import { detectPlatform } from "@/hooks/use-platform" +import { isDesktop as isDesktopRuntime } from "./platform" +import type { SpeechInputPrefs } from "./speech-prefs" + +export interface SpeechCapabilities { + browserStt: boolean + mediaCapture: boolean + secureContext: boolean +} + +export type InputEngineResolution = + | { engine: "browser" | "cloud" } + | { + engine: null + reason: + | "no-mic" + | "insecure-context" + | "no-engine" + | "cloud-not-configured" + } + +export interface SpeechCapabilitiesEnv { + isDesktop?: boolean + platform?: "macos" | "windows" | "linux" | "unknown" + SpeechRecognition?: unknown + webkitSpeechRecognition?: unknown + hasSpeechRecognition?: boolean + hasMediaDevices?: boolean + getUserMedia?: unknown + isSecureContext?: boolean +} + +export const LOCALE_TO_BCP47: Record = { + en: "en-US", + "zh-CN": "zh-CN", + "zh-TW": "zh-TW", + ja: "ja-JP", + ko: "ko-KR", + es: "es-ES", + de: "de-DE", + fr: "fr-FR", + pt: "pt-BR", + ar: "ar-SA", +} + +export function detectSpeechCapabilities( + env?: SpeechCapabilitiesEnv +): SpeechCapabilities { + const desktop = + env?.isDesktop ?? + (typeof window !== "undefined" ? isDesktopRuntime() : false) + const plat = + env?.platform ?? + (typeof window !== "undefined" ? detectPlatform() : "unknown") + + let hasRecognizer = false + if (typeof env?.hasSpeechRecognition === "boolean") { + hasRecognizer = env.hasSpeechRecognition + } else if ( + env?.SpeechRecognition !== undefined || + env?.webkitSpeechRecognition !== undefined + ) { + hasRecognizer = Boolean( + env.SpeechRecognition || env.webkitSpeechRecognition + ) + } else if (typeof window !== "undefined") { + const win = window as unknown as Record + hasRecognizer = Boolean( + win.SpeechRecognition || win.webkitSpeechRecognition + ) + } + + const isUnsupportedDesktop = + desktop && (plat === "windows" || plat === "linux") + const browserStt = hasRecognizer && !isUnsupportedDesktop + + const secureContext = + typeof env?.isSecureContext === "boolean" + ? env.isSecureContext + : typeof window !== "undefined" + ? Boolean(window.isSecureContext) + : false + + let hasGetUserMedia = false + if (typeof env?.hasMediaDevices === "boolean") { + hasGetUserMedia = env.hasMediaDevices + } else if (env?.getUserMedia !== undefined) { + hasGetUserMedia = Boolean(env.getUserMedia) + } else if (typeof navigator !== "undefined") { + hasGetUserMedia = Boolean(navigator.mediaDevices?.getUserMedia) + } + + const mediaCapture = hasGetUserMedia && secureContext + + return { + browserStt, + mediaCapture, + secureContext, + } +} + +export function resolveInputEngine( + pref: SpeechInputPrefs | { engine: "auto" | "browser" | "cloud" }, + caps: SpeechCapabilities, + cloudConfigured: boolean +): InputEngineResolution { + if (pref.engine === "browser") { + if (caps.browserStt) { + return { engine: "browser" } + } + return { engine: null, reason: "no-engine" } + } + + if (pref.engine === "cloud") { + if (caps.mediaCapture && cloudConfigured) { + return { engine: "cloud" } + } + if (!caps.secureContext) { + return { engine: null, reason: "insecure-context" } + } + if (!caps.mediaCapture) { + return { engine: null, reason: "no-mic" } + } + return { engine: null, reason: "cloud-not-configured" } + } + + if (caps.browserStt) { + return { engine: "browser" } + } + if (caps.mediaCapture && cloudConfigured) { + return { engine: "cloud" } + } + + if (!caps.secureContext) { + return { engine: null, reason: "insecure-context" } + } + if (!cloudConfigured) { + return { engine: null, reason: "cloud-not-configured" } + } + if (!caps.mediaCapture) { + return { engine: null, reason: "no-mic" } + } + return { engine: null, reason: "no-engine" } +} + +export function resolveSpeechLanguage( + pref: SpeechInputPrefs | { language?: string } | string, + uiLocale: string +): string { + const language = + typeof pref === "string" ? pref.trim() : (pref.language ?? "").trim() + + if (language.length > 0) { + return language + } + + if (LOCALE_TO_BCP47[uiLocale]) { + return LOCALE_TO_BCP47[uiLocale] + } + + return uiLocale.trim().length > 0 ? uiLocale.trim() : "en-US" +} diff --git a/src/lib/speech-prefs.test.ts b/src/lib/speech-prefs.test.ts new file mode 100644 index 0000000000..4e51d9333e --- /dev/null +++ b/src/lib/speech-prefs.test.ts @@ -0,0 +1,112 @@ +import { beforeEach, describe, expect, it } from "vitest" + +import { + DEFAULT_SPEECH_PREFS, + getSpeechPrefs, + loadSpeechPrefs, + parseSpeechPrefs, + resetSpeechPrefsCacheForTests, + saveSpeechPrefs, + subscribeSpeechPrefs, +} from "./speech-prefs" + +describe("speech preferences", () => { + beforeEach(() => { + localStorage.clear() + resetSpeechPrefsCacheForTests() + }) + + it("defaults to disabled with auto engine and empty language", () => { + expect(DEFAULT_SPEECH_PREFS).toEqual({ + input: { + enabled: false, + engine: "auto", + language: "", + }, + }) + expect(loadSpeechPrefs()).toEqual(DEFAULT_SPEECH_PREFS) + }) + + it("returns a fresh object copy on load to avoid mutating default", () => { + const loaded = loadSpeechPrefs() + loaded.input.enabled = true + expect(DEFAULT_SPEECH_PREFS.input.enabled).toBe(false) + }) + + it("round-trips valid preference changes through save and load", () => { + const custom = { + input: { + enabled: true, + engine: "cloud" as const, + language: "zh-CN", + }, + } + saveSpeechPrefs(custom) + expect(loadSpeechPrefs()).toEqual(custom) + }) + + it("falls back per-field for invalid or missing values", () => { + const parsed = parseSpeechPrefs({ + input: { + enabled: "yes", + engine: "invalid-engine", + language: 12345, + }, + }) + expect(parsed).toEqual(DEFAULT_SPEECH_PREFS) + + const partial = parseSpeechPrefs({ + input: { + enabled: true, + engine: "browser", + }, + }) + expect(partial.input.enabled).toBe(true) + expect(partial.input.engine).toBe("browser") + expect(partial.input.language).toBe("") + }) + + it("falls back to default on corrupt storage JSON", () => { + localStorage.setItem("settings:speech:v1", "corrupt{json") + expect(loadSpeechPrefs()).toEqual(DEFAULT_SPEECH_PREFS) + }) + + it("notifies same-window subscribers and invalidates snapshot on save", () => { + const changes: boolean[] = [] + const unsubscribe = subscribeSpeechPrefs(() => { + changes.push(getSpeechPrefs().input.enabled) + }) + + saveSpeechPrefs({ + input: { + enabled: true, + engine: "auto", + language: "en-US", + }, + }) + + unsubscribe() + expect(changes).toEqual([true]) + }) + + it("invalidates memoized snapshot on storage event", () => { + const first = getSpeechPrefs() + expect(getSpeechPrefs()).toBe(first) + + localStorage.setItem( + "settings:speech:v1", + JSON.stringify({ + input: { enabled: true, engine: "cloud", language: "ja-JP" }, + }) + ) + window.dispatchEvent( + new StorageEvent("storage", { key: "settings:speech:v1" }) + ) + + const second = getSpeechPrefs() + expect(second).not.toBe(first) + expect(second.input.enabled).toBe(true) + expect(second.input.engine).toBe("cloud") + expect(second.input.language).toBe("ja-JP") + }) +}) diff --git a/src/lib/speech-prefs.ts b/src/lib/speech-prefs.ts new file mode 100644 index 0000000000..cac8f7f6f7 --- /dev/null +++ b/src/lib/speech-prefs.ts @@ -0,0 +1,162 @@ +"use client" + +/** + * Speech preferences: speech-to-text input configuration including engine choice + * and language selection. + * + * Stored in localStorage rather than the backend because speech input configuration + * (microphone access, engine preferences) is per-device. Uses the same reactive + * pattern as `notification-sound-prefs.ts`: a custom event for the current window + * plus the native `storage` event for cross-window/tab sync. + */ + +import { useSyncExternalStore } from "react" + +const PREFS_KEY = "settings:speech:v1" +const PREFS_EVENT = "codeg:speech-prefs-changed" + +export type SpeechEnginePreference = "auto" | "browser" | "cloud" + +export interface SpeechInputPrefs { + /** Master switch for speech input. Off by default. */ + enabled: boolean + /** STT engine preference. */ + engine: SpeechEnginePreference + /** BCP-47 tag or locale string; empty string means follow UI locale. */ + language: string +} + +export interface SpeechPrefs { + input: SpeechInputPrefs +} + +export const DEFAULT_SPEECH_PREFS: SpeechPrefs = { + input: { + enabled: false, + engine: "auto", + language: "", + }, +} + +function isSpeechEnginePreference( + value: unknown +): value is SpeechEnginePreference { + return value === "auto" || value === "browser" || value === "cloud" +} + +/** + * Merge a stored blob over the defaults, field by field. Every field is + * validated independently so a partial write from an older build (or a + * hand-edited value) degrades to the default for that one field instead of + * discarding the whole preference set. + */ +export function parseSpeechPrefs(raw: unknown): SpeechPrefs { + const defaults = DEFAULT_SPEECH_PREFS + if (!raw || typeof raw !== "object") { + return { input: { ...defaults.input } } + } + const source = raw as Record + + const rawInput = source.input + if (!rawInput || typeof rawInput !== "object") { + return { input: { ...defaults.input } } + } + const inputSource = rawInput as Record + + return { + input: { + enabled: + typeof inputSource.enabled === "boolean" + ? inputSource.enabled + : defaults.input.enabled, + engine: isSpeechEnginePreference(inputSource.engine) + ? inputSource.engine + : defaults.input.engine, + language: + typeof inputSource.language === "string" + ? inputSource.language + : defaults.input.language, + }, + } +} + +export function loadSpeechPrefs(): SpeechPrefs { + const defaults = DEFAULT_SPEECH_PREFS + if (typeof window === "undefined") { + return { input: { ...defaults.input } } + } + try { + const raw = localStorage.getItem(PREFS_KEY) + if (!raw) return { input: { ...defaults.input } } + return parseSpeechPrefs(JSON.parse(raw)) + } catch { + return { input: { ...defaults.input } } + } +} + +export function saveSpeechPrefs(prefs: SpeechPrefs): void { + if (typeof window === "undefined") return + try { + localStorage.setItem(PREFS_KEY, JSON.stringify(prefs)) + } catch { + /* ignore */ + } + window.dispatchEvent(new CustomEvent(PREFS_EVENT, { detail: prefs })) +} + +// ── Shared snapshot ── + +let snapshot: SpeechPrefs | null = null +const listeners = new Set<() => void>() +let windowBound = false + +function bindWindow(): void { + if (windowBound || typeof window === "undefined") return + windowBound = true + const invalidate = () => { + snapshot = null + for (const listener of listeners) listener() + } + window.addEventListener(PREFS_EVENT, invalidate) + window.addEventListener("storage", invalidate) +} + +/** + * Current preferences, memoized. Identity only changes when the stored value + * does, so it is safe as a `useSyncExternalStore` snapshot. + */ +export function getSpeechPrefs(): SpeechPrefs { + bindWindow() + if (typeof window === "undefined") return DEFAULT_SPEECH_PREFS + snapshot ??= loadSpeechPrefs() + return snapshot +} + +/** Subscribe to preference changes from this window or any other. */ +export function subscribeSpeechPrefs(onChange: () => void): () => void { + bindWindow() + listeners.add(onChange) + return () => { + listeners.delete(onChange) + } +} + +function getServerSpeechPrefs(): SpeechPrefs { + return DEFAULT_SPEECH_PREFS +} + +/** Reactive read of speech preferences; live across windows. */ +export function useSpeechPrefs(): SpeechPrefs { + return useSyncExternalStore( + subscribeSpeechPrefs, + getSpeechPrefs, + getServerSpeechPrefs + ) +} + +/** Test seam: forget the memoized snapshot so the next read hits storage. */ +export function resetSpeechPrefsCacheForTests(): void { + snapshot = null + windowBound = false + listeners.clear() +} From 4941afdfde9c3d3e3da592a14654aaacc041f184 Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 17:18:32 +0700 Subject: [PATCH 02/15] feat(speech): add OpenAI-compatible cloud transcription backend Refs #844 --- src-tauri/src/commands/mod.rs | 1 + src-tauri/src/commands/speech.rs | 583 +++++++++++++++++++++++++++ src-tauri/src/lib.rs | 4 + src-tauri/src/web/handlers/mod.rs | 1 + src-tauri/src/web/handlers/speech.rs | 53 +++ src-tauri/src/web/router.rs | 13 + src/lib/api.ts | 27 ++ src/lib/types.ts | 11 + 8 files changed, 693 insertions(+) create mode 100644 src-tauri/src/commands/speech.rs create mode 100644 src-tauri/src/web/handlers/speech.rs diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index b1ab84f1e0..21d8332abb 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -55,5 +55,6 @@ pub mod turn_window; pub mod version_control; #[cfg(feature = "tauri-runtime")] pub mod windows; +pub mod speech; pub mod work_task; pub mod workspace_state; diff --git a/src-tauri/src/commands/speech.rs b/src-tauri/src/commands/speech.rs new file mode 100644 index 0000000000..75f0cad4bf --- /dev/null +++ b/src-tauri/src/commands/speech.rs @@ -0,0 +1,583 @@ +use crate::app_error::AppCommandError; +use crate::db::service::app_metadata_service; +#[cfg(feature = "tauri-runtime")] +use crate::db::AppDatabase; +use base64::{engine::general_purpose, Engine as _}; +use reqwest::multipart; +use serde::{Deserialize, Serialize}; +use std::sync::OnceLock; +use std::time::Duration; + +#[cfg(feature = "tauri-runtime")] +use tauri::State; + +const SPEECH_CLOUD_API_KEY: &str = "speech-cloud-api-key"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechCloudSettings { + pub base_url: String, + pub stt_model: String, + pub tts_model: String, + pub tts_voice: String, +} + +impl Default for SpeechCloudSettings { + fn default() -> Self { + Self { + base_url: "https://api.openai.com/v1".to_string(), + stt_model: "whisper-1".to_string(), + tts_model: "tts-1".to_string(), + tts_voice: "alloy".to_string(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechCloudSettingsView { + pub settings: SpeechCloudSettings, + pub api_key_set: bool, +} + +#[cfg(not(test))] +mod store { + pub fn get_secret(key: &str) -> Result, String> { + crate::keyring_store::get_secret(key) + } + + pub fn set_secret(key: &str, value: &str) -> Result<(), String> { + crate::keyring_store::set_secret(key, value) + } + + pub fn delete_secret(key: &str) -> Result<(), String> { + crate::keyring_store::delete_secret(key) + } +} + +#[cfg(test)] +mod store { + use std::collections::HashMap; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Mutex, OnceLock}; + + static STORE: OnceLock>> = OnceLock::new(); + static UNREADABLE: AtomicBool = AtomicBool::new(false); + + pub fn set_unreadable(unreadable: bool) { + UNREADABLE.store(unreadable, Ordering::SeqCst); + } + + fn get_store() -> &'static Mutex> { + STORE.get_or_init(|| Mutex::new(HashMap::new())) + } + + pub fn get_secret(key: &str) -> Result, String> { + if UNREADABLE.load(Ordering::SeqCst) { + return Err("store is unreadable".to_string()); + } + let store = get_store().lock().unwrap(); + Ok(store.get(key).cloned()) + } + + pub fn set_secret(key: &str, value: &str) -> Result<(), String> { + if UNREADABLE.load(Ordering::SeqCst) { + return Err("store is unreadable".to_string()); + } + let mut store = get_store().lock().unwrap(); + store.insert(key.to_string(), value.to_string()); + Ok(()) + } + + pub fn delete_secret(key: &str) -> Result<(), String> { + if UNREADABLE.load(Ordering::SeqCst) { + return Err("store is unreadable".to_string()); + } + let mut store = get_store().lock().unwrap(); + store.remove(key); + Ok(()) + } +} + +pub async fn get_settings_core(conn: &sea_orm::DatabaseConnection) -> SpeechCloudSettings { + match app_metadata_service::get_value(conn, "speech_cloud_settings").await { + Ok(Some(val)) => serde_json::from_str(&val).unwrap_or_default(), + _ => SpeechCloudSettings::default(), + } +} + +pub async fn speech_get_settings_core( + conn: &sea_orm::DatabaseConnection, +) -> Result { + let settings = get_settings_core(conn).await; + let api_key_set = store::get_secret(SPEECH_CLOUD_API_KEY) + .unwrap_or(None) + .is_some(); + Ok(SpeechCloudSettingsView { + settings, + api_key_set, + }) +} + +pub async fn speech_update_settings_core( + conn: &sea_orm::DatabaseConnection, + settings: SpeechCloudSettings, + api_key: Option, +) -> Result { + let mut clean_base_url = settings.base_url.trim().to_string(); + if clean_base_url.ends_with('/') { + clean_base_url.pop(); + } + if !clean_base_url.starts_with("http://") && !clean_base_url.starts_with("https://") { + return Err(AppCommandError::invalid_input( + "base_url must start with http or https", + )); + } + + let clean_settings = SpeechCloudSettings { + base_url: clean_base_url, + ..settings + }; + + let val = serde_json::to_string(&clean_settings).map_err(|e| { + AppCommandError::io_error("Failed to serialize speech settings").with_detail(e.to_string()) + })?; + app_metadata_service::upsert_value(conn, "speech_cloud_settings", &val).await?; + + if let Some(key) = api_key { + if key.is_empty() { + store::delete_secret(SPEECH_CLOUD_API_KEY).map_err(|e| { + AppCommandError::io_error("Failed to delete the speech API key").with_detail(e) + })?; + } else { + store::set_secret(SPEECH_CLOUD_API_KEY, &key).map_err(|e| { + AppCommandError::io_error("Failed to store the speech API key").with_detail(e) + })?; + } + } + + speech_get_settings_core(conn).await +} + +fn get_client() -> &'static reqwest::Client { + static CLIENT: OnceLock = OnceLock::new(); + CLIENT.get_or_init(|| { + reqwest::Client::builder() + .user_agent(format!("codeg/{}", env!("CARGO_PKG_VERSION"))) + .timeout(Duration::from_secs(60)) + .build() + .unwrap() + }) +} + +pub async fn speech_transcribe_core( + conn: &sea_orm::DatabaseConnection, + audio_base64: String, + mime_type: String, + language: Option, +) -> Result { + if audio_base64.is_empty() { + return Err(AppCommandError::invalid_input("Audio data is empty")); + } + if audio_base64.len() > 25 * 1024 * 1024 * 4 / 3 + 1024 { + return Err(AppCommandError::invalid_input("Audio size exceeds 25 MiB")); + } + + let audio_bytes = general_purpose::STANDARD + .decode(&audio_base64) + .map_err(|_| AppCommandError::invalid_input("Invalid base64 audio data"))?; + + if audio_bytes.is_empty() { + return Err(AppCommandError::invalid_input("Audio data is empty")); + } + if audio_bytes.len() > 25 * 1024 * 1024 { + return Err(AppCommandError::invalid_input("Audio size exceeds 25 MiB")); + } + + let ext = match mime_type.as_str() { + "audio/webm" => "webm", + "audio/ogg" => "ogg", + "audio/mp4" => "m4a", + "audio/wav" => "wav", + _ => "webm", + }; + let filename = format!("speech.{}", ext); + + let api_key = store::get_secret(SPEECH_CLOUD_API_KEY) + .map_err(|e| AppCommandError::io_error("Failed to read the speech API key").with_detail(e))? + .ok_or_else(|| AppCommandError::configuration_missing("Speech cloud API key not set"))?; + + let settings = get_settings_core(conn).await; + + let part = multipart::Part::bytes(audio_bytes) + .file_name(filename) + .mime_str(&mime_type) + .map_err(|e| AppCommandError::network(e.to_string()))?; + + let mut form = multipart::Form::new() + .part("file", part) + .text("model", settings.stt_model) + .text("response_format", "json"); + + if let Some(lang) = language { + let subtag = lang.split('-').next().unwrap_or(&lang).to_string(); + form = form.text("language", subtag); + } + + let url = format!("{}/audio/transcriptions", settings.base_url); + + let res = get_client() + .post(&url) + .bearer_auth(api_key) + .multipart(form) + .send() + .await + .map_err(|e| AppCommandError::network(e.to_string()))?; + + let status = res.status(); + if status == reqwest::StatusCode::UNAUTHORIZED || status == reqwest::StatusCode::FORBIDDEN { + return Err(AppCommandError::authentication_failed( + "Speech cloud API authentication failed", + )); + } + + if !status.is_success() { + let body = res + .text() + .await + .unwrap_or_else(|_| "Failed to read response body".to_string()); + let truncated: String = body.chars().take(500).collect(); + return Err(AppCommandError::network(format!( + "API error {}: {}", + status, truncated + ))); + } + + #[derive(Deserialize)] + struct TranscriptionResponse { + text: String, + } + + let json: TranscriptionResponse = res + .json() + .await + .map_err(|e| AppCommandError::network(e.to_string()))?; + + Ok(json.text.trim().to_string()) +} + +#[cfg(feature = "tauri-runtime")] +#[cfg_attr(feature = "tauri-runtime", tauri::command)] +pub async fn speech_get_settings( + db: State<'_, AppDatabase>, +) -> Result { + speech_get_settings_core(&db.conn).await +} + +#[cfg(feature = "tauri-runtime")] +#[cfg_attr(feature = "tauri-runtime", tauri::command)] +pub async fn speech_update_settings( + db: State<'_, AppDatabase>, + settings: SpeechCloudSettings, + api_key: Option, +) -> Result { + speech_update_settings_core(&db.conn, settings, api_key).await +} + +#[cfg(feature = "tauri-runtime")] +#[cfg_attr(feature = "tauri-runtime", tauri::command)] +pub async fn speech_transcribe( + db: State<'_, AppDatabase>, + audio_base64: String, + mime_type: String, + language: Option, +) -> Result { + speech_transcribe_core(&db.conn, audio_base64, mime_type, language).await +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every test that reads or writes the API key goes through the one + /// process-global test store, so they take this lock for their whole run. + static KEY_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + use crate::app_error::AppErrorCode; + use crate::db::test_helpers::fresh_in_memory_db as setup_test_db; + + #[tokio::test] + async fn test_settings_roundtrip() { + let _guard = KEY_LOCK.lock().await; + let db = setup_test_db().await; + let settings = SpeechCloudSettings { + stt_model: "custom-model".to_string(), + ..Default::default() + }; + + let view = + speech_update_settings_core(&db.conn, settings.clone(), Some("test-key".to_string())) + .await + .unwrap(); + assert_eq!(view.settings.stt_model, "custom-model"); + assert!(view.api_key_set); + + let fetched = speech_get_settings_core(&db.conn).await.unwrap(); + assert_eq!(fetched.settings.stt_model, "custom-model"); + assert!(fetched.api_key_set); + } + + #[tokio::test] + async fn test_invalid_base_url() { + let db = setup_test_db().await; + let settings = SpeechCloudSettings { + base_url: "ftp://api.openai.com/v1".to_string(), + ..Default::default() + }; + let res = speech_update_settings_core(&db.conn, settings.clone(), None).await; + assert!(res.is_err()); + assert!(matches!(res.unwrap_err().code, AppErrorCode::InvalidInput)); + } + + #[tokio::test] + async fn test_defaults_on_bad_json() { + let db = setup_test_db().await; + app_metadata_service::upsert_value(&db.conn, "speech_cloud_settings", "invalid json") + .await + .unwrap(); + let settings = get_settings_core(&db.conn).await; + assert_eq!(settings.base_url, "https://api.openai.com/v1"); + } + + #[tokio::test] + async fn test_key_tri_state() { + let _guard = KEY_LOCK.lock().await; + store::set_unreadable(false); + let db = setup_test_db().await; + let settings = SpeechCloudSettings::default(); + + speech_update_settings_core(&db.conn, settings.clone(), Some("secret".to_string())) + .await + .unwrap(); + assert_eq!( + store::get_secret(SPEECH_CLOUD_API_KEY).unwrap(), + Some("secret".to_string()) + ); + + let view = speech_update_settings_core(&db.conn, settings.clone(), None) + .await + .unwrap(); + assert!(view.api_key_set); + assert_eq!( + store::get_secret(SPEECH_CLOUD_API_KEY).unwrap(), + Some("secret".to_string()) + ); + + let view = speech_update_settings_core(&db.conn, settings.clone(), Some(String::new())) + .await + .unwrap(); + assert!(!view.api_key_set); + assert_eq!(store::get_secret(SPEECH_CLOUD_API_KEY).unwrap(), None); + } + + #[tokio::test] + async fn test_unreadable_store_on_save_leaves_key_untouched() { + let _guard = KEY_LOCK.lock().await; + + let db = setup_test_db().await; + let settings = SpeechCloudSettings::default(); + store::set_unreadable(false); + speech_update_settings_core(&db.conn, settings.clone(), Some("initial".to_string())) + .await + .unwrap(); + + store::set_unreadable(true); + let res = + speech_update_settings_core(&db.conn, settings.clone(), Some("new-secret".to_string())) + .await; + assert!(res.is_err()); + + store::set_unreadable(false); + assert_eq!( + store::get_secret(SPEECH_CLOUD_API_KEY).unwrap(), + Some("initial".to_string()) + ); + } + + #[tokio::test] + async fn test_transcribe_empty_audio() { + let db = setup_test_db().await; + let res = + speech_transcribe_core(&db.conn, "".to_string(), "audio/wav".to_string(), None).await; + assert!(res.is_err()); + assert!(matches!(res.unwrap_err().code, AppErrorCode::InvalidInput)); + } + + use axum::extract::Multipart; + use axum::http::{HeaderMap, StatusCode}; + use axum::response::IntoResponse; + use axum::routing::post; + use axum::Router; + use tokio::net::TcpListener; + + async fn mock_transcription_handler( + headers: HeaderMap, + mut multipart: Multipart, + ) -> impl IntoResponse { + if let Some(auth) = headers.get("authorization") { + if auth != "Bearer test-key" { + return (StatusCode::UNAUTHORIZED, "Unauthorized").into_response(); + } + } else { + return (StatusCode::UNAUTHORIZED, "Unauthorized").into_response(); + } + + let mut has_file = false; + let mut model = String::new(); + let mut response_format = String::new(); + let mut language = None; + + while let Some(field) = multipart.next_field().await.unwrap() { + let name = field.name().unwrap().to_string(); + if name == "file" { + let filename = field.file_name().unwrap_or_default().to_string(); + if filename.starts_with("speech.") { + has_file = true; + } + } else if name == "model" { + model = field.text().await.unwrap(); + } else if name == "response_format" { + response_format = field.text().await.unwrap(); + } else if name == "language" { + language = Some(field.text().await.unwrap()); + } + } + + if !has_file || model != "whisper-1" || response_format != "json" { + return (StatusCode::INTERNAL_SERVER_ERROR, "Bad request").into_response(); + } + + if let Some(lang) = language { + if lang == "fr" { + return ( + StatusCode::OK, + axum::Json(serde_json::json!({ "text": "bonjour" })), + ) + .into_response(); + } + } + + ( + StatusCode::OK, + axum::Json(serde_json::json!({ "text": "hello from mock" })), + ) + .into_response() + } + + async fn start_mock_server() -> String { + let app = Router::new().route("/v1/audio/transcriptions", post(mock_transcription_handler)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://{}", addr) + } + + #[tokio::test] + async fn test_transcription_success() { + let _guard = KEY_LOCK.lock().await; + + let db = setup_test_db().await; + let base_url = start_mock_server().await; + + let settings = SpeechCloudSettings { + base_url: format!("{}/v1", base_url), + ..Default::default() + }; + speech_update_settings_core(&db.conn, settings, Some("test-key".to_string())) + .await + .unwrap(); + + let audio_base64 = base64::engine::general_purpose::STANDARD.encode(b"fake audio data"); + + let res = speech_transcribe_core(&db.conn, audio_base64, "audio/wav".to_string(), None) + .await + .unwrap(); + assert_eq!(res, "hello from mock"); + } + + #[tokio::test] + async fn test_transcription_language() { + let _guard = KEY_LOCK.lock().await; + + let db = setup_test_db().await; + let base_url = start_mock_server().await; + + let settings = SpeechCloudSettings { + base_url: format!("{}/v1", base_url), + ..Default::default() + }; + speech_update_settings_core(&db.conn, settings, Some("test-key".to_string())) + .await + .unwrap(); + + let audio_base64 = base64::engine::general_purpose::STANDARD.encode(b"fake audio data"); + + let res = speech_transcribe_core( + &db.conn, + audio_base64, + "audio/wav".to_string(), + Some("fr-CA".to_string()), + ) + .await + .unwrap(); + assert_eq!(res, "bonjour"); + } + + #[tokio::test] + async fn test_transcription_unauthorized() { + let _guard = KEY_LOCK.lock().await; + + let db = setup_test_db().await; + let base_url = start_mock_server().await; + + let settings = SpeechCloudSettings { + base_url: format!("{}/v1", base_url), + ..Default::default() + }; + speech_update_settings_core(&db.conn, settings, Some("wrong-key".to_string())) + .await + .unwrap(); + + let audio_base64 = base64::engine::general_purpose::STANDARD.encode(b"fake audio data"); + + let err = speech_transcribe_core(&db.conn, audio_base64, "audio/wav".to_string(), None) + .await + .unwrap_err(); + assert!(matches!(err.code, AppErrorCode::AuthenticationFailed)); + } + + #[tokio::test] + async fn test_transcription_network_error() { + let _guard = KEY_LOCK.lock().await; + + let db = setup_test_db().await; + let base_url = start_mock_server().await; + + let settings = SpeechCloudSettings { + base_url: format!("{}/v1", base_url), + stt_model: "wrong-model".to_string(), // triggers 500 in mock + ..Default::default() + }; + speech_update_settings_core(&db.conn, settings, Some("test-key".to_string())) + .await + .unwrap(); + + let audio_base64 = base64::engine::general_purpose::STANDARD.encode(b"fake audio data"); + + let err = speech_transcribe_core(&db.conn, audio_base64, "audio/wav".to_string(), None) + .await + .unwrap_err(); + assert!(matches!(err.code, AppErrorCode::NetworkError)); + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 89a412a58d..534a3a3b5c 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -89,6 +89,7 @@ mod tauri_app { clipboard as clipboard_commands, config_sync, conversations, + speech, custom_skills as custom_skills_commands, deepseek_settings as deepseek_settings_commands, delegation as delegation_commands, experts as experts_commands, feedback as feedback_commands, file_io, folder_commands, @@ -1975,6 +1976,9 @@ mod tauri_app { config_sync::config_sync_peek_file, config_sync::config_sync_import_file, config_sync::config_sync_get_settings, + speech::speech_get_settings, + speech::speech_update_settings, + speech::speech_transcribe, config_sync::config_sync_update_settings, config_sync::config_sync_get_state, config_sync::config_sync_test_connection, diff --git a/src-tauri/src/web/handlers/mod.rs b/src-tauri/src/web/handlers/mod.rs index c6d95935a7..b8b3eebeaa 100644 --- a/src-tauri/src/web/handlers/mod.rs +++ b/src-tauri/src/web/handlers/mod.rs @@ -39,6 +39,7 @@ pub mod terminal; pub mod token_usage; mod upload_jail; pub mod version_control; +pub mod speech; pub mod web_server; pub mod work_task; pub mod workspace_files; diff --git a/src-tauri/src/web/handlers/speech.rs b/src-tauri/src/web/handlers/speech.rs new file mode 100644 index 0000000000..5c37344def --- /dev/null +++ b/src-tauri/src/web/handlers/speech.rs @@ -0,0 +1,53 @@ +use crate::app_error::AppCommandError; +use crate::app_state::AppState; +use crate::commands::speech::{ + speech_get_settings_core, speech_transcribe_core, speech_update_settings_core, + SpeechCloudSettings, SpeechCloudSettingsView, +}; +use axum::{extract::Extension, Json}; +use serde::Deserialize; +use std::sync::Arc; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UpdateSettingsParams { + pub settings: SpeechCloudSettings, + pub api_key: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TranscribeParams { + pub audio_base64: String, + pub mime_type: String, + pub language: Option, +} + +pub async fn speech_get_settings( + Extension(state): Extension>, +) -> Result, AppCommandError> { + let view = speech_get_settings_core(&state.db.conn).await?; + Ok(Json(view)) +} + +pub async fn speech_update_settings( + Extension(state): Extension>, + Json(params): Json, +) -> Result, AppCommandError> { + let view = speech_update_settings_core(&state.db.conn, params.settings, params.api_key).await?; + Ok(Json(view)) +} + +pub async fn speech_transcribe( + Extension(state): Extension>, + Json(params): Json, +) -> Result, AppCommandError> { + let text = speech_transcribe_core( + &state.db.conn, + params.audio_base64, + params.mime_type, + params.language, + ) + .await?; + Ok(Json(text)) +} diff --git a/src-tauri/src/web/router.rs b/src-tauri/src/web/router.rs index bdcd82a8a2..0409f640f9 100644 --- a/src-tauri/src/web/router.rs +++ b/src-tauri/src/web/router.rs @@ -603,6 +603,19 @@ pub fn build_router( // by-content pair: a browser has no path to name, and the payload is // tens of KB, so it travels in the JSON body rather than through the // upload-staging machinery above. + .route( + "/speech_get_settings", + post(handlers::speech::speech_get_settings), + ) + .route( + "/speech_update_settings", + post(handlers::speech::speech_update_settings), + ) + .route( + "/speech_transcribe", + post(handlers::speech::speech_transcribe) + .layer(DefaultBodyLimit::max(36 * 1024 * 1024)), + ) .route( "/config_sync_get_settings", post(handlers::config_sync::config_sync_get_settings), diff --git a/src/lib/api.ts b/src/lib/api.ts index c0afe26d54..375e1d0736 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -28,6 +28,8 @@ import type { AutomationDraft, DeepSeekCatalogModel, DeepSeekModelCatalog, + SpeechCloudSettings, + SpeechCloudSettingsView, ForgeChangeDetail, ForgeChangedFileList, ForgeComment, @@ -5960,3 +5962,28 @@ export async function forgeSettingsSet( ): Promise { return getTransport().call("forge_settings_set", { folderId, settings }) } +export async function speechGetSettings(): Promise { + return getTransport().call("speech_get_settings", {}) +} + +export async function speechUpdateSettings( + settings: SpeechCloudSettings, + apiKey: string | null +): Promise { + return getTransport().call( + "speech_update_settings", + { settings, apiKey } + ) +} + +export async function speechTranscribe( + audioBase64: string, + mimeType: string, + language: string | null +): Promise { + return getTransport().call("speech_transcribe", { + audioBase64, + mimeType, + language, + }) +} diff --git a/src/lib/types.ts b/src/lib/types.ts index b7c77f4712..2e41708c2c 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -5074,3 +5074,14 @@ export interface DeepSeekModelCatalog { * fixed, sessions run on the agent's built-in catalog instead. */ invalid: string | null } +export interface SpeechCloudSettings { + baseUrl: string + sttModel: string + ttsModel: string + ttsVoice: string +} + +export interface SpeechCloudSettingsView { + settings: SpeechCloudSettings + apiKeySet: boolean +} From b48dade4d47d79fadbaf1e99e2352d90a4ae5abe Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 17:18:32 +0700 Subject: [PATCH 03/15] feat(desktop): allow microphone capture in app windows Refs #844 --- src-tauri/Entitlements.plist | 8 +++++++ src-tauri/Info.plist | 10 +++++++++ src-tauri/src/commands/windows.rs | 35 +++++++++++++++++++++++++++++++ src-tauri/tauri.conf.json | 3 +++ 4 files changed, 56 insertions(+) create mode 100644 src-tauri/Entitlements.plist create mode 100644 src-tauri/Info.plist diff --git a/src-tauri/Entitlements.plist b/src-tauri/Entitlements.plist new file mode 100644 index 0000000000..b572d9c04e --- /dev/null +++ b/src-tauri/Entitlements.plist @@ -0,0 +1,8 @@ + + + + + com.apple.security.device.audio-input + + + diff --git a/src-tauri/Info.plist b/src-tauri/Info.plist new file mode 100644 index 0000000000..87757c9a9c --- /dev/null +++ b/src-tauri/Info.plist @@ -0,0 +1,10 @@ + + + + + NSMicrophoneUsageDescription + Codeg uses the microphone to turn your speech into chat prompts. + NSSpeechRecognitionUsageDescription + Codeg uses speech recognition to turn your speech into chat prompts. + + diff --git a/src-tauri/src/commands/windows.rs b/src-tauri/src/commands/windows.rs index e642f2f7a9..2aeccba380 100644 --- a/src-tauri/src/commands/windows.rs +++ b/src-tauri/src/commands/windows.rs @@ -283,6 +283,41 @@ fn ensure_windows_undecorated(_window: &tauri::WebviewWindow) {} /// Apply platform-specific post-creation setup. pub(crate) fn post_window_setup(window: &tauri::WebviewWindow) { ensure_windows_undecorated(window); + #[cfg(target_os = "linux")] + enable_linux_audio_capture(window); +} + +/// WebKitGTK ships with media capture off and denies every permission request +/// nobody answers, so `getUserMedia({ audio: true })` fails in app windows until +/// both are enabled here. Only microphone requests are granted; camera and any +/// other request keep WebKit's default handling. +#[cfg(target_os = "linux")] +fn enable_linux_audio_capture(window: &tauri::WebviewWindow) { + use gtk::prelude::*; + use webkit2gtk::{ + PermissionRequestExt, SettingsExt, UserMediaPermissionRequest, + UserMediaPermissionRequestExt, WebViewExt, + }; + + let result = window.with_webview(|platform| { + let webview = platform.inner(); + if let Some(settings) = WebViewExt::settings(&webview) { + settings.set_enable_media_stream(true); + settings.set_enable_mediasource(true); + } + webview.connect_permission_request(|_, request| { + match request.downcast_ref::() { + Some(media) if media.is_for_audio_device() && !media.is_for_video_device() => { + request.allow(); + true + } + _ => false, + } + }); + }); + if let Err(err) = result { + tracing::warn!("[windows] microphone capture not enabled: {err}"); + } } impl SettingsWindowState { diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 010d418986..936b52cbf7 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -31,6 +31,9 @@ "../out": "web/" }, "externalBin": ["binaries/codeg-mcp"], + "macOS": { + "entitlements": "./Entitlements.plist" + }, "windows": { "nsis": { "installerHooks": "./windows/installer-hooks.nsh" From e13a7b489a8d225f2946d4b5f9d769cea48a2a15 Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 17:18:32 +0700 Subject: [PATCH 04/15] feat(chat): add speech input hook with browser and cloud engines Refs #844 --- .../chat/composer/use-speech-input.test.ts | 323 ++++++++++++ .../chat/composer/use-speech-input.ts | 476 ++++++++++++++++++ 2 files changed, 799 insertions(+) create mode 100644 src/components/chat/composer/use-speech-input.test.ts create mode 100644 src/components/chat/composer/use-speech-input.ts diff --git a/src/components/chat/composer/use-speech-input.test.ts b/src/components/chat/composer/use-speech-input.test.ts new file mode 100644 index 0000000000..d19c528b02 --- /dev/null +++ b/src/components/chat/composer/use-speech-input.test.ts @@ -0,0 +1,323 @@ +import { act, renderHook, waitFor } from "@testing-library/react" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" + +import type { SpeechCapabilities } from "@/lib/speech-capabilities" +import type { SpeechPrefs } from "@/lib/speech-prefs" + +vi.mock("next-intl", () => ({ useLocale: () => "en" })) + +vi.mock("@/lib/api", () => ({ + speechGetSettings: vi.fn(), + speechTranscribe: vi.fn(), +})) + +let prefs: SpeechPrefs = { + input: { enabled: true, engine: "auto", language: "" }, +} +vi.mock("@/lib/speech-prefs", () => ({ useSpeechPrefs: () => prefs })) + +let caps: SpeechCapabilities = { + browserStt: true, + mediaCapture: true, + secureContext: true, +} +vi.mock("@/lib/speech-capabilities", async (importOriginal) => ({ + ...(await importOriginal()), + detectSpeechCapabilities: () => caps, +})) + +import { speechGetSettings, speechTranscribe } from "@/lib/api" +import { MAX_RECORDING_MS, useSpeechInput } from "./use-speech-input" + +const mockGetSettings = vi.mocked(speechGetSettings) +const mockTranscribe = vi.mocked(speechTranscribe) + +type ResultInit = { transcript: string; isFinal: boolean } + +class FakeRecognition { + static instances: FakeRecognition[] = [] + continuous = false + interimResults = false + lang = "" + onresult: ((event: unknown) => void) | null = null + onerror: ((event: { error: string }) => void) | null = null + onend: (() => void) | null = null + start = vi.fn() + stop = vi.fn(() => this.onend?.()) + abort = vi.fn() + + constructor() { + FakeRecognition.instances.push(this) + } + + emit(results: ResultInit[], resultIndex = 0) { + this.onresult?.({ + resultIndex, + results: results.map((r) => + Object.assign([{ transcript: r.transcript }], { isFinal: r.isFinal }) + ), + }) + } +} + +class FakeTrack { + stop = vi.fn() +} + +class FakeRecorder { + static instances: FakeRecorder[] = [] + static isTypeSupported = vi.fn((type: string) => type.startsWith("audio/ogg")) + state: "inactive" | "recording" = "inactive" + mimeType: string + ondataavailable: ((event: { data: Blob }) => void) | null = null + onstop: (() => void) | null = null + + constructor( + public stream: { getTracks: () => FakeTrack[] }, + options?: { mimeType?: string } + ) { + this.mimeType = options?.mimeType ?? "" + FakeRecorder.instances.push(this) + } + + start() { + this.state = "recording" + } + + stop() { + this.state = "inactive" + this.ondataavailable?.({ data: new Blob(["voice"], { type: "audio/ogg" }) }) + this.onstop?.() + } +} + +let tracks: FakeTrack[] = [] +const getUserMedia = vi.fn() + +function lastRecognition() { + return FakeRecognition.instances[FakeRecognition.instances.length - 1] +} + +function lastRecorder() { + return FakeRecorder.instances[FakeRecorder.instances.length - 1] +} + +function renderSpeech() { + const onFinalText = vi.fn() + const onError = vi.fn() + const hook = renderHook(() => useSpeechInput({ onFinalText, onError })) + return { ...hook, onFinalText, onError } +} + +beforeEach(() => { + prefs = { input: { enabled: true, engine: "auto", language: "" } } + caps = { browserStt: true, mediaCapture: true, secureContext: true } + FakeRecognition.instances = [] + FakeRecorder.instances = [] + tracks = [new FakeTrack()] + getUserMedia.mockReset() + getUserMedia.mockImplementation(async () => ({ getTracks: () => tracks })) + mockGetSettings.mockReset() + mockGetSettings.mockResolvedValue({ + settings: { + baseUrl: "https://api.openai.com/v1", + sttModel: "whisper-1", + ttsModel: "tts-1", + ttsVoice: "alloy", + }, + apiKeySet: true, + }) + mockTranscribe.mockReset() + vi.stubGlobal("SpeechRecognition", FakeRecognition) + vi.stubGlobal("MediaRecorder", FakeRecorder) + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: { getUserMedia }, + }) +}) + +afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +describe("useSpeechInput - browser engine", () => { + it("streams interim text and hands each final result to onFinalText", async () => { + const { result, onFinalText } = renderSpeech() + await waitFor(() => expect(result.current.status).toBe("idle")) + + act(() => result.current.start()) + await waitFor(() => expect(result.current.status).toBe("listening")) + + const recognition = lastRecognition() + expect(recognition.continuous).toBe(true) + expect(recognition.interimResults).toBe(true) + expect(recognition.lang).toBe("en-US") + + act(() => recognition.emit([{ transcript: "hello wor", isFinal: false }])) + expect(result.current.interimText).toBe("hello wor") + + act(() => + recognition.emit([{ transcript: " hello world ", isFinal: true }]) + ) + expect(onFinalText).toHaveBeenCalledWith("hello world") + expect(result.current.interimText).toBe("") + + act(() => result.current.stop()) + expect(recognition.stop).toHaveBeenCalled() + expect(result.current.status).toBe("idle") + }) + + it("reports mic-denied and releases the recognizer", async () => { + const { result, onError } = renderSpeech() + act(() => result.current.start()) + await waitFor(() => expect(result.current.status).toBe("listening")) + + const recognition = lastRecognition() + act(() => recognition.onerror?.({ error: "not-allowed" })) + + expect(onError).toHaveBeenCalledWith("mic-denied") + expect(recognition.abort).toHaveBeenCalled() + expect(result.current.status).toBe("idle") + }) + + it("maps a network failure to engine-failed", async () => { + const { result, onError } = renderSpeech() + act(() => result.current.start()) + await waitFor(() => expect(result.current.status).toBe("listening")) + + act(() => lastRecognition().onerror?.({ error: "network" })) + expect(onError).toHaveBeenCalledWith("engine-failed") + }) + + it("returns to idle when the engine ends on its own", async () => { + const { result } = renderSpeech() + act(() => result.current.start()) + await waitFor(() => expect(result.current.status).toBe("listening")) + + act(() => lastRecognition().onend?.()) + expect(result.current.status).toBe("idle") + }) +}) + +describe("useSpeechInput - cloud engine", () => { + beforeEach(() => { + prefs = { input: { enabled: true, engine: "cloud", language: "de-DE" } } + }) + + it("records, transcribes with the bare mime type and language, then inserts", async () => { + mockTranscribe.mockResolvedValue(" hallo welt ") + const { result, onFinalText } = renderSpeech() + + act(() => result.current.start()) + await waitFor(() => expect(result.current.status).toBe("listening")) + expect(getUserMedia).toHaveBeenCalledWith({ audio: true }) + expect(lastRecorder().mimeType).toBe("audio/ogg;codecs=opus") + + act(() => result.current.stop()) + await waitFor(() => expect(onFinalText).toHaveBeenCalledWith("hallo welt")) + + const [audio, mimeType, language] = mockTranscribe.mock.calls[0] + expect(atob(audio)).toBe("voice") + expect(mimeType).toBe("audio/ogg") + expect(language).toBe("de-DE") + expect(tracks[0].stop).toHaveBeenCalled() + expect(result.current.status).toBe("idle") + }) + + it("cancel discards the recording, sends nothing and stops the tracks", async () => { + const { result, onFinalText } = renderSpeech() + act(() => result.current.start()) + await waitFor(() => expect(result.current.status).toBe("listening")) + + act(() => result.current.cancel()) + + expect(mockTranscribe).not.toHaveBeenCalled() + expect(onFinalText).not.toHaveBeenCalled() + expect(tracks[0].stop).toHaveBeenCalled() + expect(result.current.status).toBe("idle") + }) + + it("stops the tracks when unmounted mid-recording", async () => { + const { result, unmount } = renderSpeech() + act(() => result.current.start()) + await waitFor(() => expect(result.current.status).toBe("listening")) + + unmount() + + expect(tracks[0].stop).toHaveBeenCalled() + expect(mockTranscribe).not.toHaveBeenCalled() + }) + + it("maps authentication_failed to cloud-auth", async () => { + mockTranscribe.mockRejectedValue({ + code: "authentication_failed", + message: "Unauthorized", + }) + const { result, onError, onFinalText } = renderSpeech() + act(() => result.current.start()) + await waitFor(() => expect(result.current.status).toBe("listening")) + + act(() => result.current.stop()) + await waitFor(() => expect(onError).toHaveBeenCalledWith("cloud-auth")) + expect(onFinalText).not.toHaveBeenCalled() + expect(result.current.status).toBe("idle") + }) + + it("reports mic-denied when getUserMedia is refused", async () => { + getUserMedia.mockRejectedValue( + Object.assign(new Error("denied"), { name: "NotAllowedError" }) + ) + const { result, onError } = renderSpeech() + act(() => result.current.start()) + + await waitFor(() => expect(onError).toHaveBeenCalledWith("mic-denied")) + expect(result.current.status).toBe("idle") + }) + + it("stops recording on its own at the time cap", async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + mockTranscribe.mockResolvedValue("long take") + const { result, onFinalText } = renderSpeech() + act(() => result.current.start()) + await waitFor(() => expect(result.current.status).toBe("listening")) + + act(() => { + vi.advanceTimersByTime(MAX_RECORDING_MS) + }) + + await waitFor(() => expect(onFinalText).toHaveBeenCalledWith("long take")) + expect(tracks[0].stop).toHaveBeenCalled() + }) +}) + +describe("useSpeechInput - availability", () => { + it("is unavailable with a reason when no engine can run", async () => { + caps = { browserStt: false, mediaCapture: false, secureContext: false } + const { result } = renderSpeech() + + await waitFor(() => expect(result.current.status).toBe("unavailable")) + expect(result.current.unavailableReason).toBe("insecure-context") + }) + + it("reports cloud-not-configured on start when the key is missing", async () => { + caps = { browserStt: false, mediaCapture: true, secureContext: true } + mockGetSettings.mockResolvedValue({ + settings: { + baseUrl: "https://api.openai.com/v1", + sttModel: "whisper-1", + ttsModel: "tts-1", + ttsVoice: "alloy", + }, + apiKeySet: false, + }) + const { result, onError } = renderSpeech() + await waitFor(() => expect(result.current.status).toBe("unavailable")) + + act(() => result.current.start()) + await waitFor(() => + expect(onError).toHaveBeenCalledWith("cloud-not-configured") + ) + expect(getUserMedia).not.toHaveBeenCalled() + }) +}) diff --git a/src/components/chat/composer/use-speech-input.ts b/src/components/chat/composer/use-speech-input.ts new file mode 100644 index 0000000000..b4df60a275 --- /dev/null +++ b/src/components/chat/composer/use-speech-input.ts @@ -0,0 +1,476 @@ +"use client" + +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + useSyncExternalStore, +} from "react" +import { useLocale } from "next-intl" + +import { speechGetSettings, speechTranscribe } from "@/lib/api" +import { extractAppCommandError } from "@/lib/app-error" +import { + detectSpeechCapabilities, + resolveInputEngine, + resolveSpeechLanguage, + type InputEngineResolution, + type SpeechCapabilities, +} from "@/lib/speech-capabilities" +import { useSpeechPrefs } from "@/lib/speech-prefs" + +export type SpeechInputStatus = + | "idle" + | "listening" + | "transcribing" + | "unavailable" + +export type SpeechInputError = + | "mic-denied" + | "engine-failed" + | "cloud-auth" + | "cloud-not-configured" + +export type SpeechUnavailableReason = Extract< + InputEngineResolution, + { engine: null } +>["reason"] + +export interface UseSpeechInputOptions { + onFinalText: (text: string) => void + onError: (error: SpeechInputError) => void +} + +export interface UseSpeechInputResult { + status: SpeechInputStatus + interimText: string + unavailableReason: SpeechUnavailableReason | null + start: () => void + stop: () => void + cancel: () => void + toggle: () => void +} + +export const MAX_RECORDING_MS = 120_000 + +const RECORDER_MIME_TYPES = [ + "audio/webm;codecs=opus", + "audio/ogg;codecs=opus", + "audio/mp4", +] + +// The DOM lib shipped with TypeScript has no Web Speech API types. +interface RecognitionAlternativeLike { + transcript: string +} +interface RecognitionResultLike { + readonly isFinal: boolean + readonly length: number + readonly [index: number]: RecognitionAlternativeLike +} +interface RecognitionEventLike { + resultIndex: number + results: ArrayLike +} +interface RecognitionLike { + continuous: boolean + interimResults: boolean + lang: string + onresult: ((event: RecognitionEventLike) => void) | null + onerror: ((event: { error: string }) => void) | null + onend: (() => void) | null + start(): void + stop(): void + abort(): void +} +type RecognitionCtor = new () => RecognitionLike + +const subscribeNever = () => () => {} +const onClient = () => true +const onServer = () => false + +type Session = + | { kind: "browser"; recognition: RecognitionLike } + | { + kind: "cloud" + stream: MediaStream + recorder: MediaRecorder + chunks: Blob[] + mimeType: string + language: string + timer: ReturnType + } + +function recognitionCtor(): RecognitionCtor | null { + if (typeof window === "undefined") return null + const win = window as unknown as Record + const ctor = win.SpeechRecognition ?? win.webkitSpeechRecognition + return typeof ctor === "function" ? (ctor as RecognitionCtor) : null +} + +function pickRecorderMimeType(): string | undefined { + const isTypeSupported = MediaRecorder.isTypeSupported + if (typeof isTypeSupported !== "function") return undefined + return RECORDER_MIME_TYPES.find((type) => + isTypeSupported.call(MediaRecorder, type) + ) +} + +function blobToBase64(blob: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => { + const dataUrl = String(reader.result ?? "") + resolve(dataUrl.slice(dataUrl.indexOf(",") + 1)) + } + reader.onerror = () => reject(reader.error) + reader.readAsDataURL(blob) + }) +} + +function releaseSession(session: Session) { + if (session.kind === "browser") { + const { recognition } = session + recognition.onresult = null + recognition.onerror = null + recognition.onend = null + recognition.abort() + return + } + clearTimeout(session.timer) + session.recorder.ondataavailable = null + session.recorder.onstop = null + if (session.recorder.state !== "inactive") session.recorder.stop() + session.stream.getTracks().forEach((track) => track.stop()) +} + +function cloudErrorFromException(error: unknown): SpeechInputError { + switch (extractAppCommandError(error)?.code) { + case "authentication_failed": + return "cloud-auth" + case "configuration_missing": + return "cloud-not-configured" + default: + return "engine-failed" + } +} + +function micErrorFromException(error: unknown): SpeechInputError { + const name = error instanceof Error ? error.name : "" + return name === "NotAllowedError" || name === "SecurityError" + ? "mic-denied" + : "engine-failed" +} + +export function useSpeechInput({ + onFinalText, + onError, +}: UseSpeechInputOptions): UseSpeechInputResult { + const prefs = useSpeechPrefs() + const locale = useLocale() + const language = resolveSpeechLanguage(prefs.input, locale) + + const [activeStatus, setActiveStatus] = useState< + "idle" | "listening" | "transcribing" + >("idle") + const [interimText, setInterimText] = useState("") + // Both are read from the browser, so they stay unknown until mount to keep + // the prerendered markup identical to the first client render. + const mounted = useSyncExternalStore(subscribeNever, onClient, onServer) + const caps = useMemo( + () => (mounted ? detectSpeechCapabilities() : null), + [mounted] + ) + const [apiKeySet, setApiKeySet] = useState(null) + + const sessionRef = useRef(null) + // Bumped by every start, cancel and unmount; an async step that finds a + // different value belongs to a session that no longer exists. + const generationRef = useRef(0) + const busyRef = useRef(false) + const callbacksRef = useRef({ onFinalText, onError }) + const contextRef = useRef({ prefs, language, apiKeySet }) + + useEffect(() => { + callbacksRef.current = { onFinalText, onError } + }, [onFinalText, onError]) + + useEffect(() => { + contextRef.current = { prefs, language, apiKeySet } + }, [prefs, language, apiKeySet]) + + const refreshSettings = useCallback(async (): Promise => { + try { + const view = await speechGetSettings() + setApiKeySet(view.apiKeySet) + return view.apiKeySet + } catch { + setApiKeySet(false) + return false + } + }, []) + + useEffect(() => { + let alive = true + speechGetSettings().then( + (view) => { + if (alive) setApiKeySet(view.apiKeySet) + }, + () => { + if (alive) setApiKeySet(false) + } + ) + return () => { + alive = false + } + }, []) + + // A key saved in the (separate) settings window is noticed on refocus. + useEffect(() => { + if (apiKeySet !== false) return + const onFocus = () => void refreshSettings() + window.addEventListener("focus", onFocus) + return () => window.removeEventListener("focus", onFocus) + }, [apiKeySet, refreshSettings]) + + const finish = useCallback((generation: number) => { + if (generation !== generationRef.current) return + sessionRef.current = null + busyRef.current = false + setInterimText("") + setActiveStatus("idle") + }, []) + + const fail = useCallback( + (generation: number, error: SpeechInputError) => { + if (generation !== generationRef.current) return + const session = sessionRef.current + if (session) releaseSession(session) + finish(generation) + callbacksRef.current.onError(error) + }, + [finish] + ) + + const startBrowser = useCallback( + (generation: number, lang: string) => { + const Ctor = recognitionCtor() + if (!Ctor) { + fail(generation, "engine-failed") + return + } + const recognition = new Ctor() + recognition.continuous = true + recognition.interimResults = true + recognition.lang = lang + recognition.onresult = (event) => { + if (generation !== generationRef.current) return + let interim = "" + for (let i = event.resultIndex; i < event.results.length; i += 1) { + const result = event.results[i] + const transcript = result[0]?.transcript ?? "" + if (result.isFinal) { + const text = transcript.trim() + if (text) callbacksRef.current.onFinalText(text) + } else { + interim += transcript + } + } + setInterimText(interim.trim()) + } + recognition.onerror = (event) => { + if (event.error === "aborted" || event.error === "no-speech") return + fail( + generation, + event.error === "not-allowed" || event.error === "service-not-allowed" + ? "mic-denied" + : "engine-failed" + ) + } + recognition.onend = () => finish(generation) + sessionRef.current = { kind: "browser", recognition } + try { + recognition.start() + } catch { + fail(generation, "engine-failed") + return + } + setActiveStatus("listening") + }, + [fail, finish] + ) + + const transcribe = useCallback( + async (generation: number, blob: Blob, mimeType: string, lang: string) => { + try { + const audio = await blobToBase64(blob) + if (generation !== generationRef.current) return + const text = (await speechTranscribe(audio, mimeType, lang)).trim() + if (generation !== generationRef.current) return + if (text) callbacksRef.current.onFinalText(text) + finish(generation) + } catch (error) { + fail(generation, cloudErrorFromException(error)) + } + }, + [fail, finish] + ) + + const stopCloud = useCallback( + (generation: number) => { + const session = sessionRef.current + if (generation !== generationRef.current || session?.kind !== "cloud") { + return + } + if (session.recorder.state === "inactive") return + clearTimeout(session.timer) + session.recorder.onstop = () => { + session.stream.getTracks().forEach((track) => track.stop()) + const blob = new Blob(session.chunks, { type: session.mimeType }) + if (blob.size === 0) { + finish(generation) + return + } + void transcribe(generation, blob, session.mimeType, session.language) + } + session.recorder.stop() + setInterimText("") + setActiveStatus("transcribing") + }, + [finish, transcribe] + ) + + const startCloud = useCallback( + async (generation: number, lang: string) => { + let stream: MediaStream + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }) + } catch (error) { + fail(generation, micErrorFromException(error)) + return + } + if (generation !== generationRef.current) { + stream.getTracks().forEach((track) => track.stop()) + return + } + let recorder: MediaRecorder + try { + const preferred = pickRecorderMimeType() + recorder = preferred + ? new MediaRecorder(stream, { mimeType: preferred }) + : new MediaRecorder(stream) + } catch { + stream.getTracks().forEach((track) => track.stop()) + fail(generation, "engine-failed") + return + } + // The backend keys the upload's file extension off the bare type. + const mimeType = (recorder.mimeType || "audio/webm").split(";")[0].trim() + const chunks: Blob[] = [] + recorder.ondataavailable = (event) => { + if (event.data.size > 0) chunks.push(event.data) + } + const timer = setTimeout(() => stopCloud(generation), MAX_RECORDING_MS) + sessionRef.current = { + kind: "cloud", + stream, + recorder, + chunks, + mimeType, + language: lang, + timer, + } + recorder.start() + setActiveStatus("listening") + }, + [fail, stopCloud] + ) + + const start = useCallback(() => { + if (busyRef.current) return + busyRef.current = true + generationRef.current += 1 + const generation = generationRef.current + const { + prefs: current, + language: lang, + apiKeySet: known, + } = contextRef.current + + void (async () => { + const configured = known === true ? true : await refreshSettings() + if (generation !== generationRef.current) return + const resolution = resolveInputEngine( + current.input, + detectSpeechCapabilities(), + configured + ) + if (resolution.engine === null) { + finish(generation) + if (resolution.reason === "cloud-not-configured") { + callbacksRef.current.onError("cloud-not-configured") + } + } else if (resolution.engine === "browser") { + startBrowser(generation, lang) + } else { + await startCloud(generation, lang) + } + })() + }, [finish, refreshSettings, startBrowser, startCloud]) + + const stop = useCallback(() => { + const session = sessionRef.current + if (!session) return + if (session.kind === "browser") { + // `stop` (not `abort`) lets the engine deliver the last final result; + // `onend` then releases the session. + session.recognition.stop() + return + } + stopCloud(generationRef.current) + }, [stopCloud]) + + const cancel = useCallback(() => { + const generation = generationRef.current + const session = sessionRef.current + if (session) releaseSession(session) + finish(generation) + generationRef.current += 1 + }, [finish]) + + const toggle = useCallback(() => { + if (busyRef.current) stop() + else start() + }, [start, stop]) + + useEffect( + () => () => { + generationRef.current += 1 + const session = sessionRef.current + sessionRef.current = null + if (session) releaseSession(session) + }, + [] + ) + + const unavailableReason = useMemo(() => { + if (!caps || apiKeySet === null) return null + const resolution = resolveInputEngine(prefs.input, caps, apiKeySet) + return resolution.engine === null ? resolution.reason : null + }, [apiKeySet, caps, prefs.input]) + + const status: SpeechInputStatus = + activeStatus === "idle" && unavailableReason ? "unavailable" : activeStatus + + return { + status, + interimText, + unavailableReason, + start, + stop, + cancel, + toggle, + } +} From 59c2ef8e3de66e916ab628f87f88e3ba13d75f44 Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 17:18:32 +0700 Subject: [PATCH 05/15] feat(chat): add voice dictation button to the composer Refs #844 --- .../composer/composer-speech-button.test.tsx | 70 +++++++ .../chat/composer/composer-speech-button.tsx | 183 ++++++++++++++++++ src/components/chat/message-input.test.tsx | 76 ++++++++ src/components/chat/message-input.tsx | 19 +- src/i18n/messages/ar.json | 17 +- src/i18n/messages/de.json | 17 +- src/i18n/messages/en.json | 17 +- src/i18n/messages/es.json | 17 +- src/i18n/messages/fr.json | 17 +- src/i18n/messages/ja.json | 17 +- src/i18n/messages/ko.json | 17 +- src/i18n/messages/pt.json | 17 +- src/i18n/messages/zh-CN.json | 17 +- src/i18n/messages/zh-TW.json | 17 +- src/lib/keyboard-shortcuts.ts | 5 + 15 files changed, 512 insertions(+), 11 deletions(-) create mode 100644 src/components/chat/composer/composer-speech-button.test.tsx create mode 100644 src/components/chat/composer/composer-speech-button.tsx diff --git a/src/components/chat/composer/composer-speech-button.test.tsx b/src/components/chat/composer/composer-speech-button.test.tsx new file mode 100644 index 0000000000..41c2d4ce03 --- /dev/null +++ b/src/components/chat/composer/composer-speech-button.test.tsx @@ -0,0 +1,70 @@ +import { cleanup, render, screen } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { NextIntlClientProvider } from "next-intl" +import { afterEach, describe, expect, it, vi } from "vitest" + +import enMessages from "@/i18n/messages/en.json" +import { ComposerSpeechButton } from "./composer-speech-button" + +const m = enMessages.Folder.chat.messageInput + +function renderButton( + props: Partial> = {} +) { + const onToggle = vi.fn() + render( + + + + ) + return { onToggle } +} + +afterEach(() => cleanup()) + +describe("ComposerSpeechButton", () => { + it("idle: labelled start and toggles on click", async () => { + const { onToggle } = renderButton() + const button = screen.getByRole("button", { name: m.speechStart }) + expect(button).toBeEnabled() + await userEvent.click(button) + expect(onToggle).toHaveBeenCalledTimes(1) + }) + + it("listening: labelled stop, pressed, shows interim text and the Esc hint", async () => { + const { onToggle } = renderButton({ + status: "listening", + interimText: "hello wor", + }) + const button = screen.getByRole("button", { name: m.speechStop }) + expect(button).toHaveAttribute("aria-pressed", "true") + expect(screen.getByRole("status")).toHaveTextContent("hello wor") + expect(screen.getByRole("status")).toHaveTextContent(m.speechCancelHint) + await userEvent.click(button) + expect(onToggle).toHaveBeenCalledTimes(1) + }) + + it("transcribing: labelled and disabled", () => { + renderButton({ status: "transcribing" }) + expect( + screen.getByRole("button", { name: m.speechTranscribing }) + ).toBeDisabled() + expect(screen.queryByRole("status")).not.toBeInTheDocument() + }) + + it.each([ + ["no-engine", m.speechUnavailableNoEngine], + ["insecure-context", m.speechUnavailableInsecure], + ["no-mic", m.speechUnavailableNoMic], + ["cloud-not-configured", m.speechUnavailableCloud], + ] as const)("unavailable (%s): disabled with the reason", (reason, label) => { + renderButton({ status: "unavailable", unavailableReason: reason }) + expect(screen.getByRole("button", { name: label })).toBeDisabled() + }) +}) diff --git a/src/components/chat/composer/composer-speech-button.tsx b/src/components/chat/composer/composer-speech-button.tsx new file mode 100644 index 0000000000..a724c67973 --- /dev/null +++ b/src/components/chat/composer/composer-speech-button.tsx @@ -0,0 +1,183 @@ +"use client" + +import { useCallback, useEffect, type RefObject } from "react" +import { Loader2, Mic, MicOff, Square } from "lucide-react" +import { useTranslations } from "next-intl" +import { toast } from "sonner" + +import { textToInlineContent } from "@/components/chat/composer/plain-text-content" +import type { RichComposerHandle } from "@/components/chat/composer/rich-composer" +import { + useSpeechInput, + type SpeechInputError, + type SpeechInputStatus, + type SpeechUnavailableReason, +} from "@/components/chat/composer/use-speech-input" +import { Button } from "@/components/ui/button" +import { isImeCompositionKey } from "@/lib/ime-composition" +import { matchShortcutEvent } from "@/lib/keyboard-shortcuts" +import { cn } from "@/lib/utils" + +const UNAVAILABLE_KEYS = { + "no-engine": "speechUnavailableNoEngine", + "insecure-context": "speechUnavailableInsecure", + "no-mic": "speechUnavailableNoMic", + "cloud-not-configured": "speechUnavailableCloud", +} as const satisfies Record + +const ERROR_KEYS = { + "mic-denied": "speechMicDenied", + "engine-failed": "speechFailed", + "cloud-auth": "speechCloudAuthFailed", + "cloud-not-configured": "speechUnavailableCloud", +} as const satisfies Record + +interface ComposerSpeechButtonProps { + status: SpeechInputStatus + interimText: string + unavailableReason: SpeechUnavailableReason | null + onToggle: () => void +} + +export function ComposerSpeechButton({ + status, + interimText, + unavailableReason, + onToggle, +}: ComposerSpeechButtonProps) { + const t = useTranslations("Folder.chat.messageInput") + + const label = + status === "listening" + ? t("speechStop") + : status === "transcribing" + ? t("speechTranscribing") + : status === "unavailable" && unavailableReason + ? t(UNAVAILABLE_KEYS[unavailableReason]) + : t("speechStart") + + return ( +
+ {status === "listening" && ( +
+ {interimText &&

{interimText}

} +

+ {t("speechCancelHint")} +

+
+ )} + +
+ ) +} + +interface ComposerSpeechControlProps { + editorRef: RefObject + isActive: boolean + shortcut: string + onInserted: () => void +} + +/** + * Owns one dictation session for a composer: inserts the final transcript at + * the caret as literal text (never HTML, never sent), cancels on Escape, and + * toggles on the voice-input shortcut while this composer is the active one. + */ +export function ComposerSpeechControl({ + editorRef, + isActive, + shortcut, + onInserted, +}: ComposerSpeechControlProps) { + const t = useTranslations("Folder.chat.messageInput") + + const onFinalText = useCallback( + (text: string) => { + const editor = editorRef.current?.getEditor() + if (!editor || !text) return + const { from } = editor.state.selection + const before = editor.state.doc.textBetween( + Math.max(0, from - 1), + from, + "\n", + "\n" + ) + const prefix = !editor.isEmpty && before && !/\s/.test(before) ? " " : "" + editor + .chain() + .focus() + .insertContent(textToInlineContent(prefix + text)) + .run() + onInserted() + }, + [editorRef, onInserted] + ) + + const onError = useCallback( + (error: SpeechInputError) => toast.error(t(ERROR_KEYS[error])), + [t] + ) + + const speech = useSpeechInput({ onFinalText, onError }) + const { status, cancel, toggle } = speech + const busy = status === "listening" || status === "transcribing" + + useEffect(() => { + if (!isActive || !busy) return + const onKeyDown = (event: KeyboardEvent) => { + if (event.key !== "Escape" || isImeCompositionKey(event)) return + event.preventDefault() + event.stopPropagation() + cancel() + } + window.addEventListener("keydown", onKeyDown, true) + return () => window.removeEventListener("keydown", onKeyDown, true) + }, [busy, cancel, isActive]) + + useEffect(() => { + if (!isActive || !shortcut || status === "unavailable") return + const onKeyDown = (event: KeyboardEvent) => { + if (!matchShortcutEvent(event, shortcut)) return + event.preventDefault() + toggle() + } + window.addEventListener("keydown", onKeyDown) + return () => window.removeEventListener("keydown", onKeyDown) + }, [isActive, shortcut, status, toggle]) + + return ( + + ) +} diff --git a/src/components/chat/message-input.test.tsx b/src/components/chat/message-input.test.tsx index e45a5591a6..28629097bb 100644 --- a/src/components/chat/message-input.test.tsx +++ b/src/components/chat/message-input.test.tsx @@ -202,7 +202,31 @@ vi.mock("@/hooks/use-appearance", async (importOriginal) => ({ useZoomLevel: () => ({ zoomLevel: 100, setZoomLevel: () => {} }), })) +// The dictation engine itself is covered in use-speech-input.test.ts; here the +// hook is a stub whose captured `onFinalText` stands in for a finished take. +const speechHook = vi.hoisted(() => ({ + onFinalText: null as ((text: string) => void) | null, +})) +vi.mock("./composer/use-speech-input", () => ({ + useSpeechInput: (opts: { onFinalText: (text: string) => void }) => { + speechHook.onFinalText = opts.onFinalText + return { + status: "idle", + interimText: "", + unavailableReason: null, + start: () => {}, + stop: () => {}, + cancel: () => {}, + toggle: () => {}, + } + }, +})) + import enMessages from "@/i18n/messages/en.json" +import { + resetSpeechPrefsCacheForTests, + saveSpeechPrefs, +} from "@/lib/speech-prefs" import type { PromptCapabilitiesInfo, SessionConfigOptionInfo, @@ -2427,3 +2451,55 @@ describe("MessageInput folder data arriving after mount", () => { expect(handle.getText()).toBe("") }) }) + +describe("MessageInput voice input", () => { + afterEach(() => { + cleanup() + composerHandle.current = null + speechHook.onFinalText = null + localStorage.clear() + resetSpeechPrefsCacheForTests() + }) + + function enableSpeech(enabled: boolean) { + localStorage.clear() + resetSpeechPrefsCacheForTests() + saveSpeechPrefs({ input: { enabled, engine: "auto", language: "" } }) + } + + const startLabel = enMessages.Folder.chat.messageInput.speechStart + + it("hides the mic button while voice input is off", async () => { + enableSpeech(false) + renderInput({}) + await waitFor(() => + expect(composerHandle.current?.getEditor()).toBeTruthy() + ) + expect( + screen.queryByRole("button", { name: startLabel }) + ).not.toBeInTheDocument() + }) + + it("inserts a transcript as literal text at the caret without sending", async () => { + enableSpeech(true) + const onSend = vi.fn() + renderInput({ onSend }) + expect( + await screen.findByRole("button", { name: startLabel }) + ).toBeInTheDocument() + await waitFor(() => + expect(composerHandle.current?.getEditor()).toBeTruthy() + ) + const editor = composerHandle.current!.getEditor()! + act(() => { + editor.commands.setContent("note:") + editor.commands.focus("end") + }) + + act(() => speechHook.onFinalText?.("x")) + + expect(serializeDocToText(editor.state.doc)).toBe("note: x") + expect(editor.getHTML()).not.toContain("") + expect(onSend).not.toHaveBeenCalled() + }) +}) diff --git a/src/components/chat/message-input.tsx b/src/components/chat/message-input.tsx index 54c2534372..14356b5600 100644 --- a/src/components/chat/message-input.tsx +++ b/src/components/chat/message-input.tsx @@ -122,6 +122,8 @@ import { serializeDocToText, } from "@/components/chat/composer/to-prompt-blocks" import { textToInlineContent } from "@/components/chat/composer/plain-text-content" +import { ComposerSpeechControl } from "@/components/chat/composer/composer-speech-button" +import { useSpeechPrefs } from "@/lib/speech-prefs" import { isEmbeddedReferenceUri } from "@/components/chat/composer/reference-uri" import { applyExpertReference, @@ -442,6 +444,7 @@ export function MessageInput({ knownInvocationsRef.current = knownInvocations }, [knownInvocations]) const { shortcuts } = useShortcutSettings() + const speechEnabled = useSpeechPrefs().input.enabled const effectiveDraftStorageKey = draftStorageKey ?? null const resolvedPlaceholder = placeholder ?? t("askAnything") const editorRef = useRef(null) @@ -2320,7 +2323,21 @@ export function MessageInput({ )} -
{actionButtons}
+
+ {speechEnabled ? ( +
+ + {actionButtons} +
+ ) : ( + actionButtons + )} +
{showDragActive && (
diff --git a/src/i18n/messages/ar.json b/src/i18n/messages/ar.json index 226042ce7a..63831a4525 100644 --- a/src/i18n/messages/ar.json +++ b/src/i18n/messages/ar.json @@ -509,6 +509,10 @@ "title": "إيقاف/استئناف النمط المخصص", "description": "مخرج طوارئ: يوقف كل الألوان المخصصة وCSS، ويعيد تفعيلها" }, + "toggle_voice_input": { + "title": "الإدخال الصوتي", + "description": "بدء الإملاء في مربع الرسالة أو إيقافه" + }, "zoom_in": { "title": "تكبير", "description": "اجعل النافذة أكبر بدرجة واحدة" @@ -3157,7 +3161,18 @@ "mentionGroupAgent": "الوكلاء", "mentionGroupSession": "الجلسات", "mentionGroupCommit": "عمليات الإيداع", - "mentionGroupSkill": "المهارات" + "mentionGroupSkill": "المهارات", + "speechStart": "بدء الإدخال الصوتي", + "speechStop": "إيقاف وإدراج", + "speechTranscribing": "جارٍ النسخ...", + "speechCancelHint": "Esc للإلغاء", + "speechMicDenied": "تم رفض الوصول إلى الميكروفون", + "speechUnavailableNoEngine": "الإدخال الصوتي غير متاح: لا يدعم هذا المتصفح التعرّف على الكلام", + "speechUnavailableInsecure": "يتطلب الإدخال الصوتي اتصالًا آمنًا (HTTPS أو localhost)", + "speechUnavailableNoMic": "الإدخال الصوتي غير متاح: لم يُعثر على ميكروفون", + "speechUnavailableCloud": "يتطلب الإدخال الصوتي مفتاح API سحابيًا (الإعدادات ← الصوت)", + "speechCloudAuthFailed": "رفضت خدمة الصوت مفتاح API", + "speechFailed": "فشل الإدخال الصوتي" }, "messageQueue": { "addToQueue": "إضافة للقائمة", diff --git a/src/i18n/messages/de.json b/src/i18n/messages/de.json index ec0a36da86..25d4f42b7e 100644 --- a/src/i18n/messages/de.json +++ b/src/i18n/messages/de.json @@ -509,6 +509,10 @@ "title": "Eigenen Stil aussetzen/fortsetzen", "description": "Notausstieg: schaltet alle eigenen Farben und CSS aus und wieder ein" }, + "toggle_voice_input": { + "title": "Spracheingabe", + "description": "Diktat in das Nachrichtenfeld starten oder stoppen" + }, "zoom_in": { "title": "Vergrößern", "description": "Das Fenster eine Stufe größer machen" @@ -3157,7 +3161,18 @@ "mentionGroupAgent": "Agenten", "mentionGroupSession": "Sitzungen", "mentionGroupCommit": "Commits", - "mentionGroupSkill": "Fähigkeiten" + "mentionGroupSkill": "Fähigkeiten", + "speechStart": "Spracheingabe starten", + "speechStop": "Stoppen und einfügen", + "speechTranscribing": "Wird transkribiert...", + "speechCancelHint": "Esc zum Abbrechen", + "speechMicDenied": "Mikrofonzugriff wurde verweigert", + "speechUnavailableNoEngine": "Spracheingabe nicht verfügbar: Dieser Browser hat keine Spracherkennung", + "speechUnavailableInsecure": "Spracheingabe benötigt eine sichere Verbindung (HTTPS oder localhost)", + "speechUnavailableNoMic": "Spracheingabe nicht verfügbar: Kein Mikrofon gefunden", + "speechUnavailableCloud": "Spracheingabe benötigt einen Cloud-API-Schlüssel (Einstellungen → Sprache)", + "speechCloudAuthFailed": "Der Sprachdienst hat den API-Schlüssel abgelehnt", + "speechFailed": "Spracheingabe fehlgeschlagen" }, "messageQueue": { "addToQueue": "Zur Warteschlange", diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index 80600cc306..d13fea694c 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -509,6 +509,10 @@ "title": "Suspend/resume custom style", "description": "Escape hatch: turns all custom colors and CSS off, and back on" }, + "toggle_voice_input": { + "title": "Voice Input", + "description": "Start or stop dictating into the message box" + }, "zoom_in": { "title": "Zoom in", "description": "Make the window one step larger" @@ -3157,7 +3161,18 @@ "mentionGroupAgent": "Agents", "mentionGroupSession": "Sessions", "mentionGroupCommit": "Commits", - "mentionGroupSkill": "Skills" + "mentionGroupSkill": "Skills", + "speechStart": "Start voice input", + "speechStop": "Stop and insert", + "speechTranscribing": "Transcribing...", + "speechCancelHint": "Esc to cancel", + "speechMicDenied": "Microphone access was denied", + "speechUnavailableNoEngine": "Voice input unavailable: this browser has no speech recognition", + "speechUnavailableInsecure": "Voice input needs a secure connection (HTTPS or localhost)", + "speechUnavailableNoMic": "Voice input unavailable: no microphone found", + "speechUnavailableCloud": "Voice input needs a cloud API key (Settings → Speech)", + "speechCloudAuthFailed": "The speech service rejected the API key", + "speechFailed": "Voice input failed" }, "messageQueue": { "addToQueue": "Queue message", diff --git a/src/i18n/messages/es.json b/src/i18n/messages/es.json index d34bd1d8ef..1247e436db 100644 --- a/src/i18n/messages/es.json +++ b/src/i18n/messages/es.json @@ -509,6 +509,10 @@ "title": "Suspender/reanudar estilo personalizado", "description": "Vía de escape: desactiva todos los colores y el CSS personalizados, y los vuelve a activar" }, + "toggle_voice_input": { + "title": "Entrada de voz", + "description": "Iniciar o detener el dictado en el cuadro de mensaje" + }, "zoom_in": { "title": "Acercar", "description": "Amplía la ventana un nivel" @@ -3157,7 +3161,18 @@ "mentionGroupAgent": "Agentes", "mentionGroupSession": "Sesiones", "mentionGroupCommit": "Commits", - "mentionGroupSkill": "Habilidades" + "mentionGroupSkill": "Habilidades", + "speechStart": "Iniciar entrada de voz", + "speechStop": "Detener e insertar", + "speechTranscribing": "Transcribiendo...", + "speechCancelHint": "Esc para cancelar", + "speechMicDenied": "Se denegó el acceso al micrófono", + "speechUnavailableNoEngine": "Entrada de voz no disponible: este navegador no tiene reconocimiento de voz", + "speechUnavailableInsecure": "La entrada de voz requiere una conexión segura (HTTPS o localhost)", + "speechUnavailableNoMic": "Entrada de voz no disponible: no se encontró micrófono", + "speechUnavailableCloud": "La entrada de voz necesita una clave de API en la nube (Ajustes → Voz)", + "speechCloudAuthFailed": "El servicio de voz rechazó la clave de API", + "speechFailed": "Falló la entrada de voz" }, "messageQueue": { "addToQueue": "Agregar a la cola", diff --git a/src/i18n/messages/fr.json b/src/i18n/messages/fr.json index b4a8264f7a..17524a1b18 100644 --- a/src/i18n/messages/fr.json +++ b/src/i18n/messages/fr.json @@ -509,6 +509,10 @@ "title": "Suspendre/réactiver le style personnalisé", "description": "Issue de secours : désactive toutes les couleurs et le CSS personnalisés, puis les réactive" }, + "toggle_voice_input": { + "title": "Saisie vocale", + "description": "Démarrer ou arrêter la dictée dans la zone de message" + }, "zoom_in": { "title": "Zoom avant", "description": "Agrandit la fenêtre d'un cran" @@ -3157,7 +3161,18 @@ "mentionGroupAgent": "Agents", "mentionGroupSession": "Sessions", "mentionGroupCommit": "Commits", - "mentionGroupSkill": "Compétences" + "mentionGroupSkill": "Compétences", + "speechStart": "Démarrer la saisie vocale", + "speechStop": "Arrêter et insérer", + "speechTranscribing": "Transcription...", + "speechCancelHint": "Échap pour annuler", + "speechMicDenied": "L'accès au micro a été refusé", + "speechUnavailableNoEngine": "Saisie vocale indisponible : ce navigateur n'a pas de reconnaissance vocale", + "speechUnavailableInsecure": "La saisie vocale nécessite une connexion sécurisée (HTTPS ou localhost)", + "speechUnavailableNoMic": "Saisie vocale indisponible : aucun micro détecté", + "speechUnavailableCloud": "La saisie vocale nécessite une clé API cloud (Paramètres → Voix)", + "speechCloudAuthFailed": "Le service vocal a refusé la clé API", + "speechFailed": "Échec de la saisie vocale" }, "messageQueue": { "addToQueue": "Mettre en file", diff --git a/src/i18n/messages/ja.json b/src/i18n/messages/ja.json index 9c79975034..8bf92a29a8 100644 --- a/src/i18n/messages/ja.json +++ b/src/i18n/messages/ja.json @@ -509,6 +509,10 @@ "title": "カスタムスタイルの停止/再開", "description": "緊急脱出用: カスタム配色と CSS をすべてオフにし、再度押すと元に戻します" }, + "toggle_voice_input": { + "title": "音声入力", + "description": "メッセージ欄への音声入力を開始・停止します" + }, "zoom_in": { "title": "拡大", "description": "ウィンドウの表示倍率を一段階上げます" @@ -3157,7 +3161,18 @@ "mentionGroupAgent": "エージェント", "mentionGroupSession": "セッション", "mentionGroupCommit": "コミット", - "mentionGroupSkill": "スキル" + "mentionGroupSkill": "スキル", + "speechStart": "音声入力を開始", + "speechStop": "停止して挿入", + "speechTranscribing": "文字起こし中...", + "speechCancelHint": "Esc でキャンセル", + "speechMicDenied": "マイクへのアクセスが拒否されました", + "speechUnavailableNoEngine": "音声入力は利用できません: このブラウザは音声認識に対応していません", + "speechUnavailableInsecure": "音声入力には安全な接続（HTTPS または localhost）が必要です", + "speechUnavailableNoMic": "音声入力は利用できません: マイクが見つかりません", + "speechUnavailableCloud": "音声入力にはクラウドの API キーが必要です（設定 → 音声）", + "speechCloudAuthFailed": "音声サービスが API キーを拒否しました", + "speechFailed": "音声入力に失敗しました" }, "messageQueue": { "addToQueue": "キューに追加", diff --git a/src/i18n/messages/ko.json b/src/i18n/messages/ko.json index adfbbf8a2d..ed37fa5ea1 100644 --- a/src/i18n/messages/ko.json +++ b/src/i18n/messages/ko.json @@ -509,6 +509,10 @@ "title": "사용자 지정 스타일 중지/재개", "description": "비상 탈출구: 모든 사용자 지정 색상과 CSS를 끄고, 다시 누르면 되돌립니다" }, + "toggle_voice_input": { + "title": "음성 입력", + "description": "메시지 입력란에 받아쓰기를 시작하거나 중지합니다" + }, "zoom_in": { "title": "확대", "description": "창을 한 단계 더 크게 만듭니다" @@ -3157,7 +3161,18 @@ "mentionGroupAgent": "에이전트", "mentionGroupSession": "세션", "mentionGroupCommit": "커밋", - "mentionGroupSkill": "스킬" + "mentionGroupSkill": "스킬", + "speechStart": "음성 입력 시작", + "speechStop": "중지하고 삽입", + "speechTranscribing": "변환 중...", + "speechCancelHint": "Esc로 취소", + "speechMicDenied": "마이크 접근이 거부되었습니다", + "speechUnavailableNoEngine": "음성 입력 불가: 이 브라우저는 음성 인식을 지원하지 않습니다", + "speechUnavailableInsecure": "음성 입력에는 보안 연결(HTTPS 또는 localhost)이 필요합니다", + "speechUnavailableNoMic": "음성 입력 불가: 마이크를 찾을 수 없습니다", + "speechUnavailableCloud": "음성 입력에는 클라우드 API 키가 필요합니다(설정 → 음성)", + "speechCloudAuthFailed": "음성 서비스가 API 키를 거부했습니다", + "speechFailed": "음성 입력에 실패했습니다" }, "messageQueue": { "addToQueue": "대기열에 추가", diff --git a/src/i18n/messages/pt.json b/src/i18n/messages/pt.json index e8551e9bc6..e2e21a7d0d 100644 --- a/src/i18n/messages/pt.json +++ b/src/i18n/messages/pt.json @@ -509,6 +509,10 @@ "title": "Suspender/retomar estilo personalizado", "description": "Saída de emergência: desliga todas as cores e o CSS personalizados e volta a ligá-los" }, + "toggle_voice_input": { + "title": "Entrada de voz", + "description": "Iniciar ou parar o ditado na caixa de mensagem" + }, "zoom_in": { "title": "Aumentar zoom", "description": "Aumenta a janela em um nível" @@ -3157,7 +3161,18 @@ "mentionGroupAgent": "Agentes", "mentionGroupSession": "Sessões", "mentionGroupCommit": "Commits", - "mentionGroupSkill": "Habilidades" + "mentionGroupSkill": "Habilidades", + "speechStart": "Iniciar entrada de voz", + "speechStop": "Parar e inserir", + "speechTranscribing": "Transcrevendo...", + "speechCancelHint": "Esc para cancelar", + "speechMicDenied": "O acesso ao microfone foi negado", + "speechUnavailableNoEngine": "Entrada de voz indisponível: este navegador não tem reconhecimento de voz", + "speechUnavailableInsecure": "A entrada de voz exige uma conexão segura (HTTPS ou localhost)", + "speechUnavailableNoMic": "Entrada de voz indisponível: nenhum microfone encontrado", + "speechUnavailableCloud": "A entrada de voz precisa de uma chave de API em nuvem (Configurações → Voz)", + "speechCloudAuthFailed": "O serviço de voz rejeitou a chave de API", + "speechFailed": "Falha na entrada de voz" }, "messageQueue": { "addToQueue": "Adicionar à fila", diff --git a/src/i18n/messages/zh-CN.json b/src/i18n/messages/zh-CN.json index 31ecfb9fbc..0042070701 100644 --- a/src/i18n/messages/zh-CN.json +++ b/src/i18n/messages/zh-CN.json @@ -509,6 +509,10 @@ "title": "停用/恢复自定义样式", "description": "逃生舱：一键关闭全部自定义配色与 CSS，再按一次恢复" }, + "toggle_voice_input": { + "title": "语音输入", + "description": "开始或停止向输入框口述" + }, "zoom_in": { "title": "放大", "description": "把窗口缩放提高一档" @@ -3157,7 +3161,18 @@ "mentionGroupAgent": "智能体", "mentionGroupSession": "会话", "mentionGroupCommit": "提交", - "mentionGroupSkill": "技能" + "mentionGroupSkill": "技能", + "speechStart": "开始语音输入", + "speechStop": "停止并插入", + "speechTranscribing": "正在转写...", + "speechCancelHint": "按 Esc 取消", + "speechMicDenied": "麦克风权限被拒绝", + "speechUnavailableNoEngine": "语音输入不可用：此浏览器不支持语音识别", + "speechUnavailableInsecure": "语音输入需要安全连接（HTTPS 或 localhost）", + "speechUnavailableNoMic": "语音输入不可用：未找到麦克风", + "speechUnavailableCloud": "语音输入需要云端 API 密钥（设置 → 语音）", + "speechCloudAuthFailed": "语音服务拒绝了该 API 密钥", + "speechFailed": "语音输入失败" }, "messageQueue": { "addToQueue": "加入队列", diff --git a/src/i18n/messages/zh-TW.json b/src/i18n/messages/zh-TW.json index 6009646f2f..85264a0a01 100644 --- a/src/i18n/messages/zh-TW.json +++ b/src/i18n/messages/zh-TW.json @@ -509,6 +509,10 @@ "title": "停用/恢復自訂樣式", "description": "逃生艙：一鍵關閉全部自訂配色與 CSS，再按一次恢復" }, + "toggle_voice_input": { + "title": "語音輸入", + "description": "開始或停止向輸入框口述" + }, "zoom_in": { "title": "放大", "description": "把視窗縮放提高一檔" @@ -3157,7 +3161,18 @@ "mentionGroupAgent": "智能體", "mentionGroupSession": "工作階段", "mentionGroupCommit": "提交", - "mentionGroupSkill": "技能" + "mentionGroupSkill": "技能", + "speechStart": "開始語音輸入", + "speechStop": "停止並插入", + "speechTranscribing": "正在轉寫...", + "speechCancelHint": "按 Esc 取消", + "speechMicDenied": "麥克風權限遭拒", + "speechUnavailableNoEngine": "語音輸入無法使用：此瀏覽器不支援語音辨識", + "speechUnavailableInsecure": "語音輸入需要安全連線（HTTPS 或 localhost）", + "speechUnavailableNoMic": "語音輸入無法使用：找不到麥克風", + "speechUnavailableCloud": "語音輸入需要雲端 API 金鑰（設定 → 語音）", + "speechCloudAuthFailed": "語音服務拒絕了此 API 金鑰", + "speechFailed": "語音輸入失敗" }, "messageQueue": { "addToQueue": "加入佇列", diff --git a/src/lib/keyboard-shortcuts.ts b/src/lib/keyboard-shortcuts.ts index c346a069a8..c89d4b9924 100644 --- a/src/lib/keyboard-shortcuts.ts +++ b/src/lib/keyboard-shortcuts.ts @@ -25,6 +25,7 @@ export type ShortcutActionId = | "send_message" | "newline_in_message" | "toggle_custom_style" + | "toggle_voice_input" | "zoom_in" | "zoom_out" | "zoom_reset" @@ -94,6 +95,9 @@ export const SHORTCUT_DEFINITIONS: ShortcutDefinition[] = [ { id: "toggle_custom_style", }, + { + id: "toggle_voice_input", + }, { id: "zoom_in", }, @@ -161,6 +165,7 @@ export const DEFAULT_SHORTCUTS: ShortcutSettings = { // 自定义样式的逃生舱：用户把界面改到不可用时，这一路必须仍然按得动，所以选一个 // 三修饰键组合（不会与任何常用操作撞车），并在捕获阶段监听。 toggle_custom_style: "mod+alt+shift+s", + toggle_voice_input: "mod+shift+m", // Same rungs as Settings → Window zoom. `=` is what US keyboards fire for // Ctrl/+ without Shift; `+` is Shift+= and the numpad. zoom_in: "mod+=", From d2878edede7f6e457f6908f992561d4ca62eb26b Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 17:18:32 +0700 Subject: [PATCH 06/15] feat(settings): add Speech settings page Refs #844 --- src/app/settings/speech/page.tsx | 5 + src/components/settings/settings-shell.tsx | 7 + .../settings/speech-settings.test.tsx | 182 +++++++++ src/components/settings/speech-settings.tsx | 385 ++++++++++++++++++ src/i18n/messages/ar.json | 37 ++ src/i18n/messages/de.json | 37 ++ src/i18n/messages/en.json | 37 ++ src/i18n/messages/es.json | 37 ++ src/i18n/messages/fr.json | 37 ++ src/i18n/messages/ja.json | 37 ++ src/i18n/messages/ko.json | 37 ++ src/i18n/messages/pt.json | 37 ++ src/i18n/messages/zh-CN.json | 37 ++ src/i18n/messages/zh-TW.json | 37 ++ 14 files changed, 949 insertions(+) create mode 100644 src/app/settings/speech/page.tsx create mode 100644 src/components/settings/speech-settings.test.tsx create mode 100644 src/components/settings/speech-settings.tsx diff --git a/src/app/settings/speech/page.tsx b/src/app/settings/speech/page.tsx new file mode 100644 index 0000000000..a12d2ba0db --- /dev/null +++ b/src/app/settings/speech/page.tsx @@ -0,0 +1,5 @@ +import { SpeechSettings } from "@/components/settings/speech-settings" + +export default function SettingsSpeechPage() { + return +} diff --git a/src/components/settings/settings-shell.tsx b/src/components/settings/settings-shell.tsx index 977646d6da..1a108b882d 100644 --- a/src/components/settings/settings-shell.tsx +++ b/src/components/settings/settings-shell.tsx @@ -8,6 +8,7 @@ import { type ReactNode, } from "react" import { + AudioLines, Bot, BookOpenText, Boxes, @@ -45,6 +46,7 @@ interface SettingsNavItem { href: string labelKey: | "general" + | "speech" | "appearance" | "agents" | "model_providers" @@ -74,6 +76,11 @@ const SETTINGS_NAV_ITEMS: SettingsNavItem[] = [ labelKey: "general", icon: SlidersHorizontal, }, + { + href: "/settings/speech", + labelKey: "speech", + icon: AudioLines, + }, { href: "/settings/mcp", labelKey: "mcp", diff --git a/src/components/settings/speech-settings.test.tsx b/src/components/settings/speech-settings.test.tsx new file mode 100644 index 0000000000..83ada360e3 --- /dev/null +++ b/src/components/settings/speech-settings.test.tsx @@ -0,0 +1,182 @@ +import { cleanup, render, screen, waitFor } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { NextIntlClientProvider } from "next-intl" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" + +import enMessages from "@/i18n/messages/en.json" +import type { SpeechCapabilities } from "@/lib/speech-capabilities" +import { + getSpeechPrefs, + resetSpeechPrefsCacheForTests, + saveSpeechPrefs, +} from "@/lib/speech-prefs" +import type { SpeechCloudSettings, SpeechCloudSettingsView } from "@/lib/types" + +const getSettings = vi.fn<() => Promise>() +const updateSettings = + vi.fn< + ( + s: SpeechCloudSettings, + k: string | null + ) => Promise + >() +const toastError = vi.fn() +const toastSuccess = vi.fn() +let caps: SpeechCapabilities + +vi.mock("@/lib/api", () => ({ + speechGetSettings: () => getSettings(), + speechUpdateSettings: (s: SpeechCloudSettings, k: string | null) => + updateSettings(s, k), +})) +vi.mock("@/lib/speech-capabilities", async (importOriginal) => ({ + ...(await importOriginal()), + detectSpeechCapabilities: () => caps, +})) +vi.mock("sonner", () => ({ + toast: { + error: (m: string) => toastError(m), + success: (m: string) => toastSuccess(m), + }, +})) + +import { SpeechSettings } from "./speech-settings" + +const CLOUD: SpeechCloudSettings = { + baseUrl: "https://api.openai.com/v1", + sttModel: "whisper-1", + ttsModel: "tts-1", + ttsVoice: "alloy", +} + +function view(apiKeySet: boolean, settings = CLOUD): SpeechCloudSettingsView { + return { settings, apiKeySet } +} + +function renderPage() { + return render( + + + + ) +} + +function enableInput(engine: "auto" | "browser" | "cloud" = "auto") { + saveSpeechPrefs({ input: { enabled: true, engine, language: "" } }) +} + +beforeEach(() => { + localStorage.clear() + resetSpeechPrefsCacheForTests() + caps = { browserStt: true, mediaCapture: true, secureContext: true } + getSettings.mockReset() + updateSettings.mockReset() + toastError.mockClear() + toastSuccess.mockClear() + getSettings.mockResolvedValue(view(false)) +}) +afterEach(() => cleanup()) + +describe("SpeechSettings", () => { + it("persists the voice input switch to prefs", async () => { + const user = userEvent.setup() + renderPage() + + await user.click(await screen.findByRole("switch", { name: "Voice input" })) + + expect(getSpeechPrefs().input.enabled).toBe(true) + expect(await screen.findByText("Recognition engine")).toBeInTheDocument() + }) + + it("persists the engine choice", async () => { + const user = userEvent.setup() + enableInput() + renderPage() + + await user.click( + await screen.findByRole("combobox", { name: "Recognition engine" }) + ) + await user.click(await screen.findByRole("option", { name: "Cloud" })) + + expect(getSpeechPrefs().input.engine).toBe("cloud") + }) + + it("sends no key when the key field is untouched", async () => { + const user = userEvent.setup() + getSettings.mockResolvedValue(view(true)) + updateSettings.mockResolvedValue(view(true)) + renderPage() + + await user.click(await screen.findByRole("button", { name: "Save" })) + + await waitFor(() => expect(updateSettings).toHaveBeenCalledTimes(1)) + expect(updateSettings.mock.calls[0][1]).toBeNull() + expect(toastSuccess).toHaveBeenCalledWith("Speech settings saved") + }) + + it("sends a typed key and then shows it as saved", async () => { + const user = userEvent.setup() + updateSettings.mockResolvedValue(view(true)) + renderPage() + + await user.type(await screen.findByLabelText("API key"), "sk-test") + await user.click(screen.getByRole("button", { name: "Save" })) + + await waitFor(() => + expect(updateSettings).toHaveBeenCalledWith(CLOUD, "sk-test") + ) + const keyInput = await screen.findByLabelText("API key") + expect(keyInput).toHaveValue("") + expect(keyInput).toHaveAttribute("placeholder", "Saved") + }) + + it("Remove key sends an empty key", async () => { + const user = userEvent.setup() + getSettings.mockResolvedValue(view(true)) + updateSettings.mockResolvedValue(view(false)) + renderPage() + + await user.click(await screen.findByRole("button", { name: "Remove key" })) + + await waitFor(() => expect(updateSettings).toHaveBeenCalledWith(CLOUD, "")) + expect( + screen.queryByRole("button", { name: "Remove key" }) + ).not.toBeInTheDocument() + }) + + it("reports a rejected save and keeps the key state", async () => { + const user = userEvent.setup() + updateSettings.mockRejectedValue(new Error("invalid base url")) + renderPage() + + const baseUrl = await screen.findByLabelText("Base URL") + await user.clear(baseUrl) + await user.type(baseUrl, "ftp://x") + await user.click(screen.getByRole("button", { name: "Save" })) + + await waitFor(() => expect(toastError).toHaveBeenCalledTimes(1)) + expect(toastError.mock.calls[0][0]).toContain("invalid base url") + expect(toastSuccess).not.toHaveBeenCalled() + }) + + it("status line names the unavailable reason", async () => { + caps = { browserStt: false, mediaCapture: true, secureContext: true } + enableInput("auto") + renderPage() + + expect(await screen.findByTestId("speech-engine-status")).toHaveTextContent( + "Unavailable: the cloud service has no API key yet." + ) + }) + + it("status line names the engine in use", async () => { + caps = { browserStt: false, mediaCapture: true, secureContext: true } + getSettings.mockResolvedValue(view(true)) + enableInput("auto") + renderPage() + + expect(await screen.findByTestId("speech-engine-status")).toHaveTextContent( + "Using: Cloud" + ) + }) +}) diff --git a/src/components/settings/speech-settings.tsx b/src/components/settings/speech-settings.tsx new file mode 100644 index 0000000000..cbce0ca43b --- /dev/null +++ b/src/components/settings/speech-settings.tsx @@ -0,0 +1,385 @@ +"use client" + +import { + useCallback, + useEffect, + useMemo, + useState, + useSyncExternalStore, +} from "react" +import { + AudioLines, + Cloud, + Cpu, + KeyRound, + Languages, + Link, + Loader2, + Mic, +} from "lucide-react" +import { useLocale, useTranslations } from "next-intl" +import { toast } from "sonner" + +import { SettingCard, SettingRow } from "@/components/shared/setting-card" +import { + SettingsError, + SettingsSaveBar, + SettingsSection, +} from "@/components/shared/settings-section" +import { Button } from "@/components/ui/button" +import { Input } from "@/components/ui/input" +import { ScrollArea } from "@/components/ui/scroll-area" +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select" +import { Switch } from "@/components/ui/switch" +import { speechGetSettings, speechUpdateSettings } from "@/lib/api" +import { toErrorMessage } from "@/lib/app-error" +import { + LOCALE_TO_BCP47, + detectSpeechCapabilities, + resolveInputEngine, +} from "@/lib/speech-capabilities" +import { + saveSpeechPrefs, + useSpeechPrefs, + type SpeechEnginePreference, +} from "@/lib/speech-prefs" +import type { SpeechCloudSettings } from "@/lib/types" + +const LANGUAGE_FOLLOW_APP = "follow-app" +const LANGUAGE_CUSTOM = "custom" +const LANGUAGE_TAGS = Array.from(new Set(Object.values(LOCALE_TO_BCP47))) + +const REASON_KEYS = { + "no-mic": "reasonNoMic", + "insecure-context": "reasonInsecure", + "no-engine": "reasonNoEngine", + "cloud-not-configured": "reasonCloudNotConfigured", +} as const + +const subscribeNever = () => () => {} +const onClient = () => true +const onServer = () => false + +function languageName(tag: string, locale: string): string { + try { + return new Intl.DisplayNames([locale], { type: "language" }).of(tag) ?? tag + } catch { + return tag + } +} + +export function SpeechSettings() { + const t = useTranslations("SpeechSettings") + const locale = useLocale() + const prefs = useSpeechPrefs() + + const [loading, setLoading] = useState(true) + const [loadError, setLoadError] = useState(null) + const [cloud, setCloud] = useState(null) + const [apiKeySet, setApiKeySet] = useState(false) + const [apiKeyDraft, setApiKeyDraft] = useState("") + const [saving, setSaving] = useState(false) + const [customLanguageMode, setCustomLanguageMode] = useState(false) + + const mounted = useSyncExternalStore(subscribeNever, onClient, onServer) + const caps = useMemo( + () => (mounted ? detectSpeechCapabilities() : null), + [mounted] + ) + + useEffect(() => { + let alive = true + speechGetSettings().then( + (view) => { + if (!alive) return + setCloud(view.settings) + setApiKeySet(view.apiKeySet) + setLoading(false) + }, + (err) => { + if (!alive) return + setLoadError(toErrorMessage(err)) + setLoading(false) + } + ) + return () => { + alive = false + } + }, []) + + const input = prefs.input + const updateInput = useCallback( + (patch: Partial) => { + saveSpeechPrefs({ ...prefs, input: { ...input, ...patch } }) + }, + [input, prefs] + ) + + const engineStatus = useMemo(() => { + if (!caps) return null + const resolution = resolveInputEngine(input, caps, apiKeySet) + if (resolution.engine === null) { + return t(REASON_KEYS[resolution.reason]) + } + return t("engineUsing", { + engine: t( + resolution.engine === "browser" ? "engineBrowser" : "engineCloud" + ), + }) + }, [apiKeySet, caps, input, t]) + + const languageSelection = + customLanguageMode || + (input.language !== "" && !LANGUAGE_TAGS.includes(input.language)) + ? LANGUAGE_CUSTOM + : input.language || LANGUAGE_FOLLOW_APP + + const onLanguageSelect = useCallback( + (value: string) => { + if (value === LANGUAGE_CUSTOM) { + setCustomLanguageMode(true) + return + } + setCustomLanguageMode(false) + updateInput({ language: value === LANGUAGE_FOLLOW_APP ? "" : value }) + }, + [updateInput] + ) + + const persistCloud = useCallback( + async (apiKey: string | null, successMessage: string) => { + if (!cloud) return + setSaving(true) + try { + const view = await speechUpdateSettings(cloud, apiKey) + setCloud(view.settings) + setApiKeySet(view.apiKeySet) + setApiKeyDraft("") + toast.success(successMessage) + } catch (err) { + toast.error(t("saveFailed", { message: toErrorMessage(err) })) + } finally { + setSaving(false) + } + }, + [cloud, t] + ) + + if (loading) { + return ( +
+ + {t("loading")} +
+ ) + } + + return ( + +
+
+

{t("sectionTitle")}

+

+ {t("sectionDescription")} +

+
+ + updateInput({ enabled })} + /> + } + > + {input.enabled && ( + + + {engineStatus} + + ) : undefined + } + htmlFor="speech-input-engine" + control={ + + } + /> + + + + + + + {t("languageFollowApp")} + + {LANGUAGE_TAGS.map((tag) => ( + + {languageName(tag, locale)} + + ))} + + {t("languageCustom")} + + + + } + > + {languageSelection === LANGUAGE_CUSTOM && ( + + updateInput({ language: e.target.value.trim() }) + } + spellCheck={false} + /> + )} + + + )} + + + + {loadError && ( + + {t("loadFailed", { message: loadError })} + + )} + {cloud && ( + + + + setCloud({ ...cloud, baseUrl: e.target.value }) + } + spellCheck={false} + /> + + void persistCloud("", t("keyRemoved"))} + > + {t("removeKey")} + + ) : undefined + } + > + setApiKeyDraft(e.target.value)} + placeholder={ + apiKeySet ? t("apiKeySaved") : t("apiKeyPlaceholder") + } + autoComplete="new-password" + /> + + + + setCloud({ ...cloud, sttModel: e.target.value }) + } + spellCheck={false} + /> + + + void persistCloud( + apiKeyDraft ? apiKeyDraft : null, + t("saved") + ) + } + saving={saving} + label={t("save")} + savingLabel={t("saving")} + /> + + )} + +
+ + ) +} diff --git a/src/i18n/messages/ar.json b/src/i18n/messages/ar.json index 63831a4525..a4547d97e8 100644 --- a/src/i18n/messages/ar.json +++ b/src/i18n/messages/ar.json @@ -38,6 +38,7 @@ "preferences": "التفضيلات", "nav": { "general": "عام", + "speech": "الصوت", "appearance": "المظهر", "agents": "الوكلاء", "mcp": "MCP", @@ -6543,5 +6544,41 @@ "probeFailed": "تعذّر على المتصفح المدمج التأكد من أنه يصل إلى المضيف البعيد عبر النفق. حاول مرة أخرى بعد قليل." } } + }, + "SpeechSettings": { + "sectionTitle": "الصوت", + "sectionDescription": "أملِ المطالبات بصوتك واضبط خدمة الصوت.", + "loading": "جارٍ تحميل إعدادات الصوت...", + "loadFailed": "تعذّر تحميل إعدادات الصوت: {message}", + "inputTitle": "الإدخال الصوتي", + "inputDescription": "يعرض زر ميكروفون في محرّر الدردشة يحوّل الكلام إلى نص المطالبة.", + "engineLabel": "محرّك التعرّف", + "engineAuto": "تلقائي", + "engineBrowser": "المتصفح", + "engineCloud": "السحابة", + "engineUsing": "قيد الاستخدام: {engine}", + "reasonNoMic": "غير متاح: لم يُعثر على ميكروفون.", + "reasonInsecure": "غير متاح: يتطلب الميكروفون اتصالًا آمنًا (HTTPS أو localhost).", + "reasonNoEngine": "غير متاح: لا يدعم هذا المتصفح التعرّف على الكلام. اضبط الخدمة السحابية أدناه.", + "reasonCloudNotConfigured": "غير متاح: لا يوجد مفتاح API للخدمة السحابية بعد.", + "languageLabel": "لغة التعرّف", + "languageDescription": "اللغة التي تتحدث بها أثناء الإملاء.", + "languageFollowApp": "اتباع لغة التطبيق", + "languageCustom": "وسم مخصص", + "languageCustomPlaceholder": "وسم BCP-47، مثل en-GB", + "cloudTitle": "خدمة الصوت السحابية", + "cloudDescription": "نقطة نهاية متوافقة مع OpenAI. لا يُرسل الصوت إليها إلا عند استخدام المحرّك السحابي.", + "baseUrl": "عنوان URL الأساسي", + "apiKey": "مفتاح API", + "apiKeyDescription": "يُخزَّن في سلسلة مفاتيح النظام ولا يُعرض مجددًا بعد الحفظ.", + "apiKeyPlaceholder": "sk-...", + "apiKeySaved": "محفوظ", + "removeKey": "إزالة المفتاح", + "keyRemoved": "تمت إزالة مفتاح API", + "sttModel": "نموذج تحويل الكلام إلى نص", + "save": "حفظ", + "saving": "جارٍ الحفظ...", + "saved": "تم حفظ إعدادات الصوت", + "saveFailed": "تعذّر حفظ إعدادات الصوت: {message}" } } diff --git a/src/i18n/messages/de.json b/src/i18n/messages/de.json index 25d4f42b7e..9a57796a8a 100644 --- a/src/i18n/messages/de.json +++ b/src/i18n/messages/de.json @@ -38,6 +38,7 @@ "preferences": "Präferenzen", "nav": { "general": "Allgemein", + "speech": "Sprache", "appearance": "Darstellung", "agents": "Agenten", "mcp": "MCP", @@ -6543,5 +6544,41 @@ "probeFailed": "Der integrierte Browser konnte nicht bestätigen, dass er den entfernten Host über den Tunnel erreicht. Versuche es gleich noch einmal." } } + }, + "SpeechSettings": { + "sectionTitle": "Sprache", + "sectionDescription": "Prompts per Stimme diktieren und den Sprachdienst konfigurieren.", + "loading": "Spracheinstellungen werden geladen...", + "loadFailed": "Spracheinstellungen konnten nicht geladen werden: {message}", + "inputTitle": "Spracheingabe", + "inputDescription": "Zeigt im Chat-Eingabefeld eine Mikrofontaste, die Sprache in Prompt-Text umwandelt.", + "engineLabel": "Erkennungs-Engine", + "engineAuto": "Automatisch", + "engineBrowser": "Browser", + "engineCloud": "Cloud", + "engineUsing": "Aktiv: {engine}", + "reasonNoMic": "Nicht verfügbar: Kein Mikrofon gefunden.", + "reasonInsecure": "Nicht verfügbar: Das Mikrofon benötigt eine sichere Verbindung (HTTPS oder localhost).", + "reasonNoEngine": "Nicht verfügbar: Dieser Browser hat keine Spracherkennung. Konfiguriere unten den Cloud-Dienst.", + "reasonCloudNotConfigured": "Nicht verfügbar: Für den Cloud-Dienst ist noch kein API-Schlüssel hinterlegt.", + "languageLabel": "Erkennungssprache", + "languageDescription": "Die Sprache, in der du diktierst.", + "languageFollowApp": "App-Sprache verwenden", + "languageCustom": "Eigener Tag", + "languageCustomPlaceholder": "BCP-47-Tag, z. B. en-GB", + "cloudTitle": "Cloud-Sprachdienst", + "cloudDescription": "Ein OpenAI-kompatibler Endpunkt. Audio wird nur gesendet, wenn die Cloud-Engine verwendet wird.", + "baseUrl": "Basis-URL", + "apiKey": "API-Schlüssel", + "apiKeyDescription": "Wird im Schlüsselbund des Systems gespeichert und danach nicht mehr angezeigt.", + "apiKeyPlaceholder": "sk-...", + "apiKeySaved": "Gespeichert", + "removeKey": "Schlüssel entfernen", + "keyRemoved": "API-Schlüssel entfernt", + "sttModel": "Sprache-zu-Text-Modell", + "save": "Speichern", + "saving": "Wird gespeichert...", + "saved": "Spracheinstellungen gespeichert", + "saveFailed": "Spracheinstellungen konnten nicht gespeichert werden: {message}" } } diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index d13fea694c..8e04fb6f13 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -38,6 +38,7 @@ "preferences": "Preferences", "nav": { "general": "General", + "speech": "Speech", "appearance": "Appearance", "agents": "Agents", "mcp": "MCP", @@ -6543,5 +6544,41 @@ "probeFailed": "The built-in browser couldn't confirm that it reaches the remote host through the tunnel. Try again in a moment." } } + }, + "SpeechSettings": { + "sectionTitle": "Speech", + "sectionDescription": "Dictate prompts with your voice and configure the speech service.", + "loading": "Loading speech settings...", + "loadFailed": "Failed to load speech settings: {message}", + "inputTitle": "Voice input", + "inputDescription": "Show a microphone button in the chat composer that turns speech into prompt text.", + "engineLabel": "Recognition engine", + "engineAuto": "Auto", + "engineBrowser": "Browser", + "engineCloud": "Cloud", + "engineUsing": "Using: {engine}", + "reasonNoMic": "Unavailable: no microphone was found.", + "reasonInsecure": "Unavailable: the microphone needs a secure (HTTPS or localhost) connection.", + "reasonNoEngine": "Unavailable: this browser has no speech recognition. Configure the cloud service below.", + "reasonCloudNotConfigured": "Unavailable: the cloud service has no API key yet.", + "languageLabel": "Recognition language", + "languageDescription": "The language you speak when dictating.", + "languageFollowApp": "Follow app language", + "languageCustom": "Custom tag", + "languageCustomPlaceholder": "BCP-47 tag, e.g. en-GB", + "cloudTitle": "Cloud speech service", + "cloudDescription": "An OpenAI-compatible endpoint. Audio is sent to it only when the Cloud engine is in use.", + "baseUrl": "Base URL", + "apiKey": "API key", + "apiKeyDescription": "Stored in the system keychain, never shown again after saving.", + "apiKeyPlaceholder": "sk-...", + "apiKeySaved": "Saved", + "removeKey": "Remove key", + "keyRemoved": "API key removed", + "sttModel": "Speech-to-text model", + "save": "Save", + "saving": "Saving...", + "saved": "Speech settings saved", + "saveFailed": "Failed to save speech settings: {message}" } } diff --git a/src/i18n/messages/es.json b/src/i18n/messages/es.json index 1247e436db..8b067fbef7 100644 --- a/src/i18n/messages/es.json +++ b/src/i18n/messages/es.json @@ -38,6 +38,7 @@ "preferences": "Preferencias", "nav": { "general": "General", + "speech": "Voz", "appearance": "Apariencia", "agents": "Agentes", "mcp": "MCP", @@ -6543,5 +6544,41 @@ "probeFailed": "El navegador integrado no pudo confirmar que llega al host remoto a través del túnel. Vuelve a intentarlo en un momento." } } + }, + "SpeechSettings": { + "sectionTitle": "Voz", + "sectionDescription": "Dicta prompts con tu voz y configura el servicio de voz.", + "loading": "Cargando ajustes de voz...", + "loadFailed": "No se pudieron cargar los ajustes de voz: {message}", + "inputTitle": "Entrada de voz", + "inputDescription": "Muestra un botón de micrófono en el editor del chat que convierte la voz en texto del prompt.", + "engineLabel": "Motor de reconocimiento", + "engineAuto": "Automático", + "engineBrowser": "Navegador", + "engineCloud": "Nube", + "engineUsing": "En uso: {engine}", + "reasonNoMic": "No disponible: no se encontró ningún micrófono.", + "reasonInsecure": "No disponible: el micrófono requiere una conexión segura (HTTPS o localhost).", + "reasonNoEngine": "No disponible: este navegador no tiene reconocimiento de voz. Configura el servicio en la nube abajo.", + "reasonCloudNotConfigured": "No disponible: el servicio en la nube aún no tiene clave de API.", + "languageLabel": "Idioma de reconocimiento", + "languageDescription": "El idioma en el que hablas al dictar.", + "languageFollowApp": "Seguir el idioma de la app", + "languageCustom": "Etiqueta personalizada", + "languageCustomPlaceholder": "Etiqueta BCP-47, p. ej. en-GB", + "cloudTitle": "Servicio de voz en la nube", + "cloudDescription": "Un endpoint compatible con OpenAI. El audio solo se envía cuando se usa el motor en la nube.", + "baseUrl": "URL base", + "apiKey": "Clave de API", + "apiKeyDescription": "Se guarda en el llavero del sistema y no se vuelve a mostrar.", + "apiKeyPlaceholder": "sk-...", + "apiKeySaved": "Guardada", + "removeKey": "Eliminar clave", + "keyRemoved": "Clave de API eliminada", + "sttModel": "Modelo de voz a texto", + "save": "Guardar", + "saving": "Guardando...", + "saved": "Ajustes de voz guardados", + "saveFailed": "No se pudieron guardar los ajustes de voz: {message}" } } diff --git a/src/i18n/messages/fr.json b/src/i18n/messages/fr.json index 17524a1b18..91fb7388df 100644 --- a/src/i18n/messages/fr.json +++ b/src/i18n/messages/fr.json @@ -38,6 +38,7 @@ "preferences": "Préférences", "nav": { "general": "Général", + "speech": "Voix", "appearance": "Apparence", "agents": "Agents IA", "mcp": "MCP", @@ -6543,5 +6544,41 @@ "probeFailed": "Le navigateur intégré n'a pas pu confirmer qu'il atteint l'hôte distant par le tunnel. Réessayez dans un instant." } } + }, + "SpeechSettings": { + "sectionTitle": "Voix", + "sectionDescription": "Dictez vos prompts à la voix et configurez le service vocal.", + "loading": "Chargement des paramètres vocaux...", + "loadFailed": "Échec du chargement des paramètres vocaux : {message}", + "inputTitle": "Saisie vocale", + "inputDescription": "Affiche un bouton micro dans la zone de saisie du chat pour transformer la voix en texte de prompt.", + "engineLabel": "Moteur de reconnaissance", + "engineAuto": "Automatique", + "engineBrowser": "Navigateur", + "engineCloud": "Cloud", + "engineUsing": "Utilisé : {engine}", + "reasonNoMic": "Indisponible : aucun micro détecté.", + "reasonInsecure": "Indisponible : le micro nécessite une connexion sécurisée (HTTPS ou localhost).", + "reasonNoEngine": "Indisponible : ce navigateur ne propose pas de reconnaissance vocale. Configurez le service cloud ci-dessous.", + "reasonCloudNotConfigured": "Indisponible : le service cloud n'a pas encore de clé API.", + "languageLabel": "Langue de reconnaissance", + "languageDescription": "La langue que vous parlez pendant la dictée.", + "languageFollowApp": "Suivre la langue de l'app", + "languageCustom": "Balise personnalisée", + "languageCustomPlaceholder": "Balise BCP-47, p. ex. en-GB", + "cloudTitle": "Service vocal cloud", + "cloudDescription": "Un endpoint compatible OpenAI. L'audio n'y est envoyé que lorsque le moteur cloud est utilisé.", + "baseUrl": "URL de base", + "apiKey": "Clé API", + "apiKeyDescription": "Stockée dans le trousseau du système, jamais réaffichée après l'enregistrement.", + "apiKeyPlaceholder": "sk-...", + "apiKeySaved": "Enregistrée", + "removeKey": "Supprimer la clé", + "keyRemoved": "Clé API supprimée", + "sttModel": "Modèle de transcription", + "save": "Enregistrer", + "saving": "Enregistrement...", + "saved": "Paramètres vocaux enregistrés", + "saveFailed": "Échec de l'enregistrement des paramètres vocaux : {message}" } } diff --git a/src/i18n/messages/ja.json b/src/i18n/messages/ja.json index 8bf92a29a8..1bbc030adc 100644 --- a/src/i18n/messages/ja.json +++ b/src/i18n/messages/ja.json @@ -38,6 +38,7 @@ "preferences": "環境設定", "nav": { "general": "一般", + "speech": "音声", "appearance": "外観", "agents": "エージェント", "mcp": "MCP", @@ -6543,5 +6544,41 @@ "probeFailed": "内蔵ブラウザーがトンネル経由でリモートホストに届くことを確認できませんでした。しばらくしてからもう一度お試しください。" } } + }, + "SpeechSettings": { + "sectionTitle": "音声", + "sectionDescription": "音声でプロンプトを入力し、音声サービスを設定します。", + "loading": "音声設定を読み込み中...", + "loadFailed": "音声設定の読み込みに失敗しました: {message}", + "inputTitle": "音声入力", + "inputDescription": "チャット入力欄にマイクボタンを表示し、音声をプロンプトのテキストに変換します。", + "engineLabel": "認識エンジン", + "engineAuto": "自動", + "engineBrowser": "ブラウザ", + "engineCloud": "クラウド", + "engineUsing": "使用中: {engine}", + "reasonNoMic": "利用不可: マイクが見つかりません。", + "reasonInsecure": "利用不可: マイクには安全な接続（HTTPS または localhost）が必要です。", + "reasonNoEngine": "利用不可: このブラウザは音声認識に対応していません。下のクラウドサービスを設定してください。", + "reasonCloudNotConfigured": "利用不可: クラウドサービスに API キーが設定されていません。", + "languageLabel": "認識言語", + "languageDescription": "音声入力で話す言語。", + "languageFollowApp": "アプリの言語に従う", + "languageCustom": "カスタムタグ", + "languageCustomPlaceholder": "BCP-47 タグ（例: en-GB）", + "cloudTitle": "クラウド音声サービス", + "cloudDescription": "OpenAI 互換のエンドポイント。クラウドエンジン使用時のみ音声が送信されます。", + "baseUrl": "ベース URL", + "apiKey": "API キー", + "apiKeyDescription": "システムのキーチェーンに保存され、保存後は表示されません。", + "apiKeyPlaceholder": "sk-...", + "apiKeySaved": "保存済み", + "removeKey": "キーを削除", + "keyRemoved": "API キーを削除しました", + "sttModel": "音声認識モデル", + "save": "保存", + "saving": "保存中...", + "saved": "音声設定を保存しました", + "saveFailed": "音声設定の保存に失敗しました: {message}" } } diff --git a/src/i18n/messages/ko.json b/src/i18n/messages/ko.json index ed37fa5ea1..7d9e3660cf 100644 --- a/src/i18n/messages/ko.json +++ b/src/i18n/messages/ko.json @@ -38,6 +38,7 @@ "preferences": "환경설정", "nav": { "general": "일반", + "speech": "음성", "appearance": "외관", "agents": "에이전트", "mcp": "MCP", @@ -6543,5 +6544,41 @@ "probeFailed": "내장 브라우저가 터널을 통해 원격 호스트에 도달하는지 확인할 수 없습니다. 잠시 후 다시 시도하세요." } } + }, + "SpeechSettings": { + "sectionTitle": "음성", + "sectionDescription": "음성으로 프롬프트를 입력하고 음성 서비스를 설정합니다.", + "loading": "음성 설정을 불러오는 중...", + "loadFailed": "음성 설정을 불러오지 못했습니다: {message}", + "inputTitle": "음성 입력", + "inputDescription": "채팅 입력창에 마이크 버튼을 표시하여 음성을 프롬프트 텍스트로 변환합니다.", + "engineLabel": "인식 엔진", + "engineAuto": "자동", + "engineBrowser": "브라우저", + "engineCloud": "클라우드", + "engineUsing": "사용 중: {engine}", + "reasonNoMic": "사용 불가: 마이크를 찾을 수 없습니다.", + "reasonInsecure": "사용 불가: 마이크를 사용하려면 보안 연결(HTTPS 또는 localhost)이 필요합니다.", + "reasonNoEngine": "사용 불가: 이 브라우저는 음성 인식을 지원하지 않습니다. 아래에서 클라우드 서비스를 설정하세요.", + "reasonCloudNotConfigured": "사용 불가: 클라우드 서비스에 API 키가 없습니다.", + "languageLabel": "인식 언어", + "languageDescription": "받아쓰기할 때 말하는 언어입니다.", + "languageFollowApp": "앱 언어 따르기", + "languageCustom": "사용자 지정 태그", + "languageCustomPlaceholder": "BCP-47 태그, 예: en-GB", + "cloudTitle": "클라우드 음성 서비스", + "cloudDescription": "OpenAI 호환 엔드포인트입니다. 클라우드 엔진을 사용할 때만 오디오가 전송됩니다.", + "baseUrl": "기본 URL", + "apiKey": "API 키", + "apiKeyDescription": "시스템 키체인에 저장되며 저장 후에는 다시 표시되지 않습니다.", + "apiKeyPlaceholder": "sk-...", + "apiKeySaved": "저장됨", + "removeKey": "키 삭제", + "keyRemoved": "API 키가 삭제되었습니다", + "sttModel": "음성-텍스트 변환 모델", + "save": "저장", + "saving": "저장 중...", + "saved": "음성 설정이 저장되었습니다", + "saveFailed": "음성 설정을 저장하지 못했습니다: {message}" } } diff --git a/src/i18n/messages/pt.json b/src/i18n/messages/pt.json index e2e21a7d0d..0dc3bf0813 100644 --- a/src/i18n/messages/pt.json +++ b/src/i18n/messages/pt.json @@ -38,6 +38,7 @@ "preferences": "Preferências", "nav": { "general": "Geral", + "speech": "Voz", "appearance": "Aparência", "agents": "Agentes", "mcp": "MCP", @@ -6543,5 +6544,41 @@ "probeFailed": "O navegador integrado não conseguiu confirmar que alcança o host remoto pelo túnel. Tente novamente em instantes." } } + }, + "SpeechSettings": { + "sectionTitle": "Voz", + "sectionDescription": "Dite prompts com a sua voz e configure o serviço de voz.", + "loading": "Carregando configurações de voz...", + "loadFailed": "Falha ao carregar as configurações de voz: {message}", + "inputTitle": "Entrada de voz", + "inputDescription": "Mostra um botão de microfone no campo do chat que transforma a fala em texto do prompt.", + "engineLabel": "Mecanismo de reconhecimento", + "engineAuto": "Automático", + "engineBrowser": "Navegador", + "engineCloud": "Nuvem", + "engineUsing": "Em uso: {engine}", + "reasonNoMic": "Indisponível: nenhum microfone encontrado.", + "reasonInsecure": "Indisponível: o microfone exige uma conexão segura (HTTPS ou localhost).", + "reasonNoEngine": "Indisponível: este navegador não tem reconhecimento de voz. Configure o serviço em nuvem abaixo.", + "reasonCloudNotConfigured": "Indisponível: o serviço em nuvem ainda não tem chave de API.", + "languageLabel": "Idioma de reconhecimento", + "languageDescription": "O idioma que você fala ao ditar.", + "languageFollowApp": "Seguir o idioma do app", + "languageCustom": "Tag personalizada", + "languageCustomPlaceholder": "Tag BCP-47, ex.: en-GB", + "cloudTitle": "Serviço de voz em nuvem", + "cloudDescription": "Um endpoint compatível com OpenAI. O áudio só é enviado quando o mecanismo em nuvem está em uso.", + "baseUrl": "URL base", + "apiKey": "Chave de API", + "apiKeyDescription": "Armazenada no chaveiro do sistema e nunca mais exibida após salvar.", + "apiKeyPlaceholder": "sk-...", + "apiKeySaved": "Salva", + "removeKey": "Remover chave", + "keyRemoved": "Chave de API removida", + "sttModel": "Modelo de fala para texto", + "save": "Salvar", + "saving": "Salvando...", + "saved": "Configurações de voz salvas", + "saveFailed": "Falha ao salvar as configurações de voz: {message}" } } diff --git a/src/i18n/messages/zh-CN.json b/src/i18n/messages/zh-CN.json index 0042070701..b8a0dd4562 100644 --- a/src/i18n/messages/zh-CN.json +++ b/src/i18n/messages/zh-CN.json @@ -38,6 +38,7 @@ "preferences": "偏好设置", "nav": { "general": "常规", + "speech": "语音", "appearance": "外观", "agents": "智能体", "mcp": "MCP", @@ -6543,5 +6544,41 @@ "probeFailed": "内置浏览器无法确认它能通过隧道访问远端主机。请稍后重试。" } } + }, + "SpeechSettings": { + "sectionTitle": "语音", + "sectionDescription": "用语音口述提示词，并配置语音服务。", + "loading": "正在加载语音设置...", + "loadFailed": "加载语音设置失败：{message}", + "inputTitle": "语音输入", + "inputDescription": "在聊天输入框中显示麦克风按钮，将语音转换为提示词文本。", + "engineLabel": "识别引擎", + "engineAuto": "自动", + "engineBrowser": "浏览器", + "engineCloud": "云端", + "engineUsing": "当前使用：{engine}", + "reasonNoMic": "不可用：未找到麦克风。", + "reasonInsecure": "不可用：麦克风需要安全连接（HTTPS 或 localhost）。", + "reasonNoEngine": "不可用：此浏览器不支持语音识别。请在下方配置云端服务。", + "reasonCloudNotConfigured": "不可用：云端服务尚未设置 API 密钥。", + "languageLabel": "识别语言", + "languageDescription": "口述时所说的语言。", + "languageFollowApp": "跟随应用语言", + "languageCustom": "自定义标签", + "languageCustomPlaceholder": "BCP-47 标签，例如 en-GB", + "cloudTitle": "云端语音服务", + "cloudDescription": "兼容 OpenAI 的接口。仅在使用云端引擎时才会向其发送音频。", + "baseUrl": "基础 URL", + "apiKey": "API 密钥", + "apiKeyDescription": "保存在系统钥匙串中，保存后不再显示。", + "apiKeyPlaceholder": "sk-...", + "apiKeySaved": "已保存", + "removeKey": "移除密钥", + "keyRemoved": "API 密钥已移除", + "sttModel": "语音转文本模型", + "save": "保存", + "saving": "正在保存...", + "saved": "语音设置已保存", + "saveFailed": "保存语音设置失败：{message}" } } diff --git a/src/i18n/messages/zh-TW.json b/src/i18n/messages/zh-TW.json index 85264a0a01..441ffced45 100644 --- a/src/i18n/messages/zh-TW.json +++ b/src/i18n/messages/zh-TW.json @@ -38,6 +38,7 @@ "preferences": "偏好設定", "nav": { "general": "一般", + "speech": "語音", "appearance": "外觀", "agents": "智能體", "mcp": "MCP", @@ -6543,5 +6544,41 @@ "probeFailed": "內建瀏覽器無法確認它能透過通道連到遠端主機。請稍後再試。" } } + }, + "SpeechSettings": { + "sectionTitle": "語音", + "sectionDescription": "用語音口述提示詞，並設定語音服務。", + "loading": "正在載入語音設定...", + "loadFailed": "載入語音設定失敗：{message}", + "inputTitle": "語音輸入", + "inputDescription": "在聊天輸入框中顯示麥克風按鈕，將語音轉換為提示詞文字。", + "engineLabel": "辨識引擎", + "engineAuto": "自動", + "engineBrowser": "瀏覽器", + "engineCloud": "雲端", + "engineUsing": "目前使用：{engine}", + "reasonNoMic": "無法使用：找不到麥克風。", + "reasonInsecure": "無法使用：麥克風需要安全連線（HTTPS 或 localhost）。", + "reasonNoEngine": "無法使用：此瀏覽器不支援語音辨識。請在下方設定雲端服務。", + "reasonCloudNotConfigured": "無法使用：雲端服務尚未設定 API 金鑰。", + "languageLabel": "辨識語言", + "languageDescription": "口述時所說的語言。", + "languageFollowApp": "跟隨應用程式語言", + "languageCustom": "自訂標籤", + "languageCustomPlaceholder": "BCP-47 標籤，例如 en-GB", + "cloudTitle": "雲端語音服務", + "cloudDescription": "相容 OpenAI 的端點。僅在使用雲端引擎時才會傳送音訊。", + "baseUrl": "基礎 URL", + "apiKey": "API 金鑰", + "apiKeyDescription": "儲存在系統鑰匙圈中，儲存後不再顯示。", + "apiKeyPlaceholder": "sk-...", + "apiKeySaved": "已儲存", + "removeKey": "移除金鑰", + "keyRemoved": "API 金鑰已移除", + "sttModel": "語音轉文字模型", + "save": "儲存", + "saving": "正在儲存...", + "saved": "語音設定已儲存", + "saveFailed": "儲存語音設定失敗：{message}" } } From 44846ced58954589def50042201430211ab4772c Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Mon, 28 Sep 2026 19:36:46 +0700 Subject: [PATCH 07/15] fix(speech): report browser dictation unavailable on insecure origins Chromium exposes SpeechRecognition on plain-http origins but every session fails at once with not-allowed, so the mic showed "voice input failed" immediately. Browser dictation is now only offered in a secure context; the cloud engine keeps working over http. Refs #844 --- src/lib/speech-capabilities.test.ts | 1 + src/lib/speech-capabilities.ts | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lib/speech-capabilities.test.ts b/src/lib/speech-capabilities.test.ts index 81c92406ae..e2588bd402 100644 --- a/src/lib/speech-capabilities.test.ts +++ b/src/lib/speech-capabilities.test.ts @@ -75,6 +75,7 @@ describe("speech capabilities detection", () => { hasMediaDevices: true, isSecureContext: false, }) + expect(caps.browserStt).toBe(false) expect(caps.mediaCapture).toBe(false) expect(caps.secureContext).toBe(false) }) diff --git a/src/lib/speech-capabilities.ts b/src/lib/speech-capabilities.ts index 51bfb90685..121af3de8e 100644 --- a/src/lib/speech-capabilities.ts +++ b/src/lib/speech-capabilities.ts @@ -72,8 +72,6 @@ export function detectSpeechCapabilities( const isUnsupportedDesktop = desktop && (plat === "windows" || plat === "linux") - const browserStt = hasRecognizer && !isUnsupportedDesktop - const secureContext = typeof env?.isSecureContext === "boolean" ? env.isSecureContext @@ -81,6 +79,10 @@ export function detectSpeechCapabilities( ? Boolean(window.isSecureContext) : false + // Chromium exposes SpeechRecognition on insecure origins too, but it cannot + // open the microphone there and fails at once with "audio-capture". + const browserStt = hasRecognizer && !isUnsupportedDesktop && secureContext + let hasGetUserMedia = false if (typeof env?.hasMediaDevices === "boolean") { hasGetUserMedia = env.hasMediaDevices From 39f0f6b1e05f1f1932b2d33ae7b32ab56c0dc41d Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 17:18:33 +0700 Subject: [PATCH 08/15] feat(speech): add cloud speech synthesis backend Refs #844 --- src-tauri/src/commands/speech.rs | 202 +++++++++++++++++++++++++-- src-tauri/src/lib.rs | 1 + src-tauri/src/web/handlers/speech.rs | 19 ++- src-tauri/src/web/router.rs | 4 + src/lib/api.ts | 8 ++ src/lib/types.ts | 5 + 6 files changed, 227 insertions(+), 12 deletions(-) diff --git a/src-tauri/src/commands/speech.rs b/src-tauri/src/commands/speech.rs index 75f0cad4bf..89bcb93ed9 100644 --- a/src-tauri/src/commands/speech.rs +++ b/src-tauri/src/commands/speech.rs @@ -40,6 +40,16 @@ pub struct SpeechCloudSettingsView { pub api_key_set: bool, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpeechAudio { + pub audio_base64: String, + pub mime_type: String, +} + +/// The OpenAI speech endpoint's documented input ceiling. +const MAX_SYNTHESIS_CHARS: usize = 4096; + #[cfg(not(test))] mod store { pub fn get_secret(key: &str) -> Result, String> { @@ -203,9 +213,7 @@ pub async fn speech_transcribe_core( }; let filename = format!("speech.{}", ext); - let api_key = store::get_secret(SPEECH_CLOUD_API_KEY) - .map_err(|e| AppCommandError::io_error("Failed to read the speech API key").with_detail(e))? - .ok_or_else(|| AppCommandError::configuration_missing("Speech cloud API key not set"))?; + let api_key = read_api_key()?; let settings = get_settings_core(conn).await; @@ -234,6 +242,30 @@ pub async fn speech_transcribe_core( .await .map_err(|e| AppCommandError::network(e.to_string()))?; + let res = ensure_upstream_success(res).await?; + + #[derive(Deserialize)] + struct TranscriptionResponse { + text: String, + } + + let json: TranscriptionResponse = res + .json() + .await + .map_err(|e| AppCommandError::network(e.to_string()))?; + + Ok(json.text.trim().to_string()) +} + +fn read_api_key() -> Result { + store::get_secret(SPEECH_CLOUD_API_KEY) + .map_err(|e| AppCommandError::io_error("Failed to read the speech API key").with_detail(e))? + .ok_or_else(|| AppCommandError::configuration_missing("Speech cloud API key not set")) +} + +async fn ensure_upstream_success( + res: reqwest::Response, +) -> Result { let status = res.status(); if status == reqwest::StatusCode::UNAUTHORIZED || status == reqwest::StatusCode::FORBIDDEN { return Err(AppCommandError::authentication_failed( @@ -252,18 +284,69 @@ pub async fn speech_transcribe_core( status, truncated ))); } + Ok(res) +} - #[derive(Deserialize)] - struct TranscriptionResponse { - text: String, +pub async fn speech_synthesize_core( + conn: &sea_orm::DatabaseConnection, + text: String, + speed: f32, +) -> Result { + if text.trim().is_empty() { + return Err(AppCommandError::invalid_input("Text is empty")); + } + if text.chars().count() > MAX_SYNTHESIS_CHARS { + return Err(AppCommandError::invalid_input(format!( + "Text exceeds {MAX_SYNTHESIS_CHARS} characters" + ))); } + let speed = if speed.is_finite() { + speed.clamp(0.25, 4.0) + } else { + 1.0 + }; - let json: TranscriptionResponse = res - .json() + let api_key = read_api_key()?; + let settings = get_settings_core(conn).await; + let url = format!("{}/audio/speech", settings.base_url); + + let res = get_client() + .post(&url) + .bearer_auth(api_key) + .json(&serde_json::json!({ + "model": settings.tts_model, + "voice": settings.tts_voice, + "input": text, + "response_format": "mp3", + "speed": speed, + })) + .send() .await .map_err(|e| AppCommandError::network(e.to_string()))?; - Ok(json.text.trim().to_string()) + let res = ensure_upstream_success(res).await?; + + let mime_type = res + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.split(';').next()) + .map(|v| v.trim().to_string()) + .filter(|v| v.starts_with("audio/")) + .unwrap_or_else(|| "audio/mpeg".to_string()); + + let bytes = res + .bytes() + .await + .map_err(|e| AppCommandError::network(e.to_string()))?; + if bytes.is_empty() { + return Err(AppCommandError::network("Speech service returned no audio")); + } + + Ok(SpeechAudio { + audio_base64: general_purpose::STANDARD.encode(&bytes), + mime_type, + }) } #[cfg(feature = "tauri-runtime")] @@ -295,6 +378,16 @@ pub async fn speech_transcribe( speech_transcribe_core(&db.conn, audio_base64, mime_type, language).await } +#[cfg(feature = "tauri-runtime")] +#[cfg_attr(feature = "tauri-runtime", tauri::command)] +pub async fn speech_synthesize( + db: State<'_, AppDatabase>, + text: String, + speed: f32, +) -> Result { + speech_synthesize_core(&db.conn, text, speed).await +} + #[cfg(test)] mod tests { use super::*; @@ -473,8 +566,43 @@ mod tests { .into_response() } + /// Answers with WAV bytes only for the exact body the default settings + /// should produce, so a wrong model, voice, format or unclamped speed + /// surfaces as a 500 in the test. + async fn mock_speech_handler( + headers: HeaderMap, + axum::Json(body): axum::Json, + ) -> impl IntoResponse { + if headers + .get("authorization") + .is_none_or(|a| a != "Bearer test-key") + { + return (StatusCode::UNAUTHORIZED, "Unauthorized").into_response(); + } + let expected = serde_json::json!({ + "model": "tts-1", + "voice": "alloy", + "input": "Hello there.", + "response_format": "mp3", + "speed": 4.0, + }); + if body != expected { + return (StatusCode::INTERNAL_SERVER_ERROR, body.to_string()).into_response(); + } + ( + StatusCode::OK, + [(axum::http::header::CONTENT_TYPE, "audio/wav")], + MOCK_WAV.to_vec(), + ) + .into_response() + } + + const MOCK_WAV: &[u8] = b"RIFF\x24\0\0\0WAVEfmt "; + async fn start_mock_server() -> String { - let app = Router::new().route("/v1/audio/transcriptions", post(mock_transcription_handler)); + let app = Router::new() + .route("/v1/audio/transcriptions", post(mock_transcription_handler)) + .route("/v1/audio/speech", post(mock_speech_handler)); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); tokio::spawn(async move { @@ -580,4 +708,58 @@ mod tests { .unwrap_err(); assert!(matches!(err.code, AppErrorCode::NetworkError)); } + + async fn configure_mock(db: &crate::db::AppDatabase, key: &str) { + let settings = SpeechCloudSettings { + base_url: format!("{}/v1", start_mock_server().await), + ..Default::default() + }; + speech_update_settings_core(&db.conn, settings, Some(key.to_string())) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_synthesis_success_clamps_speed_and_round_trips_audio() { + let _guard = KEY_LOCK.lock().await; + let db = setup_test_db().await; + configure_mock(&db, "test-key").await; + + let audio = speech_synthesize_core(&db.conn, "Hello there.".to_string(), 9.0) + .await + .unwrap(); + assert_eq!(audio.mime_type, "audio/wav"); + assert_eq!( + general_purpose::STANDARD + .decode(audio.audio_base64) + .unwrap(), + MOCK_WAV + ); + } + + #[tokio::test] + async fn test_synthesis_unauthorized() { + let _guard = KEY_LOCK.lock().await; + let db = setup_test_db().await; + configure_mock(&db, "wrong-key").await; + + let err = speech_synthesize_core(&db.conn, "Hello there.".to_string(), 1.0) + .await + .unwrap_err(); + assert!(matches!(err.code, AppErrorCode::AuthenticationFailed)); + } + + #[tokio::test] + async fn test_synthesis_rejects_empty_and_oversized_text() { + let db = setup_test_db().await; + let err = speech_synthesize_core(&db.conn, " ".to_string(), 1.0) + .await + .unwrap_err(); + assert!(matches!(err.code, AppErrorCode::InvalidInput)); + + let err = speech_synthesize_core(&db.conn, "a".repeat(4097), 1.0) + .await + .unwrap_err(); + assert!(matches!(err.code, AppErrorCode::InvalidInput)); + } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 534a3a3b5c..d6ed953434 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1979,6 +1979,7 @@ mod tauri_app { speech::speech_get_settings, speech::speech_update_settings, speech::speech_transcribe, + speech::speech_synthesize, config_sync::config_sync_update_settings, config_sync::config_sync_get_state, config_sync::config_sync_test_connection, diff --git a/src-tauri/src/web/handlers/speech.rs b/src-tauri/src/web/handlers/speech.rs index 5c37344def..21e62294ad 100644 --- a/src-tauri/src/web/handlers/speech.rs +++ b/src-tauri/src/web/handlers/speech.rs @@ -1,8 +1,8 @@ use crate::app_error::AppCommandError; use crate::app_state::AppState; use crate::commands::speech::{ - speech_get_settings_core, speech_transcribe_core, speech_update_settings_core, - SpeechCloudSettings, SpeechCloudSettingsView, + speech_get_settings_core, speech_synthesize_core, speech_transcribe_core, + speech_update_settings_core, SpeechAudio, SpeechCloudSettings, SpeechCloudSettingsView, }; use axum::{extract::Extension, Json}; use serde::Deserialize; @@ -23,6 +23,13 @@ pub struct TranscribeParams { pub language: Option, } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SynthesizeParams { + pub text: String, + pub speed: f32, +} + pub async fn speech_get_settings( Extension(state): Extension>, ) -> Result, AppCommandError> { @@ -51,3 +58,11 @@ pub async fn speech_transcribe( .await?; Ok(Json(text)) } + +pub async fn speech_synthesize( + Extension(state): Extension>, + Json(params): Json, +) -> Result, AppCommandError> { + let audio = speech_synthesize_core(&state.db.conn, params.text, params.speed).await?; + Ok(Json(audio)) +} diff --git a/src-tauri/src/web/router.rs b/src-tauri/src/web/router.rs index 0409f640f9..193989573d 100644 --- a/src-tauri/src/web/router.rs +++ b/src-tauri/src/web/router.rs @@ -616,6 +616,10 @@ pub fn build_router( post(handlers::speech::speech_transcribe) .layer(DefaultBodyLimit::max(36 * 1024 * 1024)), ) + .route( + "/speech_synthesize", + post(handlers::speech::speech_synthesize), + ) .route( "/config_sync_get_settings", post(handlers::config_sync::config_sync_get_settings), diff --git a/src/lib/api.ts b/src/lib/api.ts index 375e1d0736..8ff78c2378 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -29,6 +29,7 @@ import type { DeepSeekCatalogModel, DeepSeekModelCatalog, SpeechCloudSettings, + SpeechAudio, SpeechCloudSettingsView, ForgeChangeDetail, ForgeChangedFileList, @@ -5987,3 +5988,10 @@ export async function speechTranscribe( language, }) } + +export async function speechSynthesize( + text: string, + speed: number +): Promise { + return getTransport().call("speech_synthesize", { text, speed }) +} diff --git a/src/lib/types.ts b/src/lib/types.ts index 2e41708c2c..c834b4ce4a 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -5085,3 +5085,8 @@ export interface SpeechCloudSettingsView { settings: SpeechCloudSettings apiKeySet: boolean } + +export interface SpeechAudio { + audioBase64: string + mimeType: string +} From 89282c35edbc1cd25e01ae57bda3823ae61ae1f4 Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 17:18:33 +0700 Subject: [PATCH 09/15] feat(speech): extract speakable text from agent replies Refs #844 --- src/lib/speakable-text.test.ts | 72 ++++++++++++++++++ src/lib/speakable-text.ts | 132 +++++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 src/lib/speakable-text.test.ts create mode 100644 src/lib/speakable-text.ts diff --git a/src/lib/speakable-text.test.ts b/src/lib/speakable-text.test.ts new file mode 100644 index 0000000000..e5a56bebf8 --- /dev/null +++ b/src/lib/speakable-text.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest" + +import { chunkSpeakableText, toSpeakableText } from "./speakable-text" + +const labels = { + codeOmitted: "Code block omitted", + tableOmitted: "Table omitted", +} + +describe("toSpeakableText", () => { + it("replaces fenced code with the label and keeps inline code", () => { + const md = "Run `pnpm test` first.\n\n```ts\nconst x = 1\n```\n\nDone" + expect(toSpeakableText(md, labels)).toBe( + "Run pnpm test first. Code block omitted. Done." + ) + }) + + it("keeps link text and drops link targets and bare URLs", () => { + const md = + "See [the docs](https://example.com/a) or https://example.com/b now" + expect(toSpeakableText(md, labels)).toBe("See the docs or now.") + }) + + it("replaces tables with the label", () => { + const md = "Results:\n\n| a | b |\n| - | - |\n| 1 | 2 |\n" + expect(toSpeakableText(md, labels)).toBe("Results: Table omitted.") + }) + + it("reads headings, list items and quotes as sentences", () => { + const md = "# Summary\n\n- first item\n- second item!\n\n> quoted" + expect(toSpeakableText(md, labels)).toBe( + "Summary. first item. second item! quoted." + ) + }) + + it("drops images, html, rules and footnote definitions; unwraps emphasis", () => { + const md = + "A **bold** _and_ ~~gone~~ word[^1]\n\n![alt](x.png)\n\n
raw
\n\n---\n\n[^1]: note" + expect(toSpeakableText(md, labels)).toBe("A bold and gone word.") + }) +}) + +describe("chunkSpeakableText", () => { + it("packs whole sentences up to maxLen", () => { + expect(chunkSpeakableText("One. Two. Three.", 10)).toEqual([ + "One. Two.", + " Three.", + ]) + }) + + it("splits CJK sentences on full-width punctuation and hard-splits long runs", () => { + const text = "你好。今天天气很好！" + "长".repeat(12) + const chunks = chunkSpeakableText(text, 5) + expect(chunks.every((c) => c.length <= 5)).toBe(true) + expect(chunks.join("")).toBe(text) + expect(chunks[0]).toBe("你好。") + }) + + it("hard-splits a long Latin sentence at the last space", () => { + const chunks = chunkSpeakableText("alpha beta gamma delta", 12) + expect(chunks).toEqual(["alpha beta ", "gamma delta"]) + }) + + it("keeps every chunk within maxLen and loses nothing on a 10 000-char input", () => { + const sentence = "The quick brown fox jumps over the lazy dog. " + const text = sentence.repeat(Math.ceil(10_000 / sentence.length)).trim() + const chunks = chunkSpeakableText(text, 220) + expect(text.length).toBeGreaterThanOrEqual(10_000) + expect(chunks.every((c) => c.length <= 220)).toBe(true) + expect(chunks.join("")).toBe(text) + }) +}) diff --git a/src/lib/speakable-text.ts b/src/lib/speakable-text.ts new file mode 100644 index 0000000000..016840ca5f --- /dev/null +++ b/src/lib/speakable-text.ts @@ -0,0 +1,132 @@ +import type { Nodes, Root } from "mdast" +import remarkGfm from "remark-gfm" +import remarkParse from "remark-parse" +import { unified } from "unified" + +export interface SpeakableLabels { + codeOmitted: string + tableOmitted: string +} + +const BARE_URL = /https?:\/\/\S+/g +const SENTENCE_END = /[.!?:;。！？：；]$/ +const CJK = /[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\uf900-\ufaff]/ + +const parser = unified().use(remarkParse).use(remarkGfm) + +function collapse(text: string): string { + return text.replace(/\s+/g, " ").trim() +} + +function asSentence(text: string): string { + const clean = collapse(text) + if (!clean) return "" + return SENTENCE_END.test(clean) ? clean : `${clean}.` +} + +function inlineText(node: Nodes, labels: SpeakableLabels): string { + switch (node.type) { + case "text": + return node.value.replace(BARE_URL, "") + case "inlineCode": + return node.value + case "break": + return " " + case "image": + case "imageReference": + case "html": + case "footnoteReference": + return "" + default: + if ("children" in node) { + return (node.children as Nodes[]) + .map((child) => inlineText(child, labels)) + .join("") + } + return "" + } +} + +function blockSentences(node: Nodes, labels: SpeakableLabels): string[] { + switch (node.type) { + case "code": + return [asSentence(labels.codeOmitted)] + case "table": + return [asSentence(labels.tableOmitted)] + case "html": + case "thematicBreak": + case "footnoteDefinition": + case "definition": + case "yaml": + return [] + case "heading": + case "paragraph": + return [asSentence(inlineText(node, labels))] + case "root": + case "blockquote": + case "list": + case "listItem": + return node.children.flatMap((child) => + blockSentences(child as Nodes, labels) + ) + default: + return [asSentence(inlineText(node, labels))] + } +} + +export function toSpeakableText( + markdown: string, + labels: SpeakableLabels +): string { + const tree = parser.parse(markdown) as Root + return blockSentences(tree, labels).filter(Boolean).join(" ") +} + +function splitSentences(text: string): string[] { + const sentences: string[] = [] + let current = "" + for (const char of text) { + current += char + if (/[.!?。！？\n]/.test(char)) { + sentences.push(current) + current = "" + } + } + if (current) sentences.push(current) + return sentences +} + +function hardSplit(sentence: string, maxLen: number): string[] { + const pieces: string[] = [] + let rest = sentence + while (rest.length > maxLen) { + const window = rest.slice(0, maxLen) + const space = window.lastIndexOf(" ") + const cut = + space > 0 && !CJK.test(window.charAt(maxLen - 1)) ? space + 1 : maxLen + pieces.push(rest.slice(0, cut)) + rest = rest.slice(cut) + } + if (rest) pieces.push(rest) + return pieces +} + +/** + * Packs whole sentences greedily into chunks of at most `maxLen` characters. + * Chunks keep their original spacing, so `chunks.join("")` is the input. + */ +export function chunkSpeakableText(text: string, maxLen: number): string[] { + const chunks: string[] = [] + let current = "" + for (const sentence of splitSentences(text)) { + for (const piece of hardSplit(sentence, maxLen)) { + if (current.length + piece.length > maxLen && current) { + chunks.push(current) + current = "" + } + current += piece + } + } + if (current) chunks.push(current) + return chunks.filter((chunk) => chunk.trim().length > 0) +} From 1e03080bc7722ebb1cc7200abf03b6875d4214b2 Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 17:18:33 +0700 Subject: [PATCH 10/15] feat(speech): add read-aloud player Refs #844 --- src/lib/speech-capabilities.test.ts | 63 ++++- src/lib/speech-capabilities.ts | 56 +++++ src/lib/speech-player.test.ts | 357 ++++++++++++++++++++++++++++ src/lib/speech-player.ts | 301 +++++++++++++++++++++++ src/lib/speech-prefs.test.ts | 49 ++++ src/lib/speech-prefs.ts | 124 +++++++--- 6 files changed, 915 insertions(+), 35 deletions(-) create mode 100644 src/lib/speech-player.test.ts create mode 100644 src/lib/speech-player.ts diff --git a/src/lib/speech-capabilities.test.ts b/src/lib/speech-capabilities.test.ts index e2588bd402..9f59ed6c82 100644 --- a/src/lib/speech-capabilities.test.ts +++ b/src/lib/speech-capabilities.test.ts @@ -1,8 +1,11 @@ -import { describe, expect, it } from "vitest" +import { describe, expect, it, vi } from "vitest" import { detectSpeechCapabilities, + hasBrowserTts, resolveInputEngine, + resolveOutputEngine, + waitForVoices, resolveSpeechLanguage, type SpeechCapabilities, } from "./speech-capabilities" @@ -230,3 +233,61 @@ describe("resolveSpeechLanguage mapping", () => { expect(resolveSpeechLanguage("", "")).toBe("en-US") }) }) + +describe("read-aloud capabilities", () => { + const voice = { voiceURI: "v", lang: "en-US" } as SpeechSynthesisVoice + + it("reports browser TTS only when voices exist", () => { + expect(hasBrowserTts(undefined)).toBe(false) + expect(hasBrowserTts({ getVoices: () => [] })).toBe(false) + expect(hasBrowserTts({ getVoices: () => [voice] })).toBe(true) + }) + + it("waitForVoices resolves on voiceschanged", async () => { + let voices: SpeechSynthesisVoice[] = [] + const target = new EventTarget() + const synth = { + getVoices: () => voices, + addEventListener: target.addEventListener.bind(target), + removeEventListener: target.removeEventListener.bind(target), + } as unknown as SpeechSynthesis + const pending = waitForVoices(synth, 60_000) + voices = [voice] + target.dispatchEvent(new Event("voiceschanged")) + await expect(pending).resolves.toEqual([voice]) + }) + + it("waitForVoices resolves empty after the timeout", async () => { + vi.useFakeTimers() + try { + const target = new EventTarget() + const synth = { + getVoices: () => [], + addEventListener: target.addEventListener.bind(target), + removeEventListener: target.removeEventListener.bind(target), + } as unknown as SpeechSynthesis + const pending = waitForVoices(synth, 1500) + vi.advanceTimersByTime(1500) + await expect(pending).resolves.toEqual([]) + } finally { + vi.useRealTimers() + } + }) + + it.each([ + ["browser", true, false, { engine: "browser" }], + ["browser", false, true, { engine: null, reason: "no-engine" }], + ["cloud", true, true, { engine: "cloud" }], + ["cloud", true, false, { engine: null, reason: "cloud-not-configured" }], + ["auto", true, true, { engine: "browser" }], + ["auto", false, true, { engine: "cloud" }], + ["auto", false, false, { engine: null, reason: "cloud-not-configured" }], + ] as const)( + "resolveOutputEngine(%s, tts=%s, cloud=%s)", + (engine, browserTts, cloudConfigured, expected) => { + expect( + resolveOutputEngine({ engine }, { browserTts }, cloudConfigured) + ).toEqual(expected) + } + ) +}) diff --git a/src/lib/speech-capabilities.ts b/src/lib/speech-capabilities.ts index 121af3de8e..18278c7232 100644 --- a/src/lib/speech-capabilities.ts +++ b/src/lib/speech-capabilities.ts @@ -145,6 +145,62 @@ export function resolveInputEngine( return { engine: null, reason: "no-engine" } } +export type OutputEngineResolution = + | { engine: "browser" | "cloud" } + | { engine: null; reason: "no-engine" | "cloud-not-configured" } + +type VoiceSource = Pick & + Partial> + +function currentSynth(): VoiceSource | undefined { + return typeof window !== "undefined" ? window.speechSynthesis : undefined +} + +export function hasBrowserTts(synth: VoiceSource | undefined = currentSynth()) { + return Boolean(synth) && synth!.getVoices().length > 0 +} + +/** Chromium fills `getVoices()` asynchronously; resolves once voices exist or the timeout passes. */ +export function waitForVoices( + synth: VoiceSource | undefined = currentSynth(), + timeoutMs = 1500 +): Promise { + if (!synth) return Promise.resolve([]) + const voices = synth.getVoices() + if (voices.length > 0 || !synth.addEventListener) { + return Promise.resolve(voices) + } + return new Promise((resolve) => { + const finish = () => { + clearTimeout(timer) + synth.removeEventListener?.("voiceschanged", finish) + resolve(synth.getVoices()) + } + const timer = setTimeout(finish, timeoutMs) + synth.addEventListener!("voiceschanged", finish) + }) +} + +export function resolveOutputEngine( + pref: { engine: "auto" | "browser" | "cloud" }, + caps: { browserTts: boolean }, + cloudConfigured: boolean +): OutputEngineResolution { + if (pref.engine === "browser") { + return caps.browserTts + ? { engine: "browser" } + : { engine: null, reason: "no-engine" } + } + if (pref.engine === "cloud") { + return cloudConfigured + ? { engine: "cloud" } + : { engine: null, reason: "cloud-not-configured" } + } + if (caps.browserTts) return { engine: "browser" } + if (cloudConfigured) return { engine: "cloud" } + return { engine: null, reason: "cloud-not-configured" } +} + export function resolveSpeechLanguage( pref: SpeechInputPrefs | { language?: string } | string, uiLocale: string diff --git a/src/lib/speech-player.test.ts b/src/lib/speech-player.test.ts new file mode 100644 index 0000000000..6ec2ed46c0 --- /dev/null +++ b/src/lib/speech-player.test.ts @@ -0,0 +1,357 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" + +const synthesize = vi.fn() +const getSettings = vi.fn() +vi.mock("@/lib/api", () => ({ + speechSynthesize: (...args: unknown[]) => synthesize(...args), + speechGetSettings: () => getSettings(), +})) + +import { + getSpeechPlayerState, + maybeAutoRead, + resetSpeechPlayerForTests, + speak, + stopSpeech, + subscribeSpeechPlayer, +} from "./speech-player" +import type { SpeakOptions } from "./speech-player" +import { + DEFAULT_SPEECH_PREFS, + resetSpeechPrefsCacheForTests, + saveSpeechPrefs, +} from "./speech-prefs" + +const labels = { codeOmitted: "Code omitted", tableOmitted: "Table omitted" } + +class FakeUtterance { + text: string + lang = "" + rate = 1 + voice: unknown = null + onstart: (() => void) | null = null + onend: (() => void) | null = null + onerror: ((event: { error: string }) => void) | null = null + constructor(text: string) { + this.text = text + } +} + +class FakeSynth { + queue: FakeUtterance[] = [] + voices = [{ voiceURI: "v-en", lang: "en-US" }] + speak = vi.fn((u: FakeUtterance): void => { + this.queue.push(u) + }) + cancel = vi.fn(() => { + this.queue = [] + }) + getVoices = () => this.voices +} + +class FakeAudio { + static instances: FakeAudio[] = [] + src = "" + paused = true + onended: (() => void) | null = null + onerror: (() => void) | null = null + played: string[] = [] + constructor() { + FakeAudio.instances.push(this) + } + play = vi.fn(() => { + this.paused = false + this.played.push(this.src) + return Promise.resolve() + }) + pause = vi.fn(() => { + this.paused = true + }) + removeAttribute = vi.fn((name: string) => { + if (name === "src") this.src = "" + }) + load = vi.fn() + finish() { + this.paused = true + this.onended?.() + } +} + +function deferred() { + let resolve!: (value: T) => void + let reject!: (error: unknown) => void + const promise = new Promise((res, rej) => { + resolve = res + reject = rej + }) + return { promise, resolve, reject } +} + +function waitForState(predicate: () => boolean): Promise { + if (predicate()) return Promise.resolve() + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + unsubscribe() + reject( + new Error( + `state never matched: ${JSON.stringify(getSpeechPlayerState())}` + ) + ) + }, 1000) + const unsubscribe = subscribeSpeechPlayer(() => { + if (!predicate()) return + clearTimeout(timer) + unsubscribe() + resolve() + }) + }) +} + +const audioPayload = { audioBase64: btoa("mp3"), mimeType: "audio/mpeg" } + +let synth: FakeSynth +let urlCount = 0 +const createObjectURL = vi.fn(() => `blob:${++urlCount}`) +const revokeObjectURL = vi.fn() + +const browser: SpeakOptions = { engine: "browser", language: "en-US", labels } +const cloud: SpeakOptions = { engine: "cloud", language: "en-US", labels } + +beforeEach(() => { + localStorage.clear() + resetSpeechPrefsCacheForTests() + synth = new FakeSynth() + FakeAudio.instances = [] + urlCount = 0 + synthesize.mockReset() + getSettings.mockReset() + createObjectURL.mockClear() + revokeObjectURL.mockClear() + vi.stubGlobal("speechSynthesis", synth) + vi.stubGlobal("SpeechSynthesisUtterance", FakeUtterance) + vi.stubGlobal("Audio", FakeAudio) + vi.stubGlobal("URL", Object.assign(URL, { createObjectURL, revokeObjectURL })) + resetSpeechPlayerForTests() +}) + +afterEach(() => { + resetSpeechPlayerForTests() + vi.unstubAllGlobals() +}) + +describe("browser engine", () => { + it("queues one utterance per chunk and tracks state until the last ends", () => { + const long = "First sentence here. ".repeat(20) + speak("turn-1", long, browser) + expect(synth.queue.length).toBeGreaterThan(1) + expect(synth.queue.every((u) => u.text.length <= 220)).toBe(true) + expect(synth.queue[0].lang).toBe("en-US") + expect(synth.queue[0].voice).toEqual(synth.voices[0]) + expect(getSpeechPlayerState()).toEqual({ + playingId: "turn-1", + status: "loading", + }) + + synth.queue[0].onstart?.() + expect(getSpeechPlayerState().status).toBe("playing") + synth.queue[0].onend?.() + expect(getSpeechPlayerState().status).toBe("playing") + synth.queue[synth.queue.length - 1].onend?.() + expect(getSpeechPlayerState()).toEqual({ playingId: null, status: "idle" }) + }) + + it("stops and reports an error, but ignores interruptions", () => { + const onError = vi.fn() + speak("turn-1", "Hello there.", { ...browser, onError }) + synth.queue[0].onerror?.({ error: "interrupted" }) + expect(onError).not.toHaveBeenCalled() + synth.queue[0].onerror?.({ error: "synthesis-failed" }) + expect(onError).toHaveBeenCalledWith("failed") + expect(getSpeechPlayerState().status).toBe("idle") + }) + + it("uses the saved rate and voice", () => { + synth.voices.push({ voiceURI: "v-other", lang: "en-GB" }) + saveSpeechPrefs({ + output: { + ...DEFAULT_SPEECH_PREFS.output, + rate: 1.5, + browserVoiceUri: "v-other", + }, + }) + speak("turn-1", "Hello.", browser) + expect(synth.queue[0].rate).toBe(1.5) + expect(synth.queue[0].voice).toEqual({ voiceURI: "v-other", lang: "en-GB" }) + }) + + it("speak while playing stops the previous playback", () => { + speak("turn-1", "One.", browser) + speak("turn-2", "Two.", browser) + expect(synth.cancel).toHaveBeenCalled() + expect(synth.queue.map((u) => u.text)).toEqual(["Two."]) + expect(getSpeechPlayerState().playingId).toBe("turn-2") + synth.queue[0].onend?.() + expect(getSpeechPlayerState().status).toBe("idle") + }) +}) + +describe("cloud engine", () => { + it("plays chunks in order, prefetches the next and revokes URLs", async () => { + const first = deferred() + const second = deferred() + synthesize + .mockReturnValueOnce(first.promise) + .mockReturnValueOnce(second.promise) + const text = `${"a".repeat(3000)}. ${"b".repeat(3000)}.` + + speak("turn-1", text, cloud) + expect(synthesize).toHaveBeenCalledTimes(1) + first.resolve(audioPayload) + await waitForState(() => getSpeechPlayerState().status === "playing") + + const audio = FakeAudio.instances[0] + expect(audio.played).toEqual(["blob:1"]) + expect(synthesize).toHaveBeenCalledTimes(2) + expect(synthesize.mock.calls[1][1]).toBe(1) + + second.resolve(audioPayload) + const secondPlay = new Promise((resolve) => { + audio.play.mockImplementationOnce(() => { + audio.played.push(audio.src) + resolve() + return Promise.resolve() + }) + }) + audio.finish() + await secondPlay + expect(audio.played).toEqual(["blob:1", "blob:2"]) + expect(revokeObjectURL).toHaveBeenCalledWith("blob:1") + + const idle = waitForState(() => getSpeechPlayerState().status === "idle") + audio.finish() + await idle + expect(revokeObjectURL).toHaveBeenCalledWith("blob:2") + expect(FakeAudio.instances).toHaveLength(1) + }) + + it("stopSpeech mid-fetch discards the late response", async () => { + const pending = deferred() + synthesize.mockReturnValueOnce(pending.promise) + speak("turn-1", "Hello.", cloud) + stopSpeech() + expect(getSpeechPlayerState().status).toBe("idle") + pending.resolve(audioPayload) + await pending.promise + await Promise.resolve() + expect(createObjectURL).not.toHaveBeenCalled() + expect(FakeAudio.instances[0].play).not.toHaveBeenCalled() + }) + + it("stopSpeech while playing pauses, clears src and revokes URLs", async () => { + synthesize.mockResolvedValue(audioPayload) + speak("turn-1", "Hello.", cloud) + await waitForState(() => getSpeechPlayerState().status === "playing") + const audio = FakeAudio.instances[0] + stopSpeech() + expect(audio.pause).toHaveBeenCalled() + expect(audio.removeAttribute).toHaveBeenCalledWith("src") + expect(audio.load).toHaveBeenCalled() + expect(revokeObjectURL).toHaveBeenCalledWith("blob:1") + expect(getSpeechPlayerState().status).toBe("idle") + }) + + it("maps an auth failure to the auth error", async () => { + synthesize.mockRejectedValue({ + code: "authentication_failed", + message: "no", + }) + const onError = vi.fn() + const failed = new Promise((resolve) => + onError.mockImplementation(() => resolve()) + ) + speak("turn-1", "Hello.", { ...cloud, onError }) + await failed + expect(onError).toHaveBeenCalledWith("auth") + expect(getSpeechPlayerState().status).toBe("idle") + }) +}) + +describe("engine resolution", () => { + const auto: SpeakOptions = { language: "en-US", labels } + + it("uses browser voices when the preference is auto", async () => { + const spoken = new Promise((resolve) => + synth.speak.mockImplementationOnce((u: FakeUtterance) => { + synth.queue.push(u) + resolve() + }) + ) + speak("turn-1", "Hello.", auto) + await spoken + expect(synth.queue.map((q) => q.text)).toEqual(["Hello."]) + expect(getSettings).not.toHaveBeenCalled() + }) + + it("falls back to cloud when no browser voice exists and a key is set", async () => { + synth.voices = [] + vi.stubGlobal("speechSynthesis", undefined) + getSettings.mockResolvedValue({ apiKeySet: true }) + synthesize.mockResolvedValue(audioPayload) + speak("turn-1", "Hello.", auto) + await waitForState(() => getSpeechPlayerState().status === "playing") + expect(synthesize).toHaveBeenCalledWith("Hello.", 1) + }) + + it("reports not-configured when neither engine is usable", async () => { + vi.stubGlobal("speechSynthesis", undefined) + getSettings.mockResolvedValue({ apiKeySet: false }) + const onError = vi.fn() + const failed = new Promise((resolve) => + onError.mockImplementation(() => resolve()) + ) + speak("turn-1", "Hello.", { ...auto, onError }) + await failed + expect(onError).toHaveBeenCalledWith("not-configured") + expect(getSpeechPlayerState().status).toBe("idle") + }) +}) + +describe("maybeAutoRead", () => { + const ctx = { + contextKey: "tab-1", + activeId: "tab-1", + visibility: "visible" as DocumentVisibilityState, + } + + function enableAutoRead(autoRead = true) { + saveSpeechPrefs({ + output: { ...DEFAULT_SPEECH_PREFS.output, enabled: true, autoRead }, + }) + } + + it("does nothing when read aloud or auto-read is off", () => { + expect(maybeAutoRead(ctx, "Hi.", browser)).toBe(false) + enableAutoRead(false) + expect(maybeAutoRead(ctx, "Hi.", browser)).toBe(false) + expect(synth.speak).not.toHaveBeenCalled() + }) + + it("skips background tabs, hidden documents and empty text", () => { + enableAutoRead() + expect(maybeAutoRead({ ...ctx, activeId: "tab-2" }, "Hi.", browser)).toBe( + false + ) + expect( + maybeAutoRead({ ...ctx, visibility: "hidden" }, "Hi.", browser) + ).toBe(false) + expect(maybeAutoRead(ctx, " \n ", browser)).toBe(false) + expect(synth.speak).not.toHaveBeenCalled() + }) + + it("speaks the active visible tab's reply", () => { + enableAutoRead() + expect(maybeAutoRead(ctx, "All done.", browser)).toBe(true) + expect(synth.queue.map((u) => u.text)).toEqual(["All done."]) + expect(getSpeechPlayerState().playingId).toBe("auto:tab-1") + }) +}) diff --git a/src/lib/speech-player.ts b/src/lib/speech-player.ts new file mode 100644 index 0000000000..4ec9f4801e --- /dev/null +++ b/src/lib/speech-player.ts @@ -0,0 +1,301 @@ +"use client" + +import { useSyncExternalStore } from "react" + +import { speechGetSettings, speechSynthesize } from "@/lib/api" +import { extractAppCommandError } from "@/lib/app-error" +import { chunkSpeakableText, toSpeakableText } from "@/lib/speakable-text" +import type { SpeakableLabels } from "@/lib/speakable-text" +import { + resolveOutputEngine, + waitForVoices, + type OutputEngineResolution, +} from "@/lib/speech-capabilities" +import { getSpeechPrefs } from "@/lib/speech-prefs" + +export type SpeechPlayerStatus = "idle" | "loading" | "playing" + +export interface SpeechPlayerState { + playingId: string | null + status: SpeechPlayerStatus +} + +export type SpeechPlaybackError = + | "auth" + | "not-configured" + | "unavailable" + | "failed" + +export interface SpeakOptions { + /** Omitted: resolved from the saved preference and what this device supports. */ + engine?: "browser" | "cloud" + language: string + labels: SpeakableLabels + onError?: (error: SpeechPlaybackError) => void +} + +const BROWSER_CHUNK = 220 +const CLOUD_CHUNK = 4000 +const IDLE: SpeechPlayerState = { playingId: null, status: "idle" } + +let state: SpeechPlayerState = IDLE +let generation = 0 +let audio: HTMLAudioElement | null = null +const objectUrls = new Set() +const listeners = new Set<() => void>() + +function setState(next: SpeechPlayerState) { + if (next.playingId === state.playingId && next.status === state.status) { + return + } + state = next + for (const listener of listeners) listener() +} + +export function getSpeechPlayerState(): SpeechPlayerState { + return state +} + +export function subscribeSpeechPlayer(listener: () => void): () => void { + listeners.add(listener) + return () => { + listeners.delete(listener) + } +} + +function getServerState(): SpeechPlayerState { + return IDLE +} + +export function useSpeechPlayer(): SpeechPlayerState { + return useSyncExternalStore( + subscribeSpeechPlayer, + getSpeechPlayerState, + getServerState + ) +} + +function revokeAll() { + for (const url of objectUrls) URL.revokeObjectURL(url) + objectUrls.clear() +} + +export function stopSpeech(): void { + generation += 1 + if (typeof window !== "undefined" && window.speechSynthesis) { + window.speechSynthesis.cancel() + } + if (audio) { + audio.onended = null + audio.onerror = null + audio.pause() + audio.removeAttribute("src") + audio.load() + } + revokeAll() + setState(IDLE) +} + +function classify(error: unknown): SpeechPlaybackError { + switch (extractAppCommandError(error)?.code) { + case "authentication_failed": + return "auth" + case "configuration_missing": + return "not-configured" + default: + return "failed" + } +} + +function pickVoice( + voices: SpeechSynthesisVoice[], + uri: string, + language: string +): SpeechSynthesisVoice | null { + if (uri) { + const chosen = voices.find((voice) => voice.voiceURI === uri) + if (chosen) return chosen + } + const lang = language.toLowerCase() + const base = lang.split("-")[0] + return ( + voices.find((voice) => voice.lang.toLowerCase() === lang) ?? + voices.find((voice) => voice.lang.toLowerCase().startsWith(base)) ?? + null + ) +} + +function speakBrowser( + id: string, + chunks: string[], + options: SpeakOptions, + run: number +) { + const synth = window.speechSynthesis + const { rate, browserVoiceUri } = getSpeechPrefs().output + const voice = pickVoice(synth.getVoices(), browserVoiceUri, options.language) + chunks.forEach((chunk, index) => { + const utterance = new SpeechSynthesisUtterance(chunk) + utterance.lang = options.language + utterance.rate = rate + if (voice) utterance.voice = voice + if (index === 0) { + utterance.onstart = () => { + if (run === generation) setState({ playingId: id, status: "playing" }) + } + } + if (index === chunks.length - 1) { + utterance.onend = () => { + if (run === generation) setState(IDLE) + } + } + utterance.onerror = (event) => { + if (run !== generation) return + if (event.error === "interrupted" || event.error === "canceled") return + stopSpeech() + options.onError?.("failed") + } + synth.speak(utterance) + }) +} + +async function fetchChunk( + chunk: string, + rate: number, + run: number +): Promise { + const { audioBase64, mimeType } = await speechSynthesize(chunk, rate) + if (run !== generation) return null + const bytes = Uint8Array.from(atob(audioBase64), (c) => c.charCodeAt(0)) + const url = URL.createObjectURL(new Blob([bytes], { type: mimeType })) + objectUrls.add(url) + return url +} + +async function speakCloud( + id: string, + chunks: string[], + options: SpeakOptions, + run: number +) { + const { rate } = getSpeechPrefs().output + audio ??= new Audio() + const player = audio + let next = fetchChunk(chunks[0], rate, run) + try { + for (let index = 0; index < chunks.length; index += 1) { + const url = await next + if (url === null || run !== generation) return + if (index + 1 < chunks.length) { + next = fetchChunk(chunks[index + 1], rate, run) + next.catch(() => {}) + } + await new Promise((resolve, reject) => { + player.onended = () => resolve() + player.onerror = () => reject(new Error("audio playback failed")) + player.src = url + player.play().then(() => { + if (run === generation) setState({ playingId: id, status: "playing" }) + }, reject) + }) + URL.revokeObjectURL(url) + objectUrls.delete(url) + if (run !== generation) return + } + setState(IDLE) + } catch (error) { + if (run !== generation) return + stopSpeech() + options.onError?.(classify(error)) + } +} + +async function resolvePlaybackEngine(): Promise { + const pref = getSpeechPrefs().output + const voices = pref.engine === "cloud" ? [] : await waitForVoices() + const browserTts = voices.length > 0 + let cloudConfigured = false + if (pref.engine === "cloud" || (pref.engine === "auto" && !browserTts)) { + cloudConfigured = (await speechGetSettings()).apiKeySet + } + return resolveOutputEngine(pref, { browserTts }, cloudConfigured) +} + +function start( + id: string, + markdown: string, + engine: "browser" | "cloud", + options: SpeakOptions, + run: number +) { + const text = toSpeakableText(markdown, options.labels) + const chunks = chunkSpeakableText( + text, + engine === "cloud" ? CLOUD_CHUNK : BROWSER_CHUNK + ) + if (chunks.length === 0) { + setState(IDLE) + return + } + if (engine === "browser") { + speakBrowser(id, chunks, options, run) + } else { + void speakCloud(id, chunks, options, run) + } +} + +export function speak(id: string, markdown: string, options: SpeakOptions) { + stopSpeech() + const run = generation + setState({ playingId: id, status: "loading" }) + if (options.engine) { + start(id, markdown, options.engine, options, run) + return + } + resolvePlaybackEngine().then( + (resolution) => { + if (run !== generation) return + if (resolution.engine === null) { + stopSpeech() + options.onError?.( + resolution.reason === "cloud-not-configured" + ? "not-configured" + : "unavailable" + ) + return + } + start(id, markdown, resolution.engine, options, run) + }, + (error: unknown) => { + if (run !== generation) return + stopSpeech() + options.onError?.(classify(error)) + } + ) +} + +export interface AutoReadContext { + contextKey: string + activeId: string | null + visibility: DocumentVisibilityState +} + +export function maybeAutoRead( + { contextKey, activeId, visibility }: AutoReadContext, + text: string, + options: SpeakOptions +): boolean { + const { enabled, autoRead } = getSpeechPrefs().output + if (!enabled || !autoRead) return false + if (contextKey !== activeId || visibility !== "visible") return false + if (!text.trim()) return false + speak(`auto:${contextKey}`, text, options) + return true +} + +export function resetSpeechPlayerForTests(): void { + stopSpeech() + audio = null + listeners.clear() + state = IDLE +} diff --git a/src/lib/speech-prefs.test.ts b/src/lib/speech-prefs.test.ts index 4e51d9333e..a5a33cacc6 100644 --- a/src/lib/speech-prefs.test.ts +++ b/src/lib/speech-prefs.test.ts @@ -23,6 +23,13 @@ describe("speech preferences", () => { engine: "auto", language: "", }, + output: { + enabled: false, + engine: "auto", + browserVoiceUri: "", + rate: 1, + autoRead: false, + }, }) expect(loadSpeechPrefs()).toEqual(DEFAULT_SPEECH_PREFS) }) @@ -40,11 +47,53 @@ describe("speech preferences", () => { engine: "cloud" as const, language: "zh-CN", }, + output: { + enabled: true, + engine: "browser" as const, + browserVoiceUri: "Google US English", + rate: 1.5, + autoRead: true, + }, } saveSpeechPrefs(custom) expect(loadSpeechPrefs()).toEqual(custom) }) + it("saving one section keeps the other", () => { + saveSpeechPrefs({ + output: { ...DEFAULT_SPEECH_PREFS.output, enabled: true, rate: 1.25 }, + }) + saveSpeechPrefs({ + input: { enabled: true, engine: "browser", language: "" }, + }) + const loaded = loadSpeechPrefs() + expect(loaded.input.enabled).toBe(true) + expect(loaded.output).toEqual({ + ...DEFAULT_SPEECH_PREFS.output, + enabled: true, + rate: 1.25, + }) + }) + + it("parses output per field and clamps the rate", () => { + expect( + parseSpeechPrefs({ + output: { + enabled: 1, + engine: "loud", + browserVoiceUri: null, + rate: "fast", + autoRead: "yes", + }, + }).output + ).toEqual(DEFAULT_SPEECH_PREFS.output) + expect(parseSpeechPrefs({ output: { rate: 9 } }).output.rate).toBe(2) + expect(parseSpeechPrefs({ output: { rate: 0.1 } }).output.rate).toBe(0.5) + expect(parseSpeechPrefs({ output: { rate: Number.NaN } }).output.rate).toBe( + 1 + ) + }) + it("falls back per-field for invalid or missing values", () => { const parsed = parseSpeechPrefs({ input: { diff --git a/src/lib/speech-prefs.ts b/src/lib/speech-prefs.ts index cac8f7f6f7..45ba6c9182 100644 --- a/src/lib/speech-prefs.ts +++ b/src/lib/speech-prefs.ts @@ -1,8 +1,7 @@ "use client" /** - * Speech preferences: speech-to-text input configuration including engine choice - * and language selection. + * Speech preferences: dictation (input) and read-aloud (output) settings. * * Stored in localStorage rather than the backend because speech input configuration * (microphone access, engine preferences) is per-device. Uses the same reactive @@ -26,16 +25,43 @@ export interface SpeechInputPrefs { language: string } +export interface SpeechOutputPrefs { + enabled: boolean + engine: SpeechEnginePreference + /** `SpeechSynthesisVoice.voiceURI`; empty means the default voice for the language. */ + browserVoiceUri: string + rate: number + autoRead: boolean +} + export interface SpeechPrefs { input: SpeechInputPrefs + output: SpeechOutputPrefs } +export const MIN_SPEECH_RATE = 0.5 +export const MAX_SPEECH_RATE = 2 + export const DEFAULT_SPEECH_PREFS: SpeechPrefs = { input: { enabled: false, engine: "auto", language: "", }, + output: { + enabled: false, + engine: "auto", + browserVoiceUri: "", + rate: 1, + autoRead: false, + }, +} + +function defaultPrefs(): SpeechPrefs { + return { + input: { ...DEFAULT_SPEECH_PREFS.input }, + output: { ...DEFAULT_SPEECH_PREFS.output }, + } } function isSpeechEnginePreference( @@ -44,6 +70,56 @@ function isSpeechEnginePreference( return value === "auto" || value === "browser" || value === "cloud" } +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" + ? (value as Record) + : null +} + +function parseInput(raw: unknown): SpeechInputPrefs { + const defaults = DEFAULT_SPEECH_PREFS.input + const source = asRecord(raw) + if (!source) return { ...defaults } + return { + enabled: + typeof source.enabled === "boolean" ? source.enabled : defaults.enabled, + engine: isSpeechEnginePreference(source.engine) + ? source.engine + : defaults.engine, + language: + typeof source.language === "string" ? source.language : defaults.language, + } +} + +export function clampSpeechRate(rate: number): number { + return Math.min(MAX_SPEECH_RATE, Math.max(MIN_SPEECH_RATE, rate)) +} + +function parseOutput(raw: unknown): SpeechOutputPrefs { + const defaults = DEFAULT_SPEECH_PREFS.output + const source = asRecord(raw) + if (!source) return { ...defaults } + return { + enabled: + typeof source.enabled === "boolean" ? source.enabled : defaults.enabled, + engine: isSpeechEnginePreference(source.engine) + ? source.engine + : defaults.engine, + browserVoiceUri: + typeof source.browserVoiceUri === "string" + ? source.browserVoiceUri + : defaults.browserVoiceUri, + rate: + typeof source.rate === "number" && Number.isFinite(source.rate) + ? clampSpeechRate(source.rate) + : defaults.rate, + autoRead: + typeof source.autoRead === "boolean" + ? source.autoRead + : defaults.autoRead, + } +} + /** * Merge a stored blob over the defaults, field by field. Every field is * validated independently so a partial write from an older build (or a @@ -51,51 +127,31 @@ function isSpeechEnginePreference( * discarding the whole preference set. */ export function parseSpeechPrefs(raw: unknown): SpeechPrefs { - const defaults = DEFAULT_SPEECH_PREFS - if (!raw || typeof raw !== "object") { - return { input: { ...defaults.input } } - } - const source = raw as Record - - const rawInput = source.input - if (!rawInput || typeof rawInput !== "object") { - return { input: { ...defaults.input } } - } - const inputSource = rawInput as Record - + const source = asRecord(raw) return { - input: { - enabled: - typeof inputSource.enabled === "boolean" - ? inputSource.enabled - : defaults.input.enabled, - engine: isSpeechEnginePreference(inputSource.engine) - ? inputSource.engine - : defaults.input.engine, - language: - typeof inputSource.language === "string" - ? inputSource.language - : defaults.input.language, - }, + input: parseInput(source?.input), + output: parseOutput(source?.output), } } export function loadSpeechPrefs(): SpeechPrefs { - const defaults = DEFAULT_SPEECH_PREFS - if (typeof window === "undefined") { - return { input: { ...defaults.input } } - } + if (typeof window === "undefined") return defaultPrefs() try { const raw = localStorage.getItem(PREFS_KEY) - if (!raw) return { input: { ...defaults.input } } + if (!raw) return defaultPrefs() return parseSpeechPrefs(JSON.parse(raw)) } catch { - return { input: { ...defaults.input } } + return defaultPrefs() } } -export function saveSpeechPrefs(prefs: SpeechPrefs): void { +/** Saves a partial update; omitted sections keep their current values. */ +export function saveSpeechPrefs(update: { + input?: SpeechInputPrefs + output?: SpeechOutputPrefs +}): void { if (typeof window === "undefined") return + const prefs = parseSpeechPrefs({ ...loadSpeechPrefs(), ...update }) try { localStorage.setItem(PREFS_KEY, JSON.stringify(prefs)) } catch { From f1f40aa8cb990be9ab3f3cec9c6cc055afd4f42b Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 17:18:33 +0700 Subject: [PATCH 11/15] feat(chat): add read-aloud action to agent replies Refs #844 --- src/components/message/message-list-view.tsx | 1 + src/components/message/turn-stats.test.tsx | 84 +++++++ src/components/message/turn-stats.tsx | 63 +++++ .../settings/speech-settings.test.tsx | 69 ++++++ src/components/settings/speech-settings.tsx | 229 +++++++++++++++++- src/i18n/messages/ar.json | 21 +- src/i18n/messages/de.json | 21 +- src/i18n/messages/en.json | 21 +- src/i18n/messages/es.json | 21 +- src/i18n/messages/fr.json | 21 +- src/i18n/messages/ja.json | 21 +- src/i18n/messages/ko.json | 21 +- src/i18n/messages/pt.json | 21 +- src/i18n/messages/zh-CN.json | 21 +- src/i18n/messages/zh-TW.json | 21 +- 15 files changed, 634 insertions(+), 22 deletions(-) diff --git a/src/components/message/message-list-view.tsx b/src/components/message/message-list-view.tsx index c41aa440da..ae1bd2f0fe 100644 --- a/src/components/message/message-list-view.tsx +++ b/src/components/message/message-list-view.tsx @@ -981,6 +981,7 @@ const HistoricalMessageGroup = memo(function HistoricalMessageGroup({ previousUserIndex={previousUserIndex} isResponseComplete={isResponseComplete} copyText={extractTextFromParts(group.parts)} + speechId={group.id} completedAt={group.completed_at} forkDisabled={forkDisabled || forkPointUnnamed} forkDisabledReason={forkPointUnnamed ? "unnamed" : "busy"} diff --git a/src/components/message/turn-stats.test.tsx b/src/components/message/turn-stats.test.tsx index 00fffb6936..86cf20eae5 100644 --- a/src/components/message/turn-stats.test.tsx +++ b/src/components/message/turn-stats.test.tsx @@ -10,7 +10,24 @@ vi.mock("./use-create-task-from-message", () => ({ useCreateTaskFromMessage: () => () => {}, })) +const playerSpies = vi.hoisted(() => ({ + speak: vi.fn(), + stopSpeech: vi.fn(), +})) +vi.mock("@/lib/speech-player", async (importOriginal) => { + const actual = await importOriginal() + playerSpies.speak.mockImplementation(actual.speak) + playerSpies.stopSpeech.mockImplementation(actual.stopSpeech) + return { ...actual, ...playerSpies } +}) + import { TurnStats } from "./turn-stats" +import { resetSpeechPlayerForTests } from "@/lib/speech-player" +import { + DEFAULT_SPEECH_PREFS, + resetSpeechPrefsCacheForTests, + saveSpeechPrefs, +} from "@/lib/speech-prefs" import { MessageScrollProvider } from "./message-scroll-context" import { ModelLabelProvider } from "./model-label-context" import type { ModelLabelResolver } from "@/hooks/use-model-labels" @@ -241,3 +258,70 @@ describe("TurnStats zeroed counters", () => { expect(screen.getByLabelText(tokenStatsLabel)).toBeInTheDocument() }) }) + +describe("TurnStats read aloud", () => { + const labels = enMessages.Folder.chat.messageList + + function setup(enabled: boolean) { + localStorage.clear() + resetSpeechPrefsCacheForTests() + resetSpeechPlayerForTests() + playerSpies.speak.mockClear() + playerSpies.stopSpeech.mockClear() + vi.stubGlobal("speechSynthesis", { + getVoices: () => [{ voiceURI: "v", lang: "en-US" }], + speak: vi.fn(), + cancel: vi.fn(), + }) + vi.stubGlobal( + "SpeechSynthesisUtterance", + class { + constructor(public text: string) {} + } + ) + saveSpeechPrefs({ output: { ...DEFAULT_SPEECH_PREFS.output, enabled } }) + } + + it("is hidden while read aloud is disabled", () => { + setup(false) + renderStats() + expect(screen.queryByLabelText(labels.readAloud)).not.toBeInTheDocument() + vi.unstubAllGlobals() + }) + + it("is hidden without text or on an unfinished reply", () => { + setup(true) + renderStats() + renderStats( + + ) + expect(screen.queryByLabelText(labels.readAloud)).not.toBeInTheDocument() + vi.unstubAllGlobals() + }) + + it("speaks the reply text, then stops on the second click", async () => { + setup(true) + const user = userEvent.setup() + renderStats() + + await user.click(screen.getByLabelText(labels.readAloud)) + expect(playerSpies.speak).toHaveBeenCalledWith( + "turn-1", + "Hello **there**", + expect.objectContaining({ language: "en-US" }) + ) + const active = screen.getByRole("button", { pressed: true }) + expect([labels.readAloudLoading, labels.stopReading]).toContain( + active.getAttribute("aria-label") + ) + + await user.click(active) + expect(playerSpies.stopSpeech).toHaveBeenCalled() + expect(screen.getByLabelText(labels.readAloud)).toBeInTheDocument() + vi.unstubAllGlobals() + }) +}) diff --git a/src/components/message/turn-stats.tsx b/src/components/message/turn-stats.tsx index 77e2d59637..b454b43f70 100644 --- a/src/components/message/turn-stats.tsx +++ b/src/components/message/turn-stats.tsx @@ -8,9 +8,13 @@ import { Coins, CopyIcon, ListTodo, + Loader2, Split, + Square, + Volume2, } from "lucide-react" import { useLocale, useTranslations } from "next-intl" +import { toast } from "sonner" import { Tooltip, TooltipContent, @@ -20,6 +24,9 @@ import { import { useMessageScroll } from "@/components/message/message-scroll-context" import { useModelLabel } from "@/components/message/model-label-context" import { useCreateTaskFromMessage } from "./use-create-task-from-message" +import { resolveSpeechLanguage } from "@/lib/speech-capabilities" +import { speak, stopSpeech, useSpeechPlayer } from "@/lib/speech-player" +import { useSpeechPrefs } from "@/lib/speech-prefs" import { formatTokenCount } from "@/lib/token-format" import { cn, copyTextToClipboard } from "@/lib/utils" import type { TurnUsage } from "@/lib/types" @@ -46,6 +53,8 @@ interface TurnStatsProps { * name the backend can resolve yet (`unnamed` — the post-turn reparse fills * it in a moment later). Only read while `forkDisabled`. */ forkDisabledReason?: "busy" | "unnamed" + /** Stable id of this reply; enables the read-aloud action when set. */ + speechId?: string } const iconButtonClass = @@ -63,6 +72,7 @@ export function TurnStats({ onForkFromHere, forkDisabled = false, forkDisabledReason = "busy", + speechId, }: TurnStatsProps) { const locale = useLocale() const t = useTranslations("Folder.chat.messageList") @@ -110,6 +120,16 @@ export function TurnStats({ (id) => modelLabel(id) ?? id ) const hasCopy = copyText.trim().length > 0 + const speechPrefs = useSpeechPrefs() + const readAloudEnabled = speechPrefs.output.enabled + const player = useSpeechPlayer() + const hasReadAloud = readAloudEnabled && hasCopy && Boolean(speechId) + const isThisSpeaking = hasReadAloud && player.playingId === speechId + const readAloudLabel = isThisSpeaking + ? player.status === "loading" + ? t("readAloudLoading") + : t("stopReading") + : t("readAloud") const hasUsage = Boolean(usage) // An all-zero usage means "nobody said", not "nothing was spent": a reply // that exists cannot have cost zero tokens. Qoder zeroes every counter for @@ -157,6 +177,24 @@ export function TurnStats({ timeoutRef.current = window.setTimeout(() => setIsCopied(false), 2000) }, [copyText, hasCopy, isCopied]) + const handleReadAloud = useCallback(() => { + if (!speechId) return + if (isThisSpeaking) { + stopSpeech() + return + } + speak(speechId, copyText, { + language: resolveSpeechLanguage(speechPrefs.input, locale), + labels: { + codeOmitted: t("speechCodeOmitted"), + tableOmitted: t("speechTableOmitted"), + }, + onError: () => { + toast.error(t("readAloudFailed")) + }, + }) + }, [copyText, isThisSpeaking, locale, speechId, speechPrefs.input, t]) + useEffect( () => () => { window.clearTimeout(timeoutRef.current) @@ -194,6 +232,31 @@ export function TurnStats({ )} + {hasReadAloud && ( + + + + + {readAloudLabel} + + )} {hasCopy && ( diff --git a/src/components/settings/speech-settings.test.tsx b/src/components/settings/speech-settings.test.tsx index 83ada360e3..1d4fdf3fab 100644 --- a/src/components/settings/speech-settings.test.tsx +++ b/src/components/settings/speech-settings.test.tsx @@ -180,3 +180,72 @@ describe("SpeechSettings", () => { ) }) }) + +describe("SpeechSettings read aloud", () => { + const m = enMessages.SpeechSettings + + afterEach(() => vi.unstubAllGlobals()) + + it("enables read aloud and shows its controls", async () => { + vi.stubGlobal("speechSynthesis", { + getVoices: () => [ + { voiceURI: "fr", name: "Amelie", lang: "fr-FR" }, + { voiceURI: "en", name: "Samantha", lang: "en-US" }, + ], + }) + const user = userEvent.setup() + renderPage() + await user.click(await screen.findByLabelText(m.outputTitle)) + expect(getSpeechPrefs().output.enabled).toBe(true) + expect(await screen.findByTestId("speech-output-status")).toHaveTextContent( + "Using: Browser" + ) + expect(screen.getByLabelText(m.rateLabel)).toBeInTheDocument() + expect(screen.getByLabelText(m.voiceLabel)).toBeInTheDocument() + + await user.click(screen.getByLabelText(m.autoReadLabel)) + expect(getSpeechPrefs().output.autoRead).toBe(true) + + await user.click(screen.getByLabelText(m.outputTitle)) + expect(getSpeechPrefs().output).toMatchObject({ + enabled: false, + autoRead: false, + }) + }) + + it("reports missing voices and key when nothing can speak", async () => { + vi.stubGlobal("speechSynthesis", undefined) + saveSpeechPrefs({ + output: { + enabled: true, + engine: "auto", + browserVoiceUri: "", + rate: 1, + autoRead: false, + }, + }) + renderPage() + expect(await screen.findByTestId("speech-output-status")).toHaveTextContent( + m.reasonCloudNotConfigured + ) + expect(screen.queryByLabelText(m.voiceLabel)).not.toBeInTheDocument() + }) + + it("saves the TTS model and voice with the cloud settings", async () => { + updateSettings.mockResolvedValue(view(false)) + const user = userEvent.setup() + renderPage() + const model = await screen.findByLabelText(m.ttsModel) + await user.clear(model) + await user.type(model, "gpt-4o-mini-tts") + const voice = screen.getByLabelText(m.ttsVoice) + await user.clear(voice) + await user.type(voice, "nova") + await user.click(screen.getByRole("button", { name: m.save })) + await waitFor(() => expect(updateSettings).toHaveBeenCalled()) + expect(updateSettings.mock.calls[0][0]).toMatchObject({ + ttsModel: "gpt-4o-mini-tts", + ttsVoice: "nova", + }) + }) +}) diff --git a/src/components/settings/speech-settings.tsx b/src/components/settings/speech-settings.tsx index cbce0ca43b..61d2bbefb8 100644 --- a/src/components/settings/speech-settings.tsx +++ b/src/components/settings/speech-settings.tsx @@ -14,8 +14,12 @@ import { KeyRound, Languages, Link, + Gauge, Loader2, Mic, + Speaker, + Volume2, + Wand2, } from "lucide-react" import { useLocale, useTranslations } from "next-intl" import { toast } from "sonner" @@ -36,6 +40,7 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select" +import { Slider } from "@/components/ui/slider" import { Switch } from "@/components/ui/switch" import { speechGetSettings, speechUpdateSettings } from "@/lib/api" import { toErrorMessage } from "@/lib/app-error" @@ -43,8 +48,13 @@ import { LOCALE_TO_BCP47, detectSpeechCapabilities, resolveInputEngine, + resolveOutputEngine, + resolveSpeechLanguage, + waitForVoices, } from "@/lib/speech-capabilities" import { + MAX_SPEECH_RATE, + MIN_SPEECH_RATE, saveSpeechPrefs, useSpeechPrefs, type SpeechEnginePreference, @@ -52,6 +62,7 @@ import { import type { SpeechCloudSettings } from "@/lib/types" const LANGUAGE_FOLLOW_APP = "follow-app" +const VOICE_DEFAULT = "default" const LANGUAGE_CUSTOM = "custom" const LANGUAGE_TAGS = Array.from(new Set(Object.values(LOCALE_TO_BCP47))) @@ -62,6 +73,11 @@ const REASON_KEYS = { "cloud-not-configured": "reasonCloudNotConfigured", } as const +const OUTPUT_REASON_KEYS = { + "no-engine": "reasonNoVoices", + "cloud-not-configured": "reasonCloudNotConfigured", +} as const + const subscribeNever = () => () => {} const onClient = () => true const onServer = () => false @@ -86,6 +102,7 @@ export function SpeechSettings() { const [apiKeyDraft, setApiKeyDraft] = useState("") const [saving, setSaving] = useState(false) const [customLanguageMode, setCustomLanguageMode] = useState(false) + const [voices, setVoices] = useState(null) const mounted = useSyncExternalStore(subscribeNever, onClient, onServer) const caps = useMemo( @@ -116,11 +133,60 @@ export function SpeechSettings() { const input = prefs.input const updateInput = useCallback( (patch: Partial) => { - saveSpeechPrefs({ ...prefs, input: { ...input, ...patch } }) + saveSpeechPrefs({ input: { ...input, ...patch } }) + }, + [input] + ) + + const output = prefs.output + const updateOutput = useCallback( + (patch: Partial) => { + saveSpeechPrefs({ output: { ...output, ...patch } }) }, - [input, prefs] + [output] ) + const outputOn = output.enabled + useEffect(() => { + if (!outputOn) return + let alive = true + void waitForVoices().then((list) => { + if (alive) setVoices(list) + }) + return () => { + alive = false + } + }, [outputOn]) + + const speechLanguage = resolveSpeechLanguage(input, locale) + const sortedVoices = useMemo(() => { + if (!voices) return [] + const base = speechLanguage.toLowerCase().split("-")[0] + const matches = (voice: SpeechSynthesisVoice) => + voice.lang.toLowerCase().split("-")[0] === base + return [ + ...voices.filter(matches), + ...voices.filter((voice) => !matches(voice)), + ] + }, [speechLanguage, voices]) + + const outputStatus = useMemo(() => { + if (voices === null) return null + const resolution = resolveOutputEngine( + output, + { browserTts: voices.length > 0 }, + apiKeySet + ) + if (resolution.engine === null) { + return t(OUTPUT_REASON_KEYS[resolution.reason]) + } + return t("engineUsing", { + engine: t( + resolution.engine === "browser" ? "engineBrowser" : "engineCloud" + ), + }) + }, [apiKeySet, output, t, voices]) + const engineStatus = useMemo(() => { if (!caps) return null const resolution = resolveInputEngine(input, caps, apiKeySet) @@ -291,6 +357,135 @@ export function SpeechSettings() { )} + + updateOutput( + enabled ? { enabled } : { enabled, autoRead: false } + ) + } + /> + } + > + {output.enabled && ( + + + {outputStatus} + + ) : undefined + } + htmlFor="speech-output-engine" + control={ + + } + /> + {output.engine !== "cloud" && sortedVoices.length > 0 && ( + + updateOutput({ + browserVoiceUri: uri === VOICE_DEFAULT ? "" : uri, + }) + } + > + + + + + + {t("voiceDefault")} + + {sortedVoices.map((voice) => ( + + {voice.name} ({voice.lang}) + + ))} + + + } + /> + )} + + {output.rate.toFixed(2)}x + + } + > + updateOutput({ rate })} + /> + + updateOutput({ autoRead })} + /> + } + /> + + )} + + + + + setCloud({ ...cloud, ttsModel: e.target.value }) + } + spellCheck={false} + /> + + + + setCloud({ ...cloud, ttsVoice: e.target.value }) + } + spellCheck={false} + /> + diff --git a/src/i18n/messages/ar.json b/src/i18n/messages/ar.json index a4547d97e8..7a13c34049 100644 --- a/src/i18n/messages/ar.json +++ b/src/i18n/messages/ar.json @@ -3460,7 +3460,13 @@ "completedAt": "وقت الإنجاز", "jumpToPreviousUserMessage": "الانتقال إلى رسالة المستخدم", "showMore": "عرض المزيد", - "showLess": "طي" + "showLess": "طي", + "readAloud": "قراءة بصوت عالٍ", + "stopReading": "إيقاف القراءة", + "readAloudLoading": "جارٍ تجهيز الصوت...", + "readAloudFailed": "تعذّرت قراءة هذا الرد بصوت عالٍ", + "speechCodeOmitted": "تم حذف كتلة التعليمات البرمجية", + "speechTableOmitted": "تم حذف الجدول" }, "liveTurnStats": { "thinking": "جارٍ التفكير...", @@ -6579,6 +6585,17 @@ "save": "حفظ", "saving": "جارٍ الحفظ...", "saved": "تم حفظ إعدادات الصوت", - "saveFailed": "تعذّر حفظ إعدادات الصوت: {message}" + "saveFailed": "تعذّر حفظ إعدادات الصوت: {message}", + "outputTitle": "القراءة بصوت عالٍ", + "outputDescription": "إظهار زر مكبّر صوت على ردود الوكيل لقراءتها.", + "outputEngineLabel": "محرك الصوت", + "reasonNoVoices": "غير متاح: لا توجد أصوات في هذا المتصفح. اضبط الخدمة السحابية أدناه.", + "voiceLabel": "الصوت", + "voiceDefault": "الصوت الافتراضي للغة", + "rateLabel": "سرعة القراءة", + "autoReadLabel": "قراءة الردود تلقائيًا", + "autoReadDescription": "قراءة كل رد مكتمل في علامة التبويب النشطة بصوت عالٍ.", + "ttsModel": "نموذج تحويل النص إلى كلام", + "ttsVoice": "صوت تحويل النص إلى كلام" } } diff --git a/src/i18n/messages/de.json b/src/i18n/messages/de.json index 9a57796a8a..c4ef54aaef 100644 --- a/src/i18n/messages/de.json +++ b/src/i18n/messages/de.json @@ -3460,7 +3460,13 @@ "completedAt": "Abgeschlossen um", "jumpToPreviousUserMessage": "Zur Benutzernachricht springen", "showMore": "Mehr anzeigen", - "showLess": "Weniger anzeigen" + "showLess": "Weniger anzeigen", + "readAloud": "Vorlesen", + "stopReading": "Vorlesen beenden", + "readAloudLoading": "Audio wird vorbereitet...", + "readAloudFailed": "Diese Antwort konnte nicht vorgelesen werden", + "speechCodeOmitted": "Codeblock ausgelassen", + "speechTableOmitted": "Tabelle ausgelassen" }, "liveTurnStats": { "thinking": "Denkt nach...", @@ -6579,6 +6585,17 @@ "save": "Speichern", "saving": "Wird gespeichert...", "saved": "Spracheinstellungen gespeichert", - "saveFailed": "Spracheinstellungen konnten nicht gespeichert werden: {message}" + "saveFailed": "Spracheinstellungen konnten nicht gespeichert werden: {message}", + "outputTitle": "Vorlesen", + "outputDescription": "Zeigt bei Agent-Antworten eine Lautsprecher-Schaltfläche zum Vorlesen.", + "outputEngineLabel": "Sprachausgabe", + "reasonNoVoices": "Nicht verfügbar: Dieser Browser hat keine Stimmen. Konfiguriere unten den Cloud-Dienst.", + "voiceLabel": "Stimme", + "voiceDefault": "Standardstimme der Sprache", + "rateLabel": "Sprechgeschwindigkeit", + "autoReadLabel": "Antworten automatisch vorlesen", + "autoReadDescription": "Liest jede fertige Antwort im aktiven Tab vor.", + "ttsModel": "Text-to-Speech-Modell", + "ttsVoice": "Text-to-Speech-Stimme" } } diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index 8e04fb6f13..abc35472cc 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -3460,7 +3460,13 @@ "completedAt": "Completed at", "jumpToPreviousUserMessage": "Jump to user message", "showMore": "Show more", - "showLess": "Show less" + "showLess": "Show less", + "readAloud": "Read aloud", + "stopReading": "Stop reading", + "readAloudLoading": "Preparing audio...", + "readAloudFailed": "Couldn't read this reply aloud", + "speechCodeOmitted": "Code block omitted", + "speechTableOmitted": "Table omitted" }, "liveTurnStats": { "thinking": "Thinking...", @@ -6579,6 +6585,17 @@ "save": "Save", "saving": "Saving...", "saved": "Speech settings saved", - "saveFailed": "Failed to save speech settings: {message}" + "saveFailed": "Failed to save speech settings: {message}", + "outputTitle": "Read aloud", + "outputDescription": "Show a speaker button on agent replies that reads them out.", + "outputEngineLabel": "Voice engine", + "reasonNoVoices": "Unavailable: this browser has no speech voices. Configure the cloud service below.", + "voiceLabel": "Voice", + "voiceDefault": "Default voice for the language", + "rateLabel": "Speaking rate", + "autoReadLabel": "Read replies automatically", + "autoReadDescription": "Read each finished reply in the active tab aloud.", + "ttsModel": "Text-to-speech model", + "ttsVoice": "Text-to-speech voice" } } diff --git a/src/i18n/messages/es.json b/src/i18n/messages/es.json index 8b067fbef7..ec80bc862d 100644 --- a/src/i18n/messages/es.json +++ b/src/i18n/messages/es.json @@ -3460,7 +3460,13 @@ "completedAt": "Completado a las", "jumpToPreviousUserMessage": "Ir al mensaje del usuario", "showMore": "Mostrar más", - "showLess": "Mostrar menos" + "showLess": "Mostrar menos", + "readAloud": "Leer en voz alta", + "stopReading": "Dejar de leer", + "readAloudLoading": "Preparando audio...", + "readAloudFailed": "No se pudo leer esta respuesta en voz alta", + "speechCodeOmitted": "Bloque de código omitido", + "speechTableOmitted": "Tabla omitida" }, "liveTurnStats": { "thinking": "Pensando...", @@ -6579,6 +6585,17 @@ "save": "Guardar", "saving": "Guardando...", "saved": "Ajustes de voz guardados", - "saveFailed": "No se pudieron guardar los ajustes de voz: {message}" + "saveFailed": "No se pudieron guardar los ajustes de voz: {message}", + "outputTitle": "Leer en voz alta", + "outputDescription": "Muestra un botón de altavoz en las respuestas del agente para leerlas.", + "outputEngineLabel": "Motor de voz", + "reasonNoVoices": "No disponible: este navegador no tiene voces. Configura el servicio en la nube abajo.", + "voiceLabel": "Voz", + "voiceDefault": "Voz predeterminada del idioma", + "rateLabel": "Velocidad de lectura", + "autoReadLabel": "Leer respuestas automáticamente", + "autoReadDescription": "Lee en voz alta cada respuesta terminada en la pestaña activa.", + "ttsModel": "Modelo de texto a voz", + "ttsVoice": "Voz de texto a voz" } } diff --git a/src/i18n/messages/fr.json b/src/i18n/messages/fr.json index 91fb7388df..eb4b3b5c3b 100644 --- a/src/i18n/messages/fr.json +++ b/src/i18n/messages/fr.json @@ -3460,7 +3460,13 @@ "completedAt": "Terminé à", "jumpToPreviousUserMessage": "Aller au message utilisateur", "showMore": "Afficher plus", - "showLess": "Afficher moins" + "showLess": "Afficher moins", + "readAloud": "Lire à voix haute", + "stopReading": "Arrêter la lecture", + "readAloudLoading": "Préparation de l'audio...", + "readAloudFailed": "Impossible de lire cette réponse à voix haute", + "speechCodeOmitted": "Bloc de code omis", + "speechTableOmitted": "Tableau omis" }, "liveTurnStats": { "thinking": "Réflexion...", @@ -6579,6 +6585,17 @@ "save": "Enregistrer", "saving": "Enregistrement...", "saved": "Paramètres vocaux enregistrés", - "saveFailed": "Échec de l'enregistrement des paramètres vocaux : {message}" + "saveFailed": "Échec de l'enregistrement des paramètres vocaux : {message}", + "outputTitle": "Lecture à voix haute", + "outputDescription": "Affiche un bouton haut-parleur sur les réponses de l'agent pour les lire.", + "outputEngineLabel": "Moteur vocal", + "reasonNoVoices": "Indisponible : ce navigateur n'a pas de voix. Configurez le service cloud ci-dessous.", + "voiceLabel": "Voix", + "voiceDefault": "Voix par défaut de la langue", + "rateLabel": "Vitesse de lecture", + "autoReadLabel": "Lire les réponses automatiquement", + "autoReadDescription": "Lit à voix haute chaque réponse terminée dans l'onglet actif.", + "ttsModel": "Modèle de synthèse vocale", + "ttsVoice": "Voix de synthèse vocale" } } diff --git a/src/i18n/messages/ja.json b/src/i18n/messages/ja.json index 1bbc030adc..3549f73517 100644 --- a/src/i18n/messages/ja.json +++ b/src/i18n/messages/ja.json @@ -3460,7 +3460,13 @@ "completedAt": "完了時刻", "jumpToPreviousUserMessage": "前のユーザーメッセージへ", "showMore": "もっと見る", - "showLess": "折りたたむ" + "showLess": "折りたたむ", + "readAloud": "読み上げ", + "stopReading": "読み上げを停止", + "readAloudLoading": "音声を準備中...", + "readAloudFailed": "この返信を読み上げられませんでした", + "speechCodeOmitted": "コードブロックは省略", + "speechTableOmitted": "表は省略" }, "liveTurnStats": { "thinking": "考え中...", @@ -6579,6 +6585,17 @@ "save": "保存", "saving": "保存中...", "saved": "音声設定を保存しました", - "saveFailed": "音声設定の保存に失敗しました: {message}" + "saveFailed": "音声設定の保存に失敗しました: {message}", + "outputTitle": "読み上げ", + "outputDescription": "エージェントの返信にスピーカーボタンを表示し、読み上げます。", + "outputEngineLabel": "音声エンジン", + "reasonNoVoices": "利用不可：このブラウザーには音声がありません。下のクラウドサービスを設定してください。", + "voiceLabel": "音声", + "voiceDefault": "言語の既定の音声", + "rateLabel": "読み上げ速度", + "autoReadLabel": "返信を自動で読み上げ", + "autoReadDescription": "アクティブなタブで完了した返信を自動で読み上げます。", + "ttsModel": "音声合成モデル", + "ttsVoice": "音声合成ボイス" } } diff --git a/src/i18n/messages/ko.json b/src/i18n/messages/ko.json index 7d9e3660cf..5bdc5af2a8 100644 --- a/src/i18n/messages/ko.json +++ b/src/i18n/messages/ko.json @@ -3460,7 +3460,13 @@ "completedAt": "완료 시각", "jumpToPreviousUserMessage": "이전 사용자 메시지로 이동", "showMore": "더보기", - "showLess": "접기" + "showLess": "접기", + "readAloud": "소리 내어 읽기", + "stopReading": "읽기 중지", + "readAloudLoading": "오디오 준비 중...", + "readAloudFailed": "이 답변을 읽을 수 없습니다", + "speechCodeOmitted": "코드 블록 생략", + "speechTableOmitted": "표 생략" }, "liveTurnStats": { "thinking": "생각 중...", @@ -6579,6 +6585,17 @@ "save": "저장", "saving": "저장 중...", "saved": "음성 설정이 저장되었습니다", - "saveFailed": "음성 설정을 저장하지 못했습니다: {message}" + "saveFailed": "음성 설정을 저장하지 못했습니다: {message}", + "outputTitle": "소리 내어 읽기", + "outputDescription": "에이전트 답변에 스피커 버튼을 표시해 답변을 읽어 줍니다.", + "outputEngineLabel": "음성 엔진", + "reasonNoVoices": "사용 불가: 이 브라우저에 음성이 없습니다. 아래에서 클라우드 서비스를 구성하세요.", + "voiceLabel": "음성", + "voiceDefault": "언어 기본 음성", + "rateLabel": "읽기 속도", + "autoReadLabel": "답변 자동 읽기", + "autoReadDescription": "활성 탭에서 완료된 답변을 자동으로 읽어 줍니다.", + "ttsModel": "텍스트 음성 변환 모델", + "ttsVoice": "텍스트 음성 변환 음성" } } diff --git a/src/i18n/messages/pt.json b/src/i18n/messages/pt.json index 0dc3bf0813..37e5bf83f5 100644 --- a/src/i18n/messages/pt.json +++ b/src/i18n/messages/pt.json @@ -3460,7 +3460,13 @@ "completedAt": "Concluído às", "jumpToPreviousUserMessage": "Ir para a mensagem do usuário", "showMore": "Mostrar mais", - "showLess": "Mostrar menos" + "showLess": "Mostrar menos", + "readAloud": "Ler em voz alta", + "stopReading": "Parar leitura", + "readAloudLoading": "Preparando áudio...", + "readAloudFailed": "Não foi possível ler esta resposta em voz alta", + "speechCodeOmitted": "Bloco de código omitido", + "speechTableOmitted": "Tabela omitida" }, "liveTurnStats": { "thinking": "Pensando...", @@ -6579,6 +6585,17 @@ "save": "Salvar", "saving": "Salvando...", "saved": "Configurações de voz salvas", - "saveFailed": "Falha ao salvar as configurações de voz: {message}" + "saveFailed": "Falha ao salvar as configurações de voz: {message}", + "outputTitle": "Ler em voz alta", + "outputDescription": "Mostra um botão de alto-falante nas respostas do agente para lê-las.", + "outputEngineLabel": "Mecanismo de voz", + "reasonNoVoices": "Indisponível: este navegador não tem vozes. Configure o serviço em nuvem abaixo.", + "voiceLabel": "Voz", + "voiceDefault": "Voz padrão do idioma", + "rateLabel": "Velocidade de leitura", + "autoReadLabel": "Ler respostas automaticamente", + "autoReadDescription": "Lê em voz alta cada resposta concluída na aba ativa.", + "ttsModel": "Modelo de texto para fala", + "ttsVoice": "Voz de texto para fala" } } diff --git a/src/i18n/messages/zh-CN.json b/src/i18n/messages/zh-CN.json index b8a0dd4562..156de5e2ee 100644 --- a/src/i18n/messages/zh-CN.json +++ b/src/i18n/messages/zh-CN.json @@ -3460,7 +3460,13 @@ "completedAt": "完成时间", "jumpToPreviousUserMessage": "跳转到上一条用户消息", "showMore": "展开", - "showLess": "收起" + "showLess": "收起", + "readAloud": "朗读", + "stopReading": "停止朗读", + "readAloudLoading": "正在准备音频...", + "readAloudFailed": "无法朗读此回复", + "speechCodeOmitted": "已省略代码块", + "speechTableOmitted": "已省略表格" }, "liveTurnStats": { "thinking": "思考中...", @@ -6579,6 +6585,17 @@ "save": "保存", "saving": "正在保存...", "saved": "语音设置已保存", - "saveFailed": "保存语音设置失败：{message}" + "saveFailed": "保存语音设置失败：{message}", + "outputTitle": "朗读", + "outputDescription": "在智能体回复上显示扬声器按钮，用于朗读回复。", + "outputEngineLabel": "语音引擎", + "reasonNoVoices": "不可用：此浏览器没有语音。请在下方配置云端服务。", + "voiceLabel": "声音", + "voiceDefault": "该语言的默认声音", + "rateLabel": "语速", + "autoReadLabel": "自动朗读回复", + "autoReadDescription": "在当前标签页中自动朗读每条完成的回复。", + "ttsModel": "文本转语音模型", + "ttsVoice": "文本转语音声音" } } diff --git a/src/i18n/messages/zh-TW.json b/src/i18n/messages/zh-TW.json index 441ffced45..25a8a0fb7e 100644 --- a/src/i18n/messages/zh-TW.json +++ b/src/i18n/messages/zh-TW.json @@ -3460,7 +3460,13 @@ "completedAt": "完成時間", "jumpToPreviousUserMessage": "跳轉到上一條使用者訊息", "showMore": "展開", - "showLess": "收合" + "showLess": "收合", + "readAloud": "朗讀", + "stopReading": "停止朗讀", + "readAloudLoading": "正在準備音訊...", + "readAloudFailed": "無法朗讀此回覆", + "speechCodeOmitted": "已省略程式碼區塊", + "speechTableOmitted": "已省略表格" }, "liveTurnStats": { "thinking": "思考中...", @@ -6579,6 +6585,17 @@ "save": "儲存", "saving": "正在儲存...", "saved": "語音設定已儲存", - "saveFailed": "儲存語音設定失敗：{message}" + "saveFailed": "儲存語音設定失敗：{message}", + "outputTitle": "朗讀", + "outputDescription": "在代理回覆上顯示揚聲器按鈕，用於朗讀回覆。", + "outputEngineLabel": "語音引擎", + "reasonNoVoices": "無法使用：此瀏覽器沒有語音。請在下方設定雲端服務。", + "voiceLabel": "聲音", + "voiceDefault": "該語言的預設聲音", + "rateLabel": "語速", + "autoReadLabel": "自動朗讀回覆", + "autoReadDescription": "在目前分頁中自動朗讀每則完成的回覆。", + "ttsModel": "文字轉語音模型", + "ttsVoice": "文字轉語音聲音" } } From c6948a840bc30c1ac5b9271a23618728d564159a Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 17:18:33 +0700 Subject: [PATCH 12/15] feat(chat): auto-read agent replies at turn end Refs #844 --- .../chat/composer/use-speech-input.test.ts | 29 ++++- .../chat/composer/use-speech-input.ts | 3 + src/components/chat/message-input.test.tsx | 66 ++++++++++ src/components/chat/message-input.tsx | 4 + src/contexts/acp-connections-context.test.tsx | 120 ++++++++++++++++++ src/contexts/acp-connections-context.tsx | 46 ++++++- 6 files changed, 265 insertions(+), 3 deletions(-) diff --git a/src/components/chat/composer/use-speech-input.test.ts b/src/components/chat/composer/use-speech-input.test.ts index d19c528b02..30b0fa90ea 100644 --- a/src/components/chat/composer/use-speech-input.test.ts +++ b/src/components/chat/composer/use-speech-input.test.ts @@ -13,6 +13,13 @@ vi.mock("@/lib/api", () => ({ let prefs: SpeechPrefs = { input: { enabled: true, engine: "auto", language: "" }, + output: { + enabled: false, + engine: "auto", + browserVoiceUri: "", + rate: 1, + autoRead: false, + }, } vi.mock("@/lib/speech-prefs", () => ({ useSpeechPrefs: () => prefs })) @@ -110,7 +117,16 @@ function renderSpeech() { } beforeEach(() => { - prefs = { input: { enabled: true, engine: "auto", language: "" } } + prefs = { + input: { enabled: true, engine: "auto", language: "" }, + output: { + enabled: false, + engine: "auto", + browserVoiceUri: "", + rate: 1, + autoRead: false, + }, + } caps = { browserStt: true, mediaCapture: true, secureContext: true } FakeRecognition.instances = [] FakeRecorder.instances = [] @@ -202,7 +218,16 @@ describe("useSpeechInput - browser engine", () => { describe("useSpeechInput - cloud engine", () => { beforeEach(() => { - prefs = { input: { enabled: true, engine: "cloud", language: "de-DE" } } + prefs = { + input: { enabled: true, engine: "cloud", language: "de-DE" }, + output: { + enabled: false, + engine: "auto", + browserVoiceUri: "", + rate: 1, + autoRead: false, + }, + } }) it("records, transcribes with the bare mime type and language, then inserts", async () => { diff --git a/src/components/chat/composer/use-speech-input.ts b/src/components/chat/composer/use-speech-input.ts index b4df60a275..6b0921fe37 100644 --- a/src/components/chat/composer/use-speech-input.ts +++ b/src/components/chat/composer/use-speech-input.ts @@ -11,6 +11,7 @@ import { import { useLocale } from "next-intl" import { speechGetSettings, speechTranscribe } from "@/lib/api" +import { stopSpeech } from "@/lib/speech-player" import { extractAppCommandError } from "@/lib/app-error" import { detectSpeechCapabilities, @@ -391,6 +392,8 @@ export function useSpeechInput({ const start = useCallback(() => { if (busyRef.current) return busyRef.current = true + // The mic must never record the speaker. + stopSpeech() generationRef.current += 1 const generation = generationRef.current const { diff --git a/src/components/chat/message-input.test.tsx b/src/components/chat/message-input.test.tsx index 28629097bb..7a39ecdad3 100644 --- a/src/components/chat/message-input.test.tsx +++ b/src/components/chat/message-input.test.tsx @@ -223,6 +223,7 @@ vi.mock("./composer/use-speech-input", () => ({ })) import enMessages from "@/i18n/messages/en.json" +import * as speechPlayer from "@/lib/speech-player" import { resetSpeechPrefsCacheForTests, saveSpeechPrefs, @@ -2503,3 +2504,68 @@ describe("MessageInput voice input", () => { expect(onSend).not.toHaveBeenCalled() }) }) + +describe("MessageInput read-aloud stop triggers", () => { + afterEach(() => { + cleanup() + composerHandle.current = null + vi.restoreAllMocks() + vi.unstubAllGlobals() + speechPlayer.resetSpeechPlayerForTests() + }) + + async function mountWithPlayback(onSend = vi.fn()) { + vi.stubGlobal("speechSynthesis", { + getVoices: () => [], + speak: vi.fn(), + cancel: vi.fn(), + }) + vi.stubGlobal( + "SpeechSynthesisUtterance", + class { + constructor(public text: string) {} + } + ) + renderInput({ onSend }) + await waitFor(() => + expect(composerHandle.current?.getEditor()).toBeTruthy() + ) + speechPlayer.speak("turn-1", "A reply.", { + engine: "browser", + language: "en-US", + labels: { codeOmitted: "", tableOmitted: "" }, + }) + expect(speechPlayer.getSpeechPlayerState().status).not.toBe("idle") + const stop = vi.spyOn(speechPlayer, "stopSpeech") + return { editor: composerHandle.current!.getEditor()!, stop } + } + + it("stops reading when the user types", async () => { + const { editor, stop } = await mountWithPlayback() + act(() => { + editor.commands.insertContent("x") + }) + expect(stop).toHaveBeenCalled() + expect(speechPlayer.getSpeechPlayerState().status).toBe("idle") + }) + + it("stops reading when the user sends", async () => { + const onSend = vi.fn() + const { editor, stop } = await mountWithPlayback(onSend) + act(() => { + editor.commands.insertContent("next question") + }) + stop.mockClear() + speechPlayer.speak("turn-1", "A reply.", { + engine: "browser", + language: "en-US", + labels: { codeOmitted: "", tableOmitted: "" }, + }) + await userEvent + .setup() + .click(screen.getByTitle(enMessages.Folder.chat.messageInput.send)) + await waitFor(() => expect(onSend).toHaveBeenCalled()) + expect(stop).toHaveBeenCalled() + expect(speechPlayer.getSpeechPlayerState().status).toBe("idle") + }) +}) diff --git a/src/components/chat/message-input.tsx b/src/components/chat/message-input.tsx index 14356b5600..20debe1eb7 100644 --- a/src/components/chat/message-input.tsx +++ b/src/components/chat/message-input.tsx @@ -124,6 +124,7 @@ import { import { textToInlineContent } from "@/components/chat/composer/plain-text-content" import { ComposerSpeechControl } from "@/components/chat/composer/composer-speech-button" import { useSpeechPrefs } from "@/lib/speech-prefs" +import { getSpeechPlayerState, stopSpeech } from "@/lib/speech-player" import { isEmbeddedReferenceUri } from "@/components/chat/composer/reference-uri" import { applyExpertReference, @@ -812,6 +813,8 @@ export function MessageInput({ }, [skillPrefix, composerReady]) const handleComposerChange = useCallback(() => { + // Typing means the user has moved on from the reply being read out. + if (getSpeechPlayerState().status !== "idle") stopSpeech() // The history's own writes are not edits. They must not end navigation, and // they must not be saved as the draft: overwriting the stored draft with a // recalled prompt would lose what the user had typed if they closed the tab @@ -1534,6 +1537,7 @@ export function MessageInput({ }, [clearAttachments, closeSlashMenu]) const handleSend = useCallback(() => { + stopSpeech() // The editor stays editable while `disabled` (the agent is busy) so the user // can keep typing, but a plain send is blocked — only enqueue / queue-edit // save go through. Mirrors the legacy textarea's keydown guard. diff --git a/src/contexts/acp-connections-context.test.tsx b/src/contexts/acp-connections-context.test.tsx index 6827d2adf0..948d7d9842 100644 --- a/src/contexts/acp-connections-context.test.tsx +++ b/src/contexts/acp-connections-context.test.tsx @@ -16,6 +16,16 @@ import { import { parsePermissionToolCall } from "@/lib/permission-request" import { subscribe } from "@/lib/platform" import { saveConfigPreference } from "@/lib/selector-prefs-storage" +import { + getSpeechPlayerState, + resetSpeechPlayerForTests, +} from "@/lib/speech-player" +import { + DEFAULT_SPEECH_PREFS, + resetSpeechPrefsCacheForTests, + saveSpeechPrefs, +} from "@/lib/speech-prefs" +import { useTabStore } from "@/stores/tab-store" import type { AttachHandlers } from "@/lib/transport/types" import type { EventEnvelope, @@ -70,6 +80,7 @@ const h = vi.hoisted(() => { }) vi.mock("next-intl", () => ({ + useLocale: () => "en", useTranslations: () => (key: string, values?: Record) => { h.tCalls.push([key, values]) return key @@ -6378,3 +6389,112 @@ describe("AIR session failures are told as notifications", () => { expect(h.recordAlert).not.toHaveBeenCalled() }) }) + +describe("AcpConnectionsProvider auto-read", () => { + let spoken: string[] = [] + let onSpoken: () => void = () => {} + const nextSpoken = () => + new Promise((resolve) => { + onSpoken = resolve + }) + + async function connectOwner(): Promise { + h.acpFindConnectionForConversation.mockResolvedValue(null) + await mountProvider() + await act(async () => { + await h.actions!.connect(TAB, "claude_code", "/tmp/x", "sess-1", 42) + }) + return latestAttachHandlers() + } + + function turn( + handlers: AttachHandlers, + seq: number, + stop_reason: string, + text = "All done." + ) { + emitAcpEvent(handlers, { + seq, + connection_id: "spawned-conn", + type: "status_changed", + status: "prompting", + }) + emitAcpEvent(handlers, { + seq: seq + 1, + connection_id: "spawned-conn", + type: "content_delta", + text, + }) + emitAcpEvent(handlers, { + seq: seq + 2, + connection_id: "spawned-conn", + type: "turn_complete", + session_id: "sess-1", + stop_reason, + }) + } + + beforeEach(() => { + spoken = [] + localStorage.clear() + resetSpeechPrefsCacheForTests() + resetSpeechPlayerForTests() + vi.stubGlobal("speechSynthesis", { + getVoices: () => [{ voiceURI: "v", lang: "en-US" }], + speak: (u: { text: string }) => { + spoken.push(u.text) + onSpoken() + }, + cancel: vi.fn(), + }) + vi.stubGlobal( + "SpeechSynthesisUtterance", + class { + constructor(public text: string) {} + } + ) + saveSpeechPrefs({ + output: { + ...DEFAULT_SPEECH_PREFS.output, + enabled: true, + engine: "browser", + autoRead: true, + }, + }) + useTabStore.setState({ activeTabId: TAB }) + }) + + afterEach(() => { + resetSpeechPlayerForTests() + vi.unstubAllGlobals() + useTabStore.setState({ activeTabId: null }) + }) + + it("reads a clean reply on the active tab once, and never a cancelled one", async () => { + const handlers = await connectOwner() + turn(handlers, 1, "cancelled") + // A skipped auto-read never leaves idle; a started one is "loading" at once. + expect(getSpeechPlayerState().status).toBe("idle") + + const said = nextSpoken() + turn(handlers, 4, "end_turn") + await said + expect(spoken).toEqual(["All done."]) + expect(getSpeechPlayerState().playingId).toBe(`auto:${TAB}`) + }) + + it("stays silent for a background tab and stops when the tab changes", async () => { + const handlers = await connectOwner() + act(() => useTabStore.setState({ activeTabId: "other-tab" })) + turn(handlers, 1, "end_turn") + expect(getSpeechPlayerState().status).toBe("idle") + + act(() => useTabStore.setState({ activeTabId: TAB })) + const said = nextSpoken() + turn(handlers, 4, "end_turn") + await said + expect(getSpeechPlayerState().status).not.toBe("idle") + act(() => useTabStore.setState({ activeTabId: "other-tab" })) + expect(getSpeechPlayerState().status).toBe("idle") + }) +}) diff --git a/src/contexts/acp-connections-context.tsx b/src/contexts/acp-connections-context.tsx index 50cfa17b32..b54f1266e0 100644 --- a/src/contexts/acp-connections-context.tsx +++ b/src/contexts/acp-connections-context.tsx @@ -9,7 +9,7 @@ import { useRef, type ReactNode, } from "react" -import { useTranslations } from "next-intl" +import { useLocale, useTranslations } from "next-intl" import { subscribe, getEventStream } from "@/lib/platform" import type { AttachHandlers, @@ -103,6 +103,10 @@ import { import { dismissNotification, notify, type NotifyAction } from "@/lib/notify" import type { SnapshotPatch } from "@/lib/snapshot-denormalize" import { getAgentLabel } from "@/lib/custom-agents" +import { resolveSpeechLanguage } from "@/lib/speech-capabilities" +import { maybeAutoRead, stopSpeech } from "@/lib/speech-player" +import { getSpeechPrefs } from "@/lib/speech-prefs" +import { useTabStore } from "@/stores/tab-store" import { localizeConfigOptionLabel, localizeConfigValueLabel, @@ -3302,6 +3306,19 @@ export function AcpConnectionsProvider({ children }: { children: ReactNode }) { useEffect(() => { folderNameRef.current = folder?.name }, [folder?.name]) + const locale = useLocale() + const localeRef = useRef(locale) + useEffect(() => { + localeRef.current = locale + }, [locale]) + // Read-aloud belongs to the tab it was started in; leaving the tab ends it. + useEffect( + () => + useTabStore.subscribe((state, prev) => { + if (state.activeTabId !== prev.activeTabId) stopSpeech() + }), + [] + ) // Depth > 0 while REPLAYED envelopes are being applied (see `onReplay`): // `handleMappedEvent` then treats them like echoes and skips the one-shot // effects — toasts, sounds, OS notifications — while the store catches up. @@ -5197,6 +5214,15 @@ export function AcpConnectionsProvider({ children }: { children: ReactNode }) { }) // Detect pending question from tool calls in the completed turn const turnConn = storeRef.current.connections.get(contextKey) + // The reply as the user read it, for auto-read below; subagent text + // (parented blocks) is not part of it. + const replyText = (turnConn?.liveMessage?.content ?? []) + .flatMap((block) => + block.type === "text" && !block.parentToolUseId + ? [block.text] + : [] + ) + .join("\n") if (turnConn?.liveMessage) { const blocks = turnConn.liveMessage.content for (let i = blocks.length - 1; i >= 0; i--) { @@ -5261,6 +5287,24 @@ export function AcpConnectionsProvider({ children }: { children: ReactNode }) { title, body: t("notificationTurnComplete", { agent: agentLabel }), }) + maybeAutoRead( + { + contextKey, + activeId: useTabStore.getState().activeTabId, + visibility: document.visibilityState, + }, + replyText, + { + language: resolveSpeechLanguage( + getSpeechPrefs().input, + localeRef.current + ), + labels: { + codeOmitted: tChat("messageList.speechCodeOmitted"), + tableOmitted: tChat("messageList.speechTableOmitted"), + }, + } + ) } } } From f05da46c18a88a7816583b30606454e2662a9bf6 Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 18:34:54 +0700 Subject: [PATCH 13/15] feat(assistant): add backend-owned workspace assistant session Add a single persistent assistant conversation owned by the backend rather than any tab: assistant_{get,set}_settings, assistant_ensure and assistant_reset, exposed as Tauri commands and web POST routes. ensure reuses the live assistant connection (remembered under the ensure lock, since a fresh spawn is linked to its conversation only on the first prompt), resumes the stored session when possible, and spawns the codeg-mcp companion with the new `assistant` feature, which also turns on `sessions`. Agents without companion support are refused. Refs #844 --- src-tauri/src/acp/connection.rs | 28 +- src-tauri/src/acp/delegation/companion.rs | 19 + src-tauri/src/acp/manager.rs | 5 + src-tauri/src/commands/assistant.rs | 447 ++++++++++++++++++++++ src-tauri/src/commands/mod.rs | 1 + src-tauri/src/lib.rs | 5 + src-tauri/src/web/handlers/assistant.rs | 51 +++ src-tauri/src/web/handlers/mod.rs | 1 + src-tauri/src/web/router.rs | 16 + src/lib/api.ts | 20 + src/lib/types.ts | 14 + 11 files changed, 604 insertions(+), 3 deletions(-) create mode 100644 src-tauri/src/commands/assistant.rs create mode 100644 src-tauri/src/web/handlers/assistant.rs diff --git a/src-tauri/src/acp/connection.rs b/src-tauri/src/acp/connection.rs index 156fa242ec..cf7d566cb5 100644 --- a/src-tauri/src/acp/connection.rs +++ b/src-tauri/src/acp/connection.rs @@ -4860,7 +4860,7 @@ async fn send_new_session_capturing_models( /// (`feedback_tool_available`, a registered delegation token pi can never use). /// `supports_mcp` stays `true` for pi (session/new tolerates the field), so this /// is a separate, narrower gate. Gate codeg-mcp injection on it. -fn agent_delivers_wire_mcp(agent_type: AgentType) -> bool { +pub(crate) fn agent_delivers_wire_mcp(agent_type: AgentType) -> bool { !matches!(agent_type, AgentType::Pi) } @@ -5125,6 +5125,8 @@ struct CompanionFeatureFlags { /// it still injects the companion so a task session always has its reporting /// tools. tasks: bool, + /// `assistant` session tools. + assistant: bool, /// `create_automation`, gated by the chat-authoring setting. automations: bool, /// `create_work_task`, gated by the chat-authoring setting. @@ -5155,7 +5157,8 @@ fn companion_features_arg(flags: CompanionFeatureFlags) -> Option { if flags.ask { features.push("ask"); } - if flags.sessions { + // The assistant's tools build on `get_session_info`, so it always gets sessions. + if flags.sessions || flags.assistant { features.push("sessions"); } if flags.tasks { @@ -5176,6 +5179,9 @@ fn companion_features_arg(flags: CompanionFeatureFlags) -> Option { if flags.browser && flags.browser_eval { features.push("browser_eval"); } + if flags.assistant { + features.push("assistant"); + } if features.is_empty() { return None; } @@ -5192,12 +5198,14 @@ struct CompanionInjection { delegation_enabled: bool, } +#[allow(clippy::too_many_arguments)] async fn inject_codeg_mcp( servers: &mut Vec, injection: &DelegationInjection, parent_connection_id: &str, working_dir: &Path, tasks_enabled: bool, + assistant_enabled: bool, host_tools: HostToolsPolicy, ) -> Option { inject_codeg_mcp_with_binary_locator( @@ -5206,18 +5214,21 @@ async fn inject_codeg_mcp( parent_connection_id, working_dir, tasks_enabled, + assistant_enabled, host_tools, locate_codeg_mcp_binary, ) .await } +#[allow(clippy::too_many_arguments)] async fn inject_codeg_mcp_with_binary_locator( servers: &mut Vec, injection: &DelegationInjection, parent_connection_id: &str, working_dir: &Path, tasks_enabled: bool, + assistant_enabled: bool, host_tools: HostToolsPolicy, locate_binary: F, ) -> Option @@ -5281,6 +5292,7 @@ where browser: cfg!(feature = "tauri-runtime") && injection.browser.is_enabled().await, browser_eval: cfg!(feature = "tauri-runtime") && injection.browser.is_eval_enabled().await, + assistant: assistant_enabled, }; // `None` (no feature enabled) short-circuits BEFORE the binary lookup, the // token registration and the server append: there is no companion to launch, @@ -6134,12 +6146,15 @@ async fn run_connection( // task_progress / task_complete tool group. let tasks_enabled = { state.read().await.owner_window_label == "work_task" }; + let assistant_enabled = + { state.read().await.owner_window_label == crate::commands::assistant::ASSISTANT_OWNER_LABEL }; inject_codeg_mcp( &mut mcp_servers, inj, &conn_id, &cwd, tasks_enabled, + assistant_enabled, host_tools, ) .await @@ -26508,6 +26523,7 @@ mod tests { "parent-conn", std::path::Path::new("/tmp"), false, + false, HostToolsPolicy::Default, ) .await; @@ -26667,6 +26683,11 @@ mod tests { // The browser group too — a user who only shares browser tabs still // gets a companion. assert_eq!(only(|f| f.browser = true), Some("browser".to_string())); + // Assistant injects assistant and sessions. + assert_eq!( + only(|f| f.assistant = true), + Some("sessions,assistant".to_string()) + ); // All on → comma-joined, in the order the companion parses. assert_eq!( companion_features_arg(CompanionFeatureFlags { @@ -26679,9 +26700,10 @@ mod tests { taskboard: true, browser: true, browser_eval: true, + assistant: true, }), Some( - "delegation,feedback,ask,sessions,tasks,automations,taskboard,browser,browser_eval" + "delegation,feedback,ask,sessions,tasks,automations,taskboard,browser,browser_eval,assistant" .to_string() ) ); diff --git a/src-tauri/src/acp/delegation/companion.rs b/src-tauri/src/acp/delegation/companion.rs index 64416b3ac6..0703abfe8d 100644 --- a/src-tauri/src/acp/delegation/companion.rs +++ b/src-tauri/src/acp/delegation/companion.rs @@ -181,6 +181,9 @@ pub struct CompanionFeatures { /// tab can picture, and this is not one of them. Never on with `browser` /// off; the parent will not emit it, and `allows_tool` requires both. pub browser_eval: bool, + /// Workspace-assistant tools — injected only into the backend-owned + /// assistant connection. + pub assistant: bool, } impl CompanionFeatures { @@ -202,6 +205,7 @@ impl CompanionFeatures { taskboard: false, browser: false, browser_eval: false, + assistant: false, }; }; let mut f = Self { @@ -214,6 +218,7 @@ impl CompanionFeatures { taskboard: false, browser: false, browser_eval: false, + assistant: false, }; for tok in s.split(',').map(str::trim).filter(|t| !t.is_empty()) { match tok { @@ -226,6 +231,7 @@ impl CompanionFeatures { "taskboard" => f.taskboard = true, "browser" => f.browser = true, "browser_eval" => f.browser_eval = true, + "assistant" => f.assistant = true, _ => {} } } @@ -2587,6 +2593,7 @@ mod tests { taskboard: false, browser: false, browser_eval: false, + assistant: false, }) } @@ -3180,6 +3187,7 @@ mod tests { taskboard: false, browser: false, browser_eval: false, + assistant: false, }; const BOTH: CompanionFeatures = CompanionFeatures { delegation: true, @@ -3191,6 +3199,7 @@ mod tests { taskboard: false, browser: false, browser_eval: false, + assistant: false, }; const ASK_ONLY: CompanionFeatures = CompanionFeatures { delegation: false, @@ -3202,6 +3211,7 @@ mod tests { taskboard: false, browser: false, browser_eval: false, + assistant: false, }; const SESSIONS_ONLY: CompanionFeatures = CompanionFeatures { delegation: false, @@ -3213,6 +3223,7 @@ mod tests { taskboard: false, browser: false, browser_eval: false, + assistant: false, }; fn list_tool_names(action: LineAction) -> Vec { @@ -3241,6 +3252,9 @@ mod tests { assert!(!ask.delegation && !ask.feedback && ask.ask); let sessions = CompanionFeatures::parse(Some("sessions")); assert!(!sessions.delegation && !sessions.feedback && !sessions.ask && sessions.sessions); + let assistant = CompanionFeatures::parse(Some("assistant")); + assert!(assistant.assistant && !assistant.delegation && !assistant.sessions); + assert!(!def.assistant && !all.assistant); // Empty string → nothing enabled. let none = CompanionFeatures::parse(Some("")); assert!(!none.delegation && !none.feedback && !none.ask && !none.sessions); @@ -3553,6 +3567,7 @@ mod tests { taskboard: false, browser: false, browser_eval: false, + assistant: false, }; const TASKBOARD_ONLY: CompanionFeatures = CompanionFeatures { delegation: false, @@ -3564,6 +3579,7 @@ mod tests { taskboard: true, browser: false, browser_eval: false, + assistant: false, }; /// The two authoring groups gate independently: enabling one must not @@ -4087,12 +4103,14 @@ mod tests { taskboard: false, browser: true, browser_eval: false, + assistant: false, }; /// The browser group with `browser_eval` on top, which is the only way /// that tool is ever advertised. const BROWSER_WITH_EVAL: CompanionFeatures = CompanionFeatures { browser_eval: true, + assistant: false, ..BROWSER_ONLY }; @@ -4148,6 +4166,7 @@ mod tests { const EVAL_WITHOUT_GROUP: CompanionFeatures = CompanionFeatures { browser: false, browser_eval: true, + assistant: false, ..BROWSER_ONLY }; let list = r#"{"jsonrpc":"2.0","id":1,"method":"tools/list"}"#; diff --git a/src-tauri/src/acp/manager.rs b/src-tauri/src/acp/manager.rs index 9df59aa3df..9f285df72d 100644 --- a/src-tauri/src/acp/manager.rs +++ b/src-tauri/src/acp/manager.rs @@ -802,6 +802,11 @@ impl ConnectionManager { /// open). Silently no-ops if the connection is missing or already /// in a terminal state — touch must never resurrect a dead /// connection or contend with the spawn/disconnect paths. + pub async fn get_owner_window_label(&self, conn_id: &str) -> Option { + let connections = self.connections.lock().await; + connections.get(conn_id).map(|c| c.owner_window_label.clone()) + } + pub async fn touch(&self, conn_id: &str) -> bool { let state_arc = { let connections = self.connections.lock().await; diff --git a/src-tauri/src/commands/assistant.rs b/src-tauri/src/commands/assistant.rs new file mode 100644 index 0000000000..1798375c31 --- /dev/null +++ b/src-tauri/src/commands/assistant.rs @@ -0,0 +1,447 @@ +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use std::sync::OnceLock; + +#[cfg(feature = "tauri-runtime")] +use tauri::State; + +use crate::acp::connection::agent_delivers_wire_mcp; +use crate::acp::manager::ConnectionManager; +use crate::app_error::AppCommandError; +use crate::commands::acp::{build_session_runtime_env, verify_agent_installed}; +use crate::commands::conversations::create_chat_conversation_core; +use crate::db::service::app_metadata_service::{get_value, upsert_value}; +#[cfg(feature = "tauri-runtime")] +use crate::db::AppDatabase; +use crate::models::AgentType; +use crate::web::event_bridge::EventEmitter; + +pub const ASSISTANT_OWNER_LABEL: &str = "assistant"; + +const KEY_AGENT_TYPE: &str = "assistant.agent_type"; +const KEY_CONVERSATION_ID: &str = "assistant.conversation_id"; +const KEY_ALLOW_SESSION_CONTROL: &str = "assistant.allow_session_control"; +const KEY_ALLOW_PERMISSION_ANSWERS: &str = "assistant.allow_permission_answers"; + +pub const ASSISTANT_PRIMER: &str = "you are Codeg's workspace assistant; you are spoken to by voice; answer in 1-3 short spoken sentences without markdown or code; use the `codeg-mcp` tools `list_sessions`, `get_session_info`, `focus_session`, `send_to_session`, `cancel_session`, `answer_permission` and `start_session` to act on the user's other sessions; never claim an action happened unless the tool result says so."; + +/// Serializes ensure/reset and remembers the live assistant connection as +/// `(conversation_id, connection_id)`. A fresh spawn is not linked to its +/// conversation until the first prompt, so it cannot be found by conversation. +static ENSURE_LOCK: OnceLock>> = OnceLock::new(); + +fn ensure_lock() -> &'static tokio::sync::Mutex> { + ENSURE_LOCK.get_or_init(|| tokio::sync::Mutex::new(None)) +} + +async fn is_live_assistant(manager: &ConnectionManager, conn_id: &str) -> bool { + manager.get_owner_window_label(conn_id).await.as_deref() == Some(ASSISTANT_OWNER_LABEL) +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AssistantSettings { + pub agent_type: Option, + pub allow_session_control: bool, + pub allow_permission_answers: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AssistantSession { + pub connection_id: String, + pub conversation_id: i32, + pub folder_id: i32, + pub agent_type: AgentType, + pub primer: Option, +} + +pub async fn assistant_get_settings_core( + db: &sea_orm::DatabaseConnection, +) -> Result { + let agent_type_str = get_value(db, KEY_AGENT_TYPE) + .await + .map_err(AppCommandError::from)?; + let agent_type = + agent_type_str.and_then(|s| serde_json::from_str::(&format!("\"{s}\"")).ok()); + + let allow_session_control = get_value(db, KEY_ALLOW_SESSION_CONTROL) + .await + .map_err(AppCommandError::from)? + .as_deref() + == Some("1"); + + let allow_permission_answers = get_value(db, KEY_ALLOW_PERMISSION_ANSWERS) + .await + .map_err(AppCommandError::from)? + .as_deref() + == Some("1"); + + Ok(AssistantSettings { + agent_type, + allow_session_control, + allow_permission_answers, + }) +} + +pub async fn assistant_set_settings_core( + db: &sea_orm::DatabaseConnection, + settings: AssistantSettings, +) -> Result<(), AppCommandError> { + let current = assistant_get_settings_core(db).await?; + + if current.agent_type != settings.agent_type { + upsert_value(db, KEY_CONVERSATION_ID, "") + .await + .map_err(AppCommandError::from)?; + } + + let agent_str = settings + .agent_type + .map(|t| t.as_wire().to_string()) + .unwrap_or_default(); + upsert_value(db, KEY_AGENT_TYPE, &agent_str) + .await + .map_err(AppCommandError::from)?; + + upsert_value( + db, + KEY_ALLOW_SESSION_CONTROL, + if settings.allow_session_control { + "1" + } else { + "0" + }, + ) + .await + .map_err(AppCommandError::from)?; + upsert_value( + db, + KEY_ALLOW_PERMISSION_ANSWERS, + if settings.allow_permission_answers { + "1" + } else { + "0" + }, + ) + .await + .map_err(AppCommandError::from)?; + + Ok(()) +} + +pub async fn assistant_reset_core( + db: &sea_orm::DatabaseConnection, + manager: &ConnectionManager, +) -> Result<(), AppCommandError> { + let mut live = ensure_lock().lock().await; + if let Some((_, conn_id)) = live.take() { + if is_live_assistant(manager, &conn_id).await { + let _ = manager.disconnect(&conn_id).await; + } + } + let conv_id_str = get_value(db, KEY_CONVERSATION_ID) + .await + .map_err(AppCommandError::from)?; + if let Some(conv_id) = conv_id_str.and_then(|s| s.parse::().ok()) { + if let Some(conn_id) = manager.find_connection_by_conversation_id(conv_id).await { + if is_live_assistant(manager, &conn_id).await { + let _ = manager.disconnect(&conn_id).await; + } + } + } + upsert_value(db, KEY_CONVERSATION_ID, "") + .await + .map_err(AppCommandError::from)?; + Ok(()) +} + +pub async fn assistant_ensure_core( + db: &crate::db::AppDatabase, + manager: &ConnectionManager, + emitter: EventEmitter, + data_dir: PathBuf, +) -> Result { + let mut live = ensure_lock().lock().await; + + let settings = assistant_get_settings_core(&db.conn).await?; + let agent_type = settings + .agent_type + .ok_or_else(|| AppCommandError::invalid_input("assistant agent not configured"))?; + + if !agent_delivers_wire_mcp(agent_type) { + return Err(AppCommandError::invalid_input( + "agent does not support codeg-mcp companion", + )); + } + + let conv_id_str = get_value(&db.conn, KEY_CONVERSATION_ID) + .await + .map_err(AppCommandError::from)?; + let mut conversation_id = conv_id_str.and_then(|s| s.parse::().ok()); + + if let Some(id) = conversation_id { + if crate::db::service::conversation_service::get_by_id(&db.conn, id) + .await + .is_err() + { + conversation_id = None; + } + } + + let mut primer = None; + + if conversation_id.is_none() { + let title = "Codeg Assistant".to_string(); + let conv = + create_chat_conversation_core(&db.conn, &data_dir, agent_type, Some(title), None) + .await?; + conversation_id = Some(conv.conversation_id); + upsert_value( + &db.conn, + KEY_CONVERSATION_ID, + &conv.conversation_id.to_string(), + ) + .await + .map_err(AppCommandError::from)?; + primer = Some(ASSISTANT_PRIMER.to_string()); + } + + let conversation_id = conversation_id.unwrap(); + let conv = crate::db::service::conversation_service::get_by_id(&db.conn, conversation_id) + .await + .map_err(AppCommandError::from)?; + + let remembered = live + .as_ref() + .filter(|(conv, _)| *conv == conversation_id) + .map(|(_, conn)| conn.clone()); + let linked = manager + .find_connection_by_conversation_id(conversation_id) + .await; + for conn_id in remembered.into_iter().chain(linked) { + if is_live_assistant(manager, &conn_id).await { + *live = Some((conversation_id, conn_id.clone())); + return Ok(AssistantSession { + connection_id: conn_id, + conversation_id, + folder_id: conv.folder_id, + agent_type, + primer, + }); + } + } + + let resume_id = conv.external_id.clone(); + + verify_agent_installed(agent_type) + .await + .map_err(|e| AppCommandError::task_execution_failed(e.to_string()))?; + + let env = build_session_runtime_env(db, agent_type, resume_id.as_deref(), &data_dir) + .await + .map_err(|e| AppCommandError::task_execution_failed(e.to_string()))?; + + let folder = crate::db::service::folder_service::get_folder_by_id(&db.conn, conv.folder_id) + .await + .map_err(AppCommandError::from)? + .ok_or_else(|| AppCommandError::not_found("folder not found"))?; + let folder_path = std::path::PathBuf::from(folder.path); + + let mut conn_id = None; + let mut last_error = None; + + if let Some(ref rid) = resume_id { + match manager + .spawn_agent( + agent_type, + Some(folder_path.to_string_lossy().to_string()), + Some(rid.clone()), + env.clone(), + ASSISTANT_OWNER_LABEL.to_string(), + emitter.clone(), + None, + std::collections::BTreeMap::new(), + ) + .await + { + Ok(info) => conn_id = Some(info), + Err(e) => last_error = Some(e), + } + } + + if conn_id.is_none() { + match manager + .spawn_agent( + agent_type, + Some(folder_path.to_string_lossy().to_string()), + None, + env, + ASSISTANT_OWNER_LABEL.to_string(), + emitter, + None, + std::collections::BTreeMap::new(), + ) + .await + { + Ok(info) => conn_id = Some(info), + Err(e) => last_error = Some(e), + } + } + + if let Some(connection_id) = conn_id { + *live = Some((conversation_id, connection_id.clone())); + Ok(AssistantSession { + connection_id, + conversation_id, + folder_id: conv.folder_id, + agent_type, + primer, + }) + } else { + Err(AppCommandError::task_execution_failed( + last_error + .map(|e| e.to_string()) + .unwrap_or_else(|| "failed to spawn assistant agent".to_string()), + )) + } +} + +#[cfg(feature = "tauri-runtime")] +#[cfg_attr(feature = "tauri-runtime", tauri::command)] +pub async fn assistant_get_settings( + db: State<'_, AppDatabase>, +) -> Result { + assistant_get_settings_core(&db.conn).await +} + +#[cfg(feature = "tauri-runtime")] +#[cfg_attr(feature = "tauri-runtime", tauri::command)] +pub async fn assistant_set_settings( + db: State<'_, AppDatabase>, + settings: AssistantSettings, +) -> Result<(), AppCommandError> { + assistant_set_settings_core(&db.conn, settings).await +} + +#[cfg(feature = "tauri-runtime")] +#[cfg_attr(feature = "tauri-runtime", tauri::command)] +pub async fn assistant_reset( + db: State<'_, AppDatabase>, + manager: State<'_, ConnectionManager>, +) -> Result<(), AppCommandError> { + assistant_reset_core(&db.conn, &manager).await +} + +#[cfg(feature = "tauri-runtime")] +#[cfg_attr(feature = "tauri-runtime", tauri::command)] +pub async fn assistant_ensure( + app: tauri::AppHandle, + db: State<'_, AppDatabase>, + manager: State<'_, ConnectionManager>, +) -> Result { + let data_dir = crate::paths::codeg_home_dir(); + assistant_ensure_core(&db, &manager, EventEmitter::Tauri(app), data_dir).await +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::test_helpers::fresh_in_memory_db; + + #[tokio::test] + async fn test_assistant_settings_roundtrip() { + let db = fresh_in_memory_db().await; + + let defaults = assistant_get_settings_core(&db.conn).await.unwrap(); + assert_eq!(defaults.agent_type, None); + assert!(!defaults.allow_session_control); + assert!(!defaults.allow_permission_answers); + + let settings = AssistantSettings { + agent_type: Some(AgentType::Codex), + allow_session_control: true, + allow_permission_answers: true, + }; + assistant_set_settings_core(&db.conn, settings) + .await + .unwrap(); + + let updated = assistant_get_settings_core(&db.conn).await.unwrap(); + assert_eq!(updated.agent_type, Some(AgentType::Codex)); + assert!(updated.allow_session_control); + assert!(updated.allow_permission_answers); + } + + #[tokio::test] + async fn test_agent_change_clears_conversation_id() { + let db = fresh_in_memory_db().await; + + assistant_set_settings_core( + &db.conn, + AssistantSettings { + agent_type: Some(AgentType::Codex), + allow_session_control: false, + allow_permission_answers: false, + }, + ) + .await + .unwrap(); + + upsert_value(&db.conn, KEY_CONVERSATION_ID, "42") + .await + .unwrap(); + + assistant_set_settings_core( + &db.conn, + AssistantSettings { + agent_type: Some(AgentType::ClaudeCode), + allow_session_control: false, + allow_permission_answers: false, + }, + ) + .await + .unwrap(); + + let conv_id = get_value(&db.conn, KEY_CONVERSATION_ID) + .await + .unwrap() + .unwrap_or_default(); + assert!(conv_id.is_empty(), "conversation id should be cleared"); + } + + #[tokio::test] + async fn test_ensure_without_agent_errors() { + let db = fresh_in_memory_db().await; + let manager = ConnectionManager::new(); + let emitter = EventEmitter::Noop; + + let err = assistant_ensure_core(&db, &manager, emitter, PathBuf::from("/tmp")) + .await + .unwrap_err(); + assert_eq!(err.message, "assistant agent not configured"); + } + + #[tokio::test] + async fn test_ensure_with_pi_errors() { + let db = fresh_in_memory_db().await; + assistant_set_settings_core( + &db.conn, + AssistantSettings { + agent_type: Some(AgentType::Pi), + allow_session_control: false, + allow_permission_answers: false, + }, + ) + .await + .unwrap(); + + let manager = ConnectionManager::new(); + let emitter = EventEmitter::Noop; + + let err = assistant_ensure_core(&db, &manager, emitter, PathBuf::from("/tmp")) + .await + .unwrap_err(); + assert_eq!(err.message, "agent does not support codeg-mcp companion"); + } +} diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 21d8332abb..9b4c5fb014 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -58,3 +58,4 @@ pub mod windows; pub mod speech; pub mod work_task; pub mod workspace_state; +pub mod assistant; diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index d6ed953434..cce8f3d269 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -90,6 +90,7 @@ mod tauri_app { config_sync, conversations, speech, + assistant, custom_skills as custom_skills_commands, deepseek_settings as deepseek_settings_commands, delegation as delegation_commands, experts as experts_commands, feedback as feedback_commands, file_io, folder_commands, @@ -1980,6 +1981,10 @@ mod tauri_app { speech::speech_update_settings, speech::speech_transcribe, speech::speech_synthesize, + assistant::assistant_get_settings, + assistant::assistant_set_settings, + assistant::assistant_reset, + assistant::assistant_ensure, config_sync::config_sync_update_settings, config_sync::config_sync_get_state, config_sync::config_sync_test_connection, diff --git a/src-tauri/src/web/handlers/assistant.rs b/src-tauri/src/web/handlers/assistant.rs new file mode 100644 index 0000000000..9c6297d8b5 --- /dev/null +++ b/src-tauri/src/web/handlers/assistant.rs @@ -0,0 +1,51 @@ +use crate::app_error::AppCommandError; +use crate::app_state::AppState; +use crate::commands::assistant::{ + assistant_ensure_core, assistant_get_settings_core, assistant_reset_core, + assistant_set_settings_core, AssistantSession, AssistantSettings, +}; +use crate::web::event_bridge::EventEmitter; +use axum::{Extension, Json}; +use serde::Deserialize; +use std::sync::Arc; + +pub async fn get_settings( + Extension(state): Extension>, +) -> Result, AppCommandError> { + let settings = assistant_get_settings_core(&state.db.conn).await?; + Ok(Json(settings)) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SetSettingsParams { + pub settings: AssistantSettings, +} + +pub async fn set_settings( + Extension(state): Extension>, + Json(params): Json, +) -> Result, AppCommandError> { + assistant_set_settings_core(&state.db.conn, params.settings).await?; + Ok(Json(())) +} + +pub async fn reset( + Extension(state): Extension>, +) -> Result, AppCommandError> { + assistant_reset_core(&state.db.conn, &state.connection_manager).await?; + Ok(Json(())) +} + +pub async fn ensure( + Extension(state): Extension>, +) -> Result, AppCommandError> { + let emitter = EventEmitter::WebOnly { + broadcaster: state.event_broadcaster.clone(), + bus: state.acp_event_bus.clone(), + }; + let data_dir = crate::paths::codeg_home_dir(); + let session = + assistant_ensure_core(&state.db, &state.connection_manager, emitter, data_dir).await?; + Ok(Json(session)) +} diff --git a/src-tauri/src/web/handlers/mod.rs b/src-tauri/src/web/handlers/mod.rs index b8b3eebeaa..59d7a33ba5 100644 --- a/src-tauri/src/web/handlers/mod.rs +++ b/src-tauri/src/web/handlers/mod.rs @@ -36,6 +36,7 @@ pub mod science; pub mod session_info; pub mod system_settings; pub mod terminal; +pub mod assistant; pub mod token_usage; mod upload_jail; pub mod version_control; diff --git a/src-tauri/src/web/router.rs b/src-tauri/src/web/router.rs index 193989573d..b3c925b9e6 100644 --- a/src-tauri/src/web/router.rs +++ b/src-tauri/src/web/router.rs @@ -620,6 +620,22 @@ pub fn build_router( "/speech_synthesize", post(handlers::speech::speech_synthesize), ) + .route( + "/assistant_get_settings", + post(handlers::assistant::get_settings), + ) + .route( + "/assistant_set_settings", + post(handlers::assistant::set_settings), + ) + .route( + "/assistant_reset", + post(handlers::assistant::reset), + ) + .route( + "/assistant_ensure", + post(handlers::assistant::ensure), + ) .route( "/config_sync_get_settings", post(handlers::config_sync::config_sync_get_settings), diff --git a/src/lib/api.ts b/src/lib/api.ts index 8ff78c2378..32f827559d 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -31,6 +31,8 @@ import type { SpeechCloudSettings, SpeechAudio, SpeechCloudSettingsView, + AssistantSettings, + AssistantSession, ForgeChangeDetail, ForgeChangedFileList, ForgeComment, @@ -5995,3 +5997,21 @@ export async function speechSynthesize( ): Promise { return getTransport().call("speech_synthesize", { text, speed }) } + +export async function assistantEnsure(): Promise { + return getTransport().call("assistant_ensure", {}) +} + +export async function assistantReset(): Promise { + return getTransport().call("assistant_reset", {}) +} + +export async function assistantGetSettings(): Promise { + return getTransport().call("assistant_get_settings", {}) +} + +export async function assistantSetSettings( + settings: AssistantSettings +): Promise { + return getTransport().call("assistant_set_settings", { settings }) +} diff --git a/src/lib/types.ts b/src/lib/types.ts index c834b4ce4a..fa83f18191 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -5090,3 +5090,17 @@ export interface SpeechAudio { audioBase64: string mimeType: string } + +export interface AssistantSettings { + agentType: AgentType | null + allowSessionControl: boolean + allowPermissionAnswers: boolean +} + +export interface AssistantSession { + connectionId: string + conversationId: number + folderId: number + agentType: AgentType + primer: string | null +} From b92214db8920aa27a154a53af1062b1b68079ffe Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Sun, 27 Sep 2026 20:05:57 +0700 Subject: [PATCH 14/15] feat(assistant): add assistant tool group with list and focus Refs #844 --- src-tauri/src/acp/assistant_tools.rs | 18 + src-tauri/src/acp/delegation/companion.rs | 309 ++++++++++++- src-tauri/src/acp/delegation/listener.rs | 411 ++++++++++++++++++ src-tauri/src/acp/delegation/service.rs | 66 +++ src-tauri/src/acp/delegation/tool_schema.json | 106 +++++ src-tauri/src/acp/delegation/transport.rs | 153 +++++++ src-tauri/src/acp/manager.rs | 149 +++++++ src-tauri/src/acp/mod.rs | 2 + src-tauri/src/bin/codeg_server.rs | 9 + src-tauri/src/commands/assistant_tools.rs | 114 +++++ src-tauri/src/commands/mod.rs | 2 + src-tauri/src/lib.rs | 13 + src-tauri/tests/delegation_e2e_uds.rs | 56 ++- 13 files changed, 1389 insertions(+), 19 deletions(-) create mode 100644 src-tauri/src/acp/assistant_tools.rs create mode 100644 src-tauri/src/commands/assistant_tools.rs diff --git a/src-tauri/src/acp/assistant_tools.rs b/src-tauri/src/acp/assistant_tools.rs new file mode 100644 index 0000000000..10608d4b53 --- /dev/null +++ b/src-tauri/src/acp/assistant_tools.rs @@ -0,0 +1,18 @@ +use super::delegation::transport::{AssistantActionResult, AssistantSessionList}; +use async_trait::async_trait; + +#[async_trait] +pub trait AssistantToolAccess: Send + Sync { + async fn is_assistant_connection(&self, conn_id: &str) -> bool; + async fn list_sessions(&self, exclude_conn_id: &str) -> AssistantSessionList; + async fn focus_session(&self, session_id: i64) -> AssistantActionResult; + async fn send_to_session(&self, session_id: i64, text: String) -> AssistantActionResult; + async fn cancel_session(&self, session_id: i64) -> AssistantActionResult; + async fn answer_permission(&self, session_id: i64, decision: String) -> AssistantActionResult; + async fn start_session( + &self, + folder_id: i64, + agent_type: String, + task: String, + ) -> AssistantActionResult; +} diff --git a/src-tauri/src/acp/delegation/companion.rs b/src-tauri/src/acp/delegation/companion.rs index 0703abfe8d..97e2a7c784 100644 --- a/src-tauri/src/acp/delegation/companion.rs +++ b/src-tauri/src/acp/delegation/companion.rs @@ -45,22 +45,27 @@ use crate::acp::chat_authoring::{ NewAutomationSpec, NewWorkTaskSpec, MAX_PROMPT_CHARS, MAX_TITLE_CHARS, }; use crate::acp::delegation::transport::{ - client_ask_round_trip, client_browser_act_round_trip, client_browser_capture_round_trip, - client_browser_console_round_trip, client_browser_eval_round_trip, - client_browser_snapshot_round_trip, client_browser_tab_op_round_trip, - client_browser_tabs_round_trip, - client_cancel, client_cancel_task_round_trip, client_commit_feedback, - client_create_automation_round_trip, client_create_work_task_round_trip, - client_feedback_round_trip, client_resume_task_round_trip, client_round_trip, - client_session_round_trip, client_status_round_trip, client_task_complete_round_trip, - client_task_progress_round_trip, BrokerAskRequest, BrokerBrowserActRequest, BrokerBrowserCaptureRequest, BrokerBrowserConsoleRequest, + client_ask_round_trip, client_assistant_answer_permission_round_trip, + client_assistant_cancel_session_round_trip, client_assistant_focus_session_round_trip, + client_assistant_list_sessions_round_trip, client_assistant_send_to_session_round_trip, + client_assistant_start_session_round_trip, client_browser_act_round_trip, + client_browser_capture_round_trip, client_browser_console_round_trip, + client_browser_eval_round_trip, client_browser_snapshot_round_trip, + client_browser_tab_op_round_trip, client_browser_tabs_round_trip, client_cancel, + client_cancel_task_round_trip, client_commit_feedback, client_create_automation_round_trip, + client_create_work_task_round_trip, client_feedback_round_trip, client_resume_task_round_trip, + client_round_trip, client_session_round_trip, client_status_round_trip, + client_task_complete_round_trip, client_task_progress_round_trip, BrokerAskRequest, + BrokerAssistantAnswerPermissionRequest, BrokerAssistantCancelSessionRequest, + BrokerAssistantFocusSessionRequest, BrokerAssistantListSessionsRequest, + BrokerAssistantSendToSessionRequest, BrokerAssistantStartSessionRequest, + BrokerBrowserActRequest, BrokerBrowserCaptureRequest, BrokerBrowserConsoleRequest, BrokerBrowserEvalRequest, BrokerBrowserSnapshotRequest, BrokerBrowserTabOpRequest, - BrokerBrowserTabsRequest, - BrokerCancelRequest, - BrokerCancelTaskRequest, BrokerCommitFeedbackRequest, BrokerCreateAutomationRequest, - BrokerCreateWorkTaskRequest, BrokerFeedbackRequest, BrokerRequest, BrokerResponse, - BrokerResumeTaskRequest, BrokerSessionRequest, BrokerStatusRequest, - BrokerTaskCompleteRequest, BrokerTaskProgressRequest, + BrokerBrowserTabsRequest, BrokerCancelRequest, BrokerCancelTaskRequest, + BrokerCommitFeedbackRequest, BrokerCreateAutomationRequest, BrokerCreateWorkTaskRequest, + BrokerFeedbackRequest, BrokerRequest, BrokerResponse, BrokerResumeTaskRequest, + BrokerSessionRequest, BrokerStatusRequest, BrokerTaskCompleteRequest, + BrokerTaskProgressRequest, }; use crate::acp::question::parse_questions; use crate::acp::session_info::MAX_SESSION_MESSAGES; @@ -255,7 +260,11 @@ impl CompanionFeatures { // parent bug, or someone editing the agent's MCP config by hand — // cannot leave the strongest tool as the only one present. "browser_eval" => self.browser && self.browser_eval, - "delegate_to_agent" | "get_delegation_status" | "cancel_delegation" + "list_sessions" | "focus_session" | "send_to_session" | "cancel_session" + | "answer_permission" | "start_session" => self.assistant, + "delegate_to_agent" + | "get_delegation_status" + | "cancel_delegation" | "resume_delegation" => self.delegation, _ => false, } @@ -858,6 +867,152 @@ async fn build_tools_call_spawn( Box::pin(async move { client_browser_tab_op_round_trip(&socket, &req).await }); register_and_spawn(inflight, id, None, round_trip, render_browser_tab_op_result).await } + "list_sessions" => { + let req = BrokerAssistantListSessionsRequest { + token: ctx.token.clone(), + }; + let round_trip = + Box::pin( + async move { client_assistant_list_sessions_round_trip(&socket, &req).await }, + ); + register_and_spawn( + inflight, + id, + None, + round_trip, + render_assistant_list_sessions_result, + ) + .await + } + "focus_session" => { + let session_id = match arguments.get("session_id").and_then(|v| v.as_i64()) { + Some(v) => v, + None => return LineAction::Respond(err(id, -32602, "Missing session_id")), + }; + let req = BrokerAssistantFocusSessionRequest { + token: ctx.token.clone(), + session_id, + }; + let round_trip = + Box::pin( + async move { client_assistant_focus_session_round_trip(&socket, &req).await }, + ); + register_and_spawn( + inflight, + id, + None, + round_trip, + render_assistant_action_result, + ) + .await + } + "send_to_session" => { + let session_id = match arguments.get("session_id").and_then(|v| v.as_i64()) { + Some(v) => v, + None => return LineAction::Respond(err(id, -32602, "Missing session_id")), + }; + let text = match arguments.get("text").and_then(|v| v.as_str()) { + Some(v) => v.to_string(), + None => return LineAction::Respond(err(id, -32602, "Missing text")), + }; + let req = BrokerAssistantSendToSessionRequest { + token: ctx.token.clone(), + session_id, + text, + }; + let round_trip = Box::pin(async move { + client_assistant_send_to_session_round_trip(&socket, &req).await + }); + register_and_spawn( + inflight, + id, + None, + round_trip, + render_assistant_action_result, + ) + .await + } + "cancel_session" => { + let session_id = match arguments.get("session_id").and_then(|v| v.as_i64()) { + Some(v) => v, + None => return LineAction::Respond(err(id, -32602, "Missing session_id")), + }; + let req = BrokerAssistantCancelSessionRequest { + token: ctx.token.clone(), + session_id, + }; + let round_trip = + Box::pin( + async move { client_assistant_cancel_session_round_trip(&socket, &req).await }, + ); + register_and_spawn( + inflight, + id, + None, + round_trip, + render_assistant_action_result, + ) + .await + } + "answer_permission" => { + let session_id = match arguments.get("session_id").and_then(|v| v.as_i64()) { + Some(v) => v, + None => return LineAction::Respond(err(id, -32602, "Missing session_id")), + }; + let decision = match arguments.get("decision").and_then(|v| v.as_str()) { + Some(v) => v.to_string(), + None => return LineAction::Respond(err(id, -32602, "Missing decision")), + }; + let req = BrokerAssistantAnswerPermissionRequest { + token: ctx.token.clone(), + session_id, + decision, + }; + let round_trip = Box::pin(async move { + client_assistant_answer_permission_round_trip(&socket, &req).await + }); + register_and_spawn( + inflight, + id, + None, + round_trip, + render_assistant_action_result, + ) + .await + } + "start_session" => { + let folder_id = match arguments.get("folder_id").and_then(|v| v.as_i64()) { + Some(v) => v, + None => return LineAction::Respond(err(id, -32602, "Missing folder_id")), + }; + let agent_type = match arguments.get("agent_type").and_then(|v| v.as_str()) { + Some(v) => v.to_string(), + None => return LineAction::Respond(err(id, -32602, "Missing agent_type")), + }; + let task = match arguments.get("task").and_then(|v| v.as_str()) { + Some(v) => v.to_string(), + None => return LineAction::Respond(err(id, -32602, "Missing task")), + }; + let req = BrokerAssistantStartSessionRequest { + token: ctx.token.clone(), + folder_id, + agent_type, + task, + }; + let round_trip = + Box::pin( + async move { client_assistant_start_session_round_trip(&socket, &req).await }, + ); + register_and_spawn( + inflight, + id, + None, + round_trip, + render_assistant_action_result, + ) + .await + } + "task_progress" => { let message = arguments .get("message") @@ -2576,6 +2731,38 @@ pub fn render_task_report(report: &Value) -> Value { }) } +/// Map the `list_sessions` round-trip outcome (a serialized +/// `AssistantSessionList`) into an MCP `tools/call` result. MCP hosts hand the +/// model only the `content` blocks, so the list is serialized there as JSON +/// text; the same envelope rides along in `structuredContent`. +fn render_assistant_list_sessions_result(outcome: &Value) -> Value { + json!({ + "content": [{ "type": "text", "text": outcome.to_string() }], + "isError": false, + "structuredContent": outcome.clone(), + }) +} + +/// Map an assistant action round-trip outcome (a serialized +/// `AssistantActionResult`) into an MCP `tools/call` result. A refused or +/// failed action is readable text with `isError: false` (the model reports it +/// and moves on), never a protocol error. +fn render_assistant_action_result(outcome: &Value) -> Value { + let text = match ( + outcome.get("outcome").and_then(|v| v.as_str()), + outcome.get("message").and_then(|v| v.as_str()), + ) { + (Some(o), Some(m)) if !m.is_empty() => format!("{o}: {m}"), + (Some(o), _) => o.to_string(), + _ => outcome.to_string(), + }; + json!({ + "content": [{ "type": "text", "text": text }], + "isError": false, + "structuredContent": outcome.clone(), + }) +} + #[cfg(test)] mod tests { use super::*; @@ -3494,6 +3681,66 @@ mod tests { assert_eq!(names, vec!["get_session_info".to_string()]); } + const ASSISTANT_TOOL_NAMES: [&str; 6] = [ + "list_sessions", + "focus_session", + "send_to_session", + "cancel_session", + "answer_permission", + "start_session", + ]; + + const ASSISTANT_ONLY: CompanionFeatures = CompanionFeatures { + delegation: false, + feedback: false, + ask: false, + sessions: false, + tasks: false, + automations: false, + taskboard: false, + browser: false, + browser_eval: false, + assistant: true, + }; + + #[test] + fn allows_tool_gates_all_six_assistant_tools_on_the_assistant_switch() { + for name in ASSISTANT_TOOL_NAMES { + assert!(ASSISTANT_ONLY.allows_tool(name), "{name} with assistant on"); + assert!( + !SESSIONS_ONLY.allows_tool(name), + "{name} with assistant off" + ); + } + } + + /// The schema carries each assistant tool exactly once, and the listing + /// shows all six only when the assistant group is on. + #[tokio::test] + async fn tools_list_includes_assistant_tools_only_when_enabled() { + let all: Vec = serde_json::from_str(TOOL_SCHEMA_JSON).unwrap(); + for name in ASSISTANT_TOOL_NAMES { + let count = all.iter().filter(|t| t["name"] == name).count(); + assert_eq!(count, 1, "{name} in tool_schema.json"); + } + + let list = r#"{"jsonrpc":"2.0","id":1,"method":"tools/list"}"#; + let names = list_tool_names(dispatch_for_test(list).await); + assert!(ASSISTANT_TOOL_NAMES + .iter() + .all(|n| !names.contains(&n.to_string()))); + let names = list_tool_names(dispatch_with_features(SESSIONS_ONLY, list).await); + assert!(!names.contains(&"list_sessions".to_string())); + + let names = list_tool_names(dispatch_with_features(ASSISTANT_ONLY, list).await); + let mut sorted = names.clone(); + sorted.sort(); + let mut expected: Vec = + ASSISTANT_TOOL_NAMES.iter().map(|n| n.to_string()).collect(); + expected.sort(); + assert_eq!(sorted, expected); + } + #[tokio::test] async fn get_session_info_spawns_when_valid_and_enabled() { let line = json!({ @@ -3828,6 +4075,35 @@ mod tests { assert_eq!(parse_max_messages(&json!({ "max_messages": true })), 20); } + /// MCP hosts hand the model only `content`, so the session list must be + /// there: a bare `{ "result": .. }` reached opencode's model as `null`. + #[test] + fn render_assistant_list_sessions_result_puts_the_list_in_content() { + let outcome = json!({ + "sessions": [{ "session_id": 14, "title": "QA tab", "status": "idle" }], + "folders": [{ "folder_id": 1, "name": "ws" }] + }); + let rendered = render_assistant_list_sessions_result(&outcome); + assert_eq!(rendered["isError"], false); + let text = rendered["content"][0]["text"].as_str().unwrap(); + let parsed: Value = serde_json::from_str(text).unwrap(); + assert_eq!(parsed["sessions"][0]["session_id"], 14); + assert_eq!(rendered["structuredContent"], outcome); + } + + #[test] + fn render_assistant_action_result_is_soft_text_with_outcome() { + let outcome = json!({ "outcome": "unsupported", "message": "not an assistant connection" }); + let rendered = render_assistant_action_result(&outcome); + assert_eq!(rendered["isError"], false); + assert_eq!(rendered["content"][0]["type"], "text"); + assert!(rendered["content"][0]["text"] + .as_str() + .unwrap() + .starts_with("unsupported")); + assert_eq!(rendered["structuredContent"]["outcome"], "unsupported"); + } + #[test] fn render_session_result_not_found_is_soft_with_note_text() { let outcome = json!({ @@ -4893,5 +5169,4 @@ mod tests { // Being refused is not a failed tool call: the turn carries on. assert_eq!(refused["isError"], false); } - } diff --git a/src-tauri/src/acp/delegation/listener.rs b/src-tauri/src/acp/delegation/listener.rs index ef32b6bcdd..88deae4b84 100644 --- a/src-tauri/src/acp/delegation/listener.rs +++ b/src-tauri/src/acp/delegation/listener.rs @@ -161,6 +161,7 @@ pub struct DelegationListener { /// exists only in the desktop build, because a browser tab is a native /// webview — server mode gets `NoBrowserTabs`. pub browser: Arc, + pub assistant: Arc, } impl DelegationListener { @@ -175,6 +176,7 @@ impl DelegationListener { tasks: Arc, authoring: Arc, browser: Arc, + assistant: Arc, ) -> Arc { Arc::new(Self { broker, @@ -186,6 +188,7 @@ impl DelegationListener { tasks, authoring, browser, + assistant, }) } @@ -423,6 +426,198 @@ impl DelegationListener { // Untokened on purpose — see `BrokerMessage::Ping`. Answered before // anything else is touched so the probe measures the serve path and // nothing more. + BrokerMessage::AssistantListSessions(req) => { + let Some(entry) = self.tokens.lookup(&req.token).await else { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "not_found", "message": "invalid token"}), + }, + ) + .await?; + return Ok(()); + }; + if !self + .assistant + .is_assistant_connection(&entry.parent_connection_id) + .await + { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "unsupported", "message": "not an assistant connection"}), + }, + ) + .await?; + return Ok(()); + } + let list = self + .assistant + .list_sessions(&entry.parent_connection_id) + .await; + BrokerResponse { + outcome: serde_json::to_value(list).unwrap_or_else(|_| serde_json::Value::Null), + } + } + BrokerMessage::AssistantFocusSession(req) => { + let Some(entry) = self.tokens.lookup(&req.token).await else { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "not_found", "message": "invalid token"}), + }, + ) + .await?; + return Ok(()); + }; + if !self + .assistant + .is_assistant_connection(&entry.parent_connection_id) + .await + { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "unsupported", "message": "not an assistant connection"}), + }, + ) + .await?; + return Ok(()); + } + let res = self.assistant.focus_session(req.session_id).await; + BrokerResponse { + outcome: serde_json::to_value(res).unwrap_or_else(|_| serde_json::Value::Null), + } + } + BrokerMessage::AssistantSendToSession(req) => { + let Some(entry) = self.tokens.lookup(&req.token).await else { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "not_found", "message": "invalid token"}), + }, + ) + .await?; + return Ok(()); + }; + if !self + .assistant + .is_assistant_connection(&entry.parent_connection_id) + .await + { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "unsupported", "message": "not an assistant connection"}), + }, + ) + .await?; + return Ok(()); + } + let res = self + .assistant + .send_to_session(req.session_id, req.text) + .await; + BrokerResponse { + outcome: serde_json::to_value(res).unwrap_or_else(|_| serde_json::Value::Null), + } + } + BrokerMessage::AssistantCancelSession(req) => { + let Some(entry) = self.tokens.lookup(&req.token).await else { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "not_found", "message": "invalid token"}), + }, + ) + .await?; + return Ok(()); + }; + if !self + .assistant + .is_assistant_connection(&entry.parent_connection_id) + .await + { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "unsupported", "message": "not an assistant connection"}), + }, + ) + .await?; + return Ok(()); + } + let res = self.assistant.cancel_session(req.session_id).await; + BrokerResponse { + outcome: serde_json::to_value(res).unwrap_or_else(|_| serde_json::Value::Null), + } + } + BrokerMessage::AssistantAnswerPermission(req) => { + let Some(entry) = self.tokens.lookup(&req.token).await else { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "not_found", "message": "invalid token"}), + }, + ) + .await?; + return Ok(()); + }; + if !self + .assistant + .is_assistant_connection(&entry.parent_connection_id) + .await + { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "unsupported", "message": "not an assistant connection"}), + }, + ) + .await?; + return Ok(()); + } + let res = self + .assistant + .answer_permission(req.session_id, req.decision) + .await; + BrokerResponse { + outcome: serde_json::to_value(res).unwrap_or_else(|_| serde_json::Value::Null), + } + } + BrokerMessage::AssistantStartSession(req) => { + let Some(entry) = self.tokens.lookup(&req.token).await else { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "not_found", "message": "invalid token"}), + }, + ) + .await?; + return Ok(()); + }; + if !self + .assistant + .is_assistant_connection(&entry.parent_connection_id) + .await + { + write_frame( + conn, + &BrokerResponse { + outcome: serde_json::json!({"outcome": "unsupported", "message": "not an assistant connection"}), + }, + ) + .await?; + return Ok(()); + } + let res = self + .assistant + .start_session(req.folder_id, req.agent_type, req.task) + .await; + BrokerResponse { + outcome: serde_json::to_value(res).unwrap_or_else(|_| serde_json::Value::Null), + } + } BrokerMessage::Ping => BrokerResponse { outcome: serde_json::json!({ "ok": true }), }, @@ -1740,6 +1935,216 @@ mod tests { broker } + struct StubAssistant; + #[async_trait] + impl crate::acp::assistant_tools::AssistantToolAccess for StubAssistant { + async fn is_assistant_connection(&self, _conn_id: &str) -> bool { + true + } + async fn list_sessions( + &self, + _exclude_conn_id: &str, + ) -> crate::acp::delegation::transport::AssistantSessionList { + crate::acp::delegation::transport::AssistantSessionList { + sessions: vec![], + folders: vec![], + } + } + async fn focus_session( + &self, + _session_id: i64, + ) -> crate::acp::delegation::transport::AssistantActionResult { + crate::acp::delegation::transport::AssistantActionResult { + outcome: "not_found".to_string(), + message: String::new(), + } + } + async fn send_to_session( + &self, + _session_id: i64, + _text: String, + ) -> crate::acp::delegation::transport::AssistantActionResult { + crate::acp::delegation::transport::AssistantActionResult { + outcome: "disabled".to_string(), + message: String::new(), + } + } + async fn cancel_session( + &self, + _session_id: i64, + ) -> crate::acp::delegation::transport::AssistantActionResult { + crate::acp::delegation::transport::AssistantActionResult { + outcome: "disabled".to_string(), + message: String::new(), + } + } + async fn answer_permission( + &self, + _session_id: i64, + _decision: String, + ) -> crate::acp::delegation::transport::AssistantActionResult { + crate::acp::delegation::transport::AssistantActionResult { + outcome: "disabled".to_string(), + message: String::new(), + } + } + async fn start_session( + &self, + _folder_id: i64, + _agent_type: String, + _task: String, + ) -> crate::acp::delegation::transport::AssistantActionResult { + crate::acp::delegation::transport::AssistantActionResult { + outcome: "disabled".to_string(), + message: String::new(), + } + } + } + + /// Treats only `assistant_conn` as the assistant and lists one fixed + /// session, so the tests can tell a refused call from a served one. + struct ScopedAssistant { + assistant_conn: &'static str, + list_calls: std::sync::atomic::AtomicUsize, + } + #[async_trait] + impl crate::acp::assistant_tools::AssistantToolAccess for ScopedAssistant { + async fn is_assistant_connection(&self, conn_id: &str) -> bool { + conn_id == self.assistant_conn + } + async fn list_sessions( + &self, + _exclude_conn_id: &str, + ) -> crate::acp::delegation::transport::AssistantSessionList { + self.list_calls + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + crate::acp::delegation::transport::AssistantSessionList { + sessions: vec![crate::acp::delegation::transport::AssistantSessionEntry { + session_id: 7, + title: "Fix login".into(), + agent_type: "codex".into(), + folder_id: 1, + folder_name: "app".into(), + status: "idle".into(), + pending_action: None, + }], + folders: vec![], + } + } + async fn focus_session( + &self, + session_id: i64, + ) -> crate::acp::delegation::transport::AssistantActionResult { + StubAssistant.focus_session(session_id).await + } + async fn send_to_session( + &self, + session_id: i64, + text: String, + ) -> crate::acp::delegation::transport::AssistantActionResult { + StubAssistant.send_to_session(session_id, text).await + } + async fn cancel_session( + &self, + session_id: i64, + ) -> crate::acp::delegation::transport::AssistantActionResult { + StubAssistant.cancel_session(session_id).await + } + async fn answer_permission( + &self, + session_id: i64, + decision: String, + ) -> crate::acp::delegation::transport::AssistantActionResult { + StubAssistant.answer_permission(session_id, decision).await + } + async fn start_session( + &self, + folder_id: i64, + agent_type: String, + task: String, + ) -> crate::acp::delegation::transport::AssistantActionResult { + StubAssistant + .start_session(folder_id, agent_type, task) + .await + } + } + + async fn list_sessions_as(token: &str) -> (serde_json::Value, usize) { + let tokens = Arc::new(TokenRegistry::default()); + for (tok, parent) in [("assistant-tok", "assistant-conn"), ("tab-tok", "tab-conn")] { + tokens + .register( + tok.into(), + TokenEntry { + parent_connection_id: parent.into(), + working_dir: PathBuf::from("/repo"), + }, + ) + .await; + } + let assistant = Arc::new(ScopedAssistant { + assistant_conn: "assistant-conn", + list_calls: Default::default(), + }); + let broker = Arc::new(DelegationBroker::new( + Arc::new(MockSpawner::new()) as Arc, + Arc::new(AlwaysRootLookup) as Arc, + )); + let listener = DelegationListener::new( + broker, + tokens, + Arc::new(StaticParentLookup(Some(1))), + Arc::new(StubFeedback::default()), + Arc::new(StubQuestion::default()), + Arc::new(StubSessionInfo::default()), + Arc::new(StubTaskTools), + Arc::new(StubAuthoring::default()), + Arc::new(NoBrowserTabs), + assistant.clone(), + ); + + let (mut client, mut server) = duplex(8 * 1024); + let server_task = tokio::spawn(async move { + listener.serve_one(&mut server).await.unwrap(); + }); + let msg = BrokerMessage::AssistantListSessions( + crate::acp::delegation::transport::BrokerAssistantListSessionsRequest { + token: token.into(), + }, + ); + write_frame(&mut client, &msg).await.unwrap(); + let resp: BrokerResponse = read_frame(&mut client).await.unwrap(); + server_task.await.unwrap(); + let calls = assistant + .list_calls + .load(std::sync::atomic::Ordering::SeqCst); + (resp.outcome, calls) + } + + #[tokio::test] + async fn assistant_list_sessions_with_a_bad_token_is_not_found() { + let (outcome, calls) = list_sessions_as("bogus").await; + assert_eq!(outcome["outcome"], "not_found"); + assert_eq!(calls, 0); + } + + /// A normal tab whose MCP config was hand-edited to `--features assistant` + /// still cannot list sessions: the listener checks the parent connection. + #[tokio::test] + async fn assistant_list_sessions_from_a_non_assistant_parent_is_unsupported() { + let (outcome, calls) = list_sessions_as("tab-tok").await; + assert_eq!(outcome["outcome"], "unsupported"); + assert_eq!(calls, 0); + } + + #[tokio::test] + async fn assistant_list_sessions_from_the_assistant_lists_via_the_access_impl() { + let (outcome, calls) = list_sessions_as("assistant-tok").await; + assert_eq!(calls, 1); + assert_eq!(outcome["sessions"][0]["session_id"], 7); + assert_eq!(outcome["sessions"][0]["title"], "Fix login"); + } + fn make_listener( broker: Arc, tokens: Arc, @@ -1755,6 +2160,7 @@ mod tests { Arc::new(StubTaskTools), Arc::new(StubAuthoring::default()), Arc::new(NoBrowserTabs), + Arc::new(StubAssistant), ) } @@ -1778,6 +2184,7 @@ mod tests { Arc::new(StubTaskTools), Arc::new(StubAuthoring::default()), Arc::new(NoBrowserTabs), + Arc::new(StubAssistant), ) } @@ -1802,6 +2209,7 @@ mod tests { Arc::new(StubTaskTools), Arc::new(StubAuthoring::default()), Arc::new(NoBrowserTabs), + Arc::new(StubAssistant), ) } @@ -1825,6 +2233,7 @@ mod tests { Arc::new(StubTaskTools), Arc::new(StubAuthoring::default()), Arc::new(NoBrowserTabs), + Arc::new(StubAssistant), ) } @@ -1850,6 +2259,7 @@ mod tests { Arc::new(StubTaskTools), authoring, Arc::new(NoBrowserTabs), + Arc::new(StubAssistant), ) } @@ -1874,6 +2284,7 @@ mod tests { Arc::new(StubTaskTools), Arc::new(StubAuthoring::default()), browser, + Arc::new(StubAssistant), ) } diff --git a/src-tauri/src/acp/delegation/service.rs b/src-tauri/src/acp/delegation/service.rs index b051be3a42..5778ff103e 100644 --- a/src-tauri/src/acp/delegation/service.rs +++ b/src-tauri/src/acp/delegation/service.rs @@ -434,6 +434,71 @@ mod tests { } } + #[async_trait] + impl crate::acp::assistant_tools::AssistantToolAccess for Stub { + async fn is_assistant_connection(&self, _conn_id: &str) -> bool { + false + } + async fn list_sessions( + &self, + _exclude_conn_id: &str, + ) -> crate::acp::delegation::transport::AssistantSessionList { + crate::acp::delegation::transport::AssistantSessionList { + sessions: vec![], + folders: vec![], + } + } + async fn focus_session( + &self, + _session_id: i64, + ) -> crate::acp::delegation::transport::AssistantActionResult { + crate::acp::delegation::transport::AssistantActionResult { + outcome: "not_found".to_string(), + message: String::new(), + } + } + async fn send_to_session( + &self, + _session_id: i64, + _text: String, + ) -> crate::acp::delegation::transport::AssistantActionResult { + crate::acp::delegation::transport::AssistantActionResult { + outcome: "disabled".to_string(), + message: "not implemented yet".to_string(), + } + } + async fn cancel_session( + &self, + _session_id: i64, + ) -> crate::acp::delegation::transport::AssistantActionResult { + crate::acp::delegation::transport::AssistantActionResult { + outcome: "disabled".to_string(), + message: "not implemented yet".to_string(), + } + } + async fn answer_permission( + &self, + _session_id: i64, + _decision: String, + ) -> crate::acp::delegation::transport::AssistantActionResult { + crate::acp::delegation::transport::AssistantActionResult { + outcome: "disabled".to_string(), + message: "not implemented yet".to_string(), + } + } + async fn start_session( + &self, + _folder_id: i64, + _agent_type: String, + _task: String, + ) -> crate::acp::delegation::transport::AssistantActionResult { + crate::acp::delegation::transport::AssistantActionResult { + outcome: "disabled".to_string(), + message: "not implemented yet".to_string(), + } + } + } + /// A temp directory short enough to bind a socket inside, whatever the /// ambient `$TMPDIR` happens to be. /// @@ -497,6 +562,7 @@ mod tests { Arc::new(Stub), Arc::new(Stub), Arc::new(Stub), + Arc::new(Stub), ); DelegationService::new(listener, socket_path) } diff --git a/src-tauri/src/acp/delegation/tool_schema.json b/src-tauri/src/acp/delegation/tool_schema.json index a14f46c605..9986fb8b7f 100644 --- a/src-tauri/src/acp/delegation/tool_schema.json +++ b/src-tauri/src/acp/delegation/tool_schema.json @@ -655,5 +655,111 @@ } } } + }, + { + "name": "list_sessions", + "description": "List all active sessions in the workspace, excluding the current one.", + "inputSchema": { + "type": "object", + "properties": {}, + "required": [] + } + }, + { + "name": "focus_session", + "description": "Focus a specific session in the workspace, bringing the app to the foreground. Shows the user a confirmation and returns declined if the user says no.", + "inputSchema": { + "type": "object", + "properties": { + "session_id": { + "type": "integer" + } + }, + "required": [ + "session_id" + ] + } + }, + { + "name": "send_to_session", + "description": "Send a message to another session. Shows the user a confirmation and returns declined if the user says no.", + "inputSchema": { + "type": "object", + "properties": { + "session_id": { + "type": "integer" + }, + "text": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + } + }, + "required": [ + "session_id", + "text" + ] + } + }, + { + "name": "cancel_session", + "description": "Cancel the current turn of another session. Shows the user a confirmation and returns declined if the user says no.", + "inputSchema": { + "type": "object", + "properties": { + "session_id": { + "type": "integer" + } + }, + "required": [ + "session_id" + ] + } + }, + { + "name": "answer_permission", + "description": "Answer a pending permission prompt in another session. Shows the user a confirmation and returns declined if the user says no.", + "inputSchema": { + "type": "object", + "properties": { + "session_id": { + "type": "integer" + }, + "decision": { + "type": "string", + "enum": [ + "approve", + "deny" + ] + } + }, + "required": [ + "session_id", + "decision" + ] + } + }, + { + "name": "start_session", + "description": "Start a new session in a specific folder. Shows the user a confirmation and returns declined if the user says no.", + "inputSchema": { + "type": "object", + "properties": { + "folder_id": { + "type": "integer" + }, + "agent_type": { + "type": "string" + }, + "task": { + "type": "string" + } + }, + "required": [ + "folder_id", + "agent_type", + "task" + ] + } } ] diff --git a/src-tauri/src/acp/delegation/transport.rs b/src-tauri/src/acp/delegation/transport.rs index 4ccc252cbb..832bf0b5bb 100644 --- a/src-tauri/src/acp/delegation/transport.rs +++ b/src-tauri/src/acp/delegation/transport.rs @@ -357,6 +357,12 @@ pub enum BrokerMessage { BrowserCapture(BrokerBrowserCaptureRequest), BrowserEval(BrokerBrowserEvalRequest), BrowserTabOp(BrokerBrowserTabOpRequest), + AssistantListSessions(BrokerAssistantListSessionsRequest), + AssistantFocusSession(BrokerAssistantFocusSessionRequest), + AssistantSendToSession(BrokerAssistantSendToSessionRequest), + AssistantCancelSession(BrokerAssistantCancelSessionRequest), + AssistantAnswerPermission(BrokerAssistantAnswerPermissionRequest), + AssistantStartSession(BrokerAssistantStartSessionRequest), /// Liveness probe. Unlike every other variant this one is NOT sent by a /// companion — it comes from codeg's own service-status check /// (`acp::delegation::service`), which is why it carries no `token`: a @@ -621,6 +627,84 @@ pub async fn client_browser_tab_op_round_trip( message_round_trip(socket_path, &BrokerMessage::BrowserTabOp(req.clone())).await } +/// Dispatch an `assistant_list_sessions` request and read back the serialized +/// [`AssistantSessionList`]. +pub async fn client_assistant_list_sessions_round_trip( + socket_path: &str, + req: &BrokerAssistantListSessionsRequest, +) -> io::Result { + message_round_trip( + socket_path, + &BrokerMessage::AssistantListSessions(req.clone()), + ) + .await +} + +/// Dispatch an `assistant_focus_session` request and read back the serialized +/// [`AssistantActionResult`]. +pub async fn client_assistant_focus_session_round_trip( + socket_path: &str, + req: &BrokerAssistantFocusSessionRequest, +) -> io::Result { + message_round_trip( + socket_path, + &BrokerMessage::AssistantFocusSession(req.clone()), + ) + .await +} + +/// Dispatch an `assistant_send_to_session` request and read back the serialized +/// [`AssistantActionResult`]. +pub async fn client_assistant_send_to_session_round_trip( + socket_path: &str, + req: &BrokerAssistantSendToSessionRequest, +) -> io::Result { + message_round_trip( + socket_path, + &BrokerMessage::AssistantSendToSession(req.clone()), + ) + .await +} + +/// Dispatch an `assistant_cancel_session` request and read back the serialized +/// [`AssistantActionResult`]. +pub async fn client_assistant_cancel_session_round_trip( + socket_path: &str, + req: &BrokerAssistantCancelSessionRequest, +) -> io::Result { + message_round_trip( + socket_path, + &BrokerMessage::AssistantCancelSession(req.clone()), + ) + .await +} + +/// Dispatch an `assistant_answer_permission` request and read back the serialized +/// [`AssistantActionResult`]. +pub async fn client_assistant_answer_permission_round_trip( + socket_path: &str, + req: &BrokerAssistantAnswerPermissionRequest, +) -> io::Result { + message_round_trip( + socket_path, + &BrokerMessage::AssistantAnswerPermission(req.clone()), + ) + .await +} + +/// Dispatch an `assistant_start_session` request and read back the serialized +/// [`AssistantActionResult`]. +pub async fn client_assistant_start_session_round_trip( + socket_path: &str, + req: &BrokerAssistantStartSessionRequest, +) -> io::Result { + message_round_trip( + socket_path, + &BrokerMessage::AssistantStartSession(req.clone()), + ) + .await +} + /// Probe the listener: write a [`BrokerMessage::Ping`] and read the /// `{"ok": true}` answer back. Used by the codeg-mcp service-status indicator /// to tell "listening" from "socket file exists but nobody is accepting". @@ -898,3 +982,72 @@ mod tests { server.await.unwrap(); } } + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrokerAssistantListSessionsRequest { + pub token: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrokerAssistantFocusSessionRequest { + pub token: String, + pub session_id: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrokerAssistantSendToSessionRequest { + pub token: String, + pub session_id: i64, + pub text: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrokerAssistantCancelSessionRequest { + pub token: String, + pub session_id: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrokerAssistantAnswerPermissionRequest { + pub token: String, + pub session_id: i64, + pub decision: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrokerAssistantStartSessionRequest { + pub token: String, + pub folder_id: i64, + pub agent_type: String, + pub task: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssistantSessionEntry { + pub session_id: i64, + pub title: String, + pub agent_type: String, + pub folder_id: i64, + pub folder_name: String, + pub status: String, + pub pending_action: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssistantFolderEntry { + pub folder_id: i64, + pub name: String, + pub path: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssistantSessionList { + pub sessions: Vec, + pub folders: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssistantActionResult { + pub outcome: String, + pub message: String, +} diff --git a/src-tauri/src/acp/manager.rs b/src-tauri/src/acp/manager.rs index 9f285df72d..5a4be4f723 100644 --- a/src-tauri/src/acp/manager.rs +++ b/src-tauri/src/acp/manager.rs @@ -2870,6 +2870,107 @@ impl ConnectionManager { /// it. Lock discipline mirrors `find_connection_by_conversation_id`: hold /// the connections mutex while taking each per-session read lock (the /// reads are microseconds and released each iteration). + pub async fn list_linked_sessions( + &self, + exclude_conn_id: &str, + db: &crate::db::AppDatabase, + ) -> crate::acp::delegation::transport::AssistantSessionList { + use crate::acp::delegation::transport::{AssistantFolderEntry, AssistantSessionEntry}; + use crate::acp::types::ConnectionStatus; + use crate::db::service::{conversation_service, folder_service}; + + let mut out = Vec::new(); + { + let connections = self.connections.lock().await; + for (id, conn) in connections.iter() { + if id == exclude_conn_id + || conn.owner_window_label == "work_task" + || conn.owner_window_label == crate::commands::assistant::ASSISTANT_OWNER_LABEL + { + continue; + } + let state = conn.state.read().await; + let (Some(conversation_id), Some(folder_id)) = + (state.conversation_id, state.folder_id) + else { + continue; + }; + + let mut status = "idle"; + let mut pending_action = None; + if let Some(ref p) = state.pending_permission { + status = "needs_approval"; + // Extract a human-readable label from the tool_call JSON + let action_str = p + .tool_call + .get("name") + .or_else(|| p.tool_call.get("type")) + .and_then(|v| v.as_str()) + .unwrap_or("unknown action") + .to_string(); + pending_action = Some(action_str.chars().take(160).collect()); + } else if state.pending_question.is_some() { + status = "needs_answer"; + } else if matches!(state.status, ConnectionStatus::Prompting) { + status = "working"; + } else if matches!(state.status, ConnectionStatus::Error) { + status = "error"; + } + + out.push(AssistantSessionEntry { + session_id: conversation_id as i64, + title: String::new(), + agent_type: state.agent_type.to_string(), + folder_id: folder_id as i64, + folder_name: String::new(), + status: status.to_string(), + pending_action, + }); + } + } + + let db_conn = &db.conn; + let mut folders_map = std::collections::HashMap::new(); + + // Fill titles and folders + for entry in out.iter_mut() { + if let Ok(conv) = + conversation_service::get_by_id(db_conn, entry.session_id as i32).await + { + if let Some(t) = conv.title { + entry.title = t; + } + } + if let std::collections::hash_map::Entry::Vacant(e) = folders_map.entry(entry.folder_id) + { + if let Ok(Some(f)) = + folder_service::get_folder_by_id(db_conn, entry.folder_id as i32).await + { + e.insert(f); + } + } + if let Some(f) = folders_map.get(&entry.folder_id) { + entry.folder_name = f.name.clone(); + } + } + + let mut all_folders = Vec::new(); + if let Ok(fs) = folder_service::list_folders(db_conn).await { + for f in fs { + all_folders.push(AssistantFolderEntry { + folder_id: f.id as i64, + name: f.name, + path: f.path, + }); + } + } + + crate::acp::delegation::transport::AssistantSessionList { + sessions: out, + folders: all_folders, + } + } + pub async fn list_active_sessions(&self) -> Vec { let connections = self.connections.lock().await; let mut out = Vec::new(); @@ -4380,6 +4481,54 @@ mod tests { } } + /// The assistant sees the workspace's tabs, not itself or work-task + /// runners: both live in the connection map but are not sessions the + /// user would ask about. + #[tokio::test] + async fn list_linked_sessions_excludes_the_assistant_and_work_task_connections() { + use crate::db::service::{conversation_service, folder_service}; + let db = crate::db::test_helpers::fresh_in_memory_db().await; + let folder = folder_service::add_folder(&db.conn, "/tmp/qa-app") + .await + .unwrap(); + let mgr = ConnectionManager::new(); + { + let mut map = mgr.connections.lock().await; + for (id, label) in [ + ("tab", "main"), + ( + "assistant", + crate::commands::assistant::ASSISTANT_OWNER_LABEL, + ), + ("runner", "work_task"), + ] { + let conv = conversation_service::create( + &db.conn, + folder.id, + AgentType::Codex, + Some(format!("{id} chat")), + None, + ) + .await + .unwrap(); + let mut conn = fake_connection(id, Some(conv.id)); + conn.owner_window_label = label.to_string(); + conn.state.write().await.folder_id = Some(folder.id); + map.insert(id.to_string(), conn); + } + } + + let list = mgr.list_linked_sessions("assistant", &db).await; + let titles: Vec<&str> = list.sessions.iter().map(|s| s.title.as_str()).collect(); + assert_eq!(titles, vec!["tab chat"]); + assert_eq!(list.sessions[0].folder_name, folder.name); + assert_eq!(list.folders.len(), 1); + + // Excluded by owner label, not only because it is the caller. + let list = mgr.list_linked_sessions("someone-else", &db).await; + assert_eq!(list.sessions.len(), 1); + } + /// Spawn a two-level process tree: `sh` (the stand-in for the agent CLI) /// backgrounds a `sleep` grandchild (the stand-in for the agent's own /// children — an MCP server, a forked `node`) and records its pid. The diff --git a/src-tauri/src/acp/mod.rs b/src-tauri/src/acp/mod.rs index a01e52d54f..89be64ac57 100644 --- a/src-tauri/src/acp/mod.rs +++ b/src-tauri/src/acp/mod.rs @@ -79,3 +79,5 @@ pub fn continued_session_ids(agent_type: crate::models::AgentType, session_id: & } crate::acp_transcript::continuation_ancestors(registry::registry_id_for(agent_type), session_id) } + +pub mod assistant_tools; diff --git a/src-tauri/src/bin/codeg_server.rs b/src-tauri/src/bin/codeg_server.rs index acc47bf6a1..abab236c1b 100644 --- a/src-tauri/src/bin/codeg_server.rs +++ b/src-tauri/src/bin/codeg_server.rs @@ -392,6 +392,15 @@ async fn async_main() -> ExitCode { // "browser tab" is an iframe their own browser renders, which // nothing here can reach. Arc::new(codeg_lib::acp::browser_tools::NoBrowserTabs), + Arc::new( + codeg_lib::commands::assistant_tools::DbAssistantToolAccess { + manager: Arc::new(state.connection_manager.clone_ref()), + db: Arc::new(codeg_lib::db::AppDatabase { + conn: state.db.conn.clone(), + }), + emitter: Arc::new(state.emitter.clone()), + }, + ), ); // Bind through the service handle rather than a bare `listener.run` // spawn: it keeps the bind error and the accept-loop handle around, so diff --git a/src-tauri/src/commands/assistant_tools.rs b/src-tauri/src/commands/assistant_tools.rs new file mode 100644 index 0000000000..a07a30b073 --- /dev/null +++ b/src-tauri/src/commands/assistant_tools.rs @@ -0,0 +1,114 @@ +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::acp::assistant_tools::AssistantToolAccess; +use crate::acp::delegation::transport::{AssistantActionResult, AssistantSessionList}; +use crate::acp::manager::ConnectionManager; +use crate::commands::assistant::ASSISTANT_OWNER_LABEL; +use crate::db::AppDatabase; +use crate::web::event_bridge::emit_event; +use crate::web::event_bridge::EventEmitter; + +pub struct DbAssistantToolAccess { + pub manager: Arc, + pub db: Arc, + pub emitter: Arc, +} + +#[async_trait] +impl AssistantToolAccess for DbAssistantToolAccess { + async fn is_assistant_connection(&self, conn_id: &str) -> bool { + self.manager + .get_owner_window_label(conn_id) + .await + .as_deref() + == Some(ASSISTANT_OWNER_LABEL) + } + + async fn list_sessions(&self, exclude_conn_id: &str) -> AssistantSessionList { + self.manager + .list_linked_sessions(exclude_conn_id, &self.db) + .await + } + + async fn focus_session(&self, session_id: i64) -> AssistantActionResult { + match crate::db::service::conversation_service::get_by_id(&self.db.conn, session_id as i32) + .await + { + Ok(conv) => { + let folder_id = conv.folder_id; + let agent = serde_json::to_value(conv.agent_type) + .ok() + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_default(); + #[derive(serde::Serialize)] + #[serde(rename_all = "camelCase")] + struct FocusPayload { + folder_id: i32, + conversation_id: i32, + agent: String, + } + let payload = FocusPayload { + folder_id, + conversation_id: session_id as i32, + agent, + }; + emit_event(&self.emitter, "workspace://focus-conversation", payload); + + #[cfg(feature = "tauri-runtime")] + { + if let EventEmitter::Tauri(app_handle) = &*self.emitter { + crate::commands::windows::show_main_window(app_handle); + } + } + + AssistantActionResult { + outcome: "ok".to_string(), + message: "focused".to_string(), + } + } + Err(_) => AssistantActionResult { + outcome: "not_found".to_string(), + message: "session not found".to_string(), + }, + } + } + + async fn send_to_session(&self, _session_id: i64, _text: String) -> AssistantActionResult { + AssistantActionResult { + outcome: "disabled".to_string(), + message: "not implemented yet".to_string(), + } + } + + async fn cancel_session(&self, _session_id: i64) -> AssistantActionResult { + AssistantActionResult { + outcome: "disabled".to_string(), + message: "not implemented yet".to_string(), + } + } + + async fn answer_permission( + &self, + _session_id: i64, + _decision: String, + ) -> AssistantActionResult { + AssistantActionResult { + outcome: "disabled".to_string(), + message: "not implemented yet".to_string(), + } + } + + async fn start_session( + &self, + _folder_id: i64, + _agent_type: String, + _task: String, + ) -> AssistantActionResult { + AssistantActionResult { + outcome: "disabled".to_string(), + message: "not implemented yet".to_string(), + } + } +} diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 9b4c5fb014..ea21f1e350 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -59,3 +59,5 @@ pub mod speech; pub mod work_task; pub mod workspace_state; pub mod assistant; + +pub mod assistant_tools; diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index cce8f3d269..bb8ac59092 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1066,6 +1066,19 @@ mod tauri_app { browser_tools_config.clone(), ), ), + std::sync::Arc::new( + crate::commands::assistant_tools::DbAssistantToolAccess { + manager: std::sync::Arc::new(cm_state.clone_ref()), + db: std::sync::Arc::new(db::AppDatabase { + conn: db_conn.clone(), + }), + emitter: std::sync::Arc::new( + crate::web::event_bridge::EventEmitter::Tauri( + app.handle().clone(), + ), + ), + }, + ), ); // Bind through the service handle rather than a bare // `listener.run` spawn: it keeps the bind error and the diff --git a/src-tauri/tests/delegation_e2e_uds.rs b/src-tauri/tests/delegation_e2e_uds.rs index ea32e08035..88f7b3ce40 100644 --- a/src-tauri/tests/delegation_e2e_uds.rs +++ b/src-tauri/tests/delegation_e2e_uds.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; +use codeg_lib::acp::assistant_tools::AssistantToolAccess; use codeg_lib::acp::delegation::broker::{ ConversationDepthLookup, DelegationBroker, DelegationConfig, }; @@ -23,8 +24,8 @@ use codeg_lib::acp::delegation::listener::{ }; use codeg_lib::acp::delegation::spawner::{mock::MockSpawner, ConnectionSpawner}; use codeg_lib::acp::delegation::transport::{ - client_ask_round_trip, client_round_trip, client_status_round_trip, BrokerAskRequest, - BrokerRequest, BrokerStatusRequest, + client_ask_round_trip, client_round_trip, client_status_round_trip, AssistantActionResult, + AssistantSessionList, BrokerAskRequest, BrokerRequest, BrokerStatusRequest, }; use codeg_lib::acp::delegation::types::{DelegationError, DelegationOutcome, DelegationSuccess}; use codeg_lib::acp::question::{ @@ -136,6 +137,52 @@ impl codeg_lib::acp::chat_authoring::ChatAuthoringAccess for NoAuthoring { } } +/// These tests never reach the assistant tools; every call is refused. +struct NoAssistant; +#[async_trait::async_trait] +impl AssistantToolAccess for NoAssistant { + async fn is_assistant_connection(&self, _conn_id: &str) -> bool { + false + } + async fn list_sessions(&self, _exclude_conn_id: &str) -> AssistantSessionList { + AssistantSessionList { + sessions: vec![], + folders: vec![], + } + } + async fn focus_session(&self, _session_id: i64) -> AssistantActionResult { + no_assistant() + } + async fn send_to_session(&self, _session_id: i64, _text: String) -> AssistantActionResult { + no_assistant() + } + async fn cancel_session(&self, _session_id: i64) -> AssistantActionResult { + no_assistant() + } + async fn answer_permission( + &self, + _session_id: i64, + _decision: String, + ) -> AssistantActionResult { + no_assistant() + } + async fn start_session( + &self, + _folder_id: i64, + _agent_type: String, + _task: String, + ) -> AssistantActionResult { + no_assistant() + } +} + +fn no_assistant() -> AssistantActionResult { + AssistantActionResult { + outcome: "unsupported".to_string(), + message: "no assistant".to_string(), + } +} + /// Controllable question access for the ask round-trip test: `register_question` /// parks a sender keyed by a freshly-minted id; the test pops it via /// `take_pending` and resolves it, exactly as a user answering the card would. @@ -227,6 +274,7 @@ async fn end_to_end_uds_happy_path() { Arc::new(NoAuthoring) as Arc, Arc::new(codeg_lib::acp::browser_tools::NoBrowserTabs) as Arc, + Arc::new(NoAssistant) as Arc, ); // Freshly-named directory per test — no clashes across test bins. @@ -346,6 +394,7 @@ async fn end_to_end_uds_batch_status() { Arc::new(NoAuthoring) as Arc, Arc::new(codeg_lib::acp::browser_tools::NoBrowserTabs) as Arc, + Arc::new(NoAssistant) as Arc, ); let dir = socket_dir(); @@ -436,6 +485,7 @@ async fn end_to_end_uds_invalid_token_rejected() { Arc::new(NoAuthoring) as Arc, Arc::new(codeg_lib::acp::browser_tools::NoBrowserTabs) as Arc, + Arc::new(NoAssistant) as Arc, ); let dir = socket_dir(); @@ -505,6 +555,7 @@ async fn end_to_end_uds_ask_question_round_trip() { Arc::new(NoAuthoring) as Arc, Arc::new(codeg_lib::acp::browser_tools::NoBrowserTabs) as Arc, + Arc::new(NoAssistant) as Arc, ); let dir = socket_dir(); @@ -648,6 +699,7 @@ async fn end_to_end_uds_ask_revoked_after_register_declines() { Arc::new(NoAuthoring) as Arc, Arc::new(codeg_lib::acp::browser_tools::NoBrowserTabs) as Arc, + Arc::new(NoAssistant) as Arc, ); let dir = socket_dir(); From cc8bf6b1b10ac06364e7879ff7fa8298403dac44 Mon Sep 17 00:00:00 2001 From: n0tlu5 Date: Mon, 28 Sep 2026 06:48:00 +0700 Subject: [PATCH 15/15] feat(assistant): add confirmed session control tools Wire send_to_session, cancel_session, answer_permission and start_session behind a Confirm/Cancel card that codeg builds from session state and registers on the requesting assistant's own connection. Each tool checks the assistant settings gates, the target's run state and (for permissions) the pending request before showing any card. answer_permission maps approve/deny to the pending request's allow_once / reject_once option id, never an allow_always / reject_always option, returns unsupported when no one-shot option is offered, and re-checks the request id after confirmation so a permission already answered on the target's own card is not answered twice. Permission prompts for codeg's own assistant tools are auto-allowed once only on the assistant connection; the codeg card is the real gate. Refs #844 --- src-tauri/src/acp/assistant_tools.rs | 23 +- src-tauri/src/acp/connection.rs | 106 ++ src-tauri/src/acp/delegation/listener.rs | 40 +- src-tauri/src/acp/delegation/service.rs | 4 + src-tauri/src/acp/question.rs | 66 + src-tauri/src/bin/codeg_server.rs | 5 + src-tauri/src/commands/assistant_tools.rs | 1405 ++++++++++++++++++++- src-tauri/src/lib.rs | 5 + src-tauri/tests/delegation_e2e_uds.rs | 15 +- 9 files changed, 1635 insertions(+), 34 deletions(-) diff --git a/src-tauri/src/acp/assistant_tools.rs b/src-tauri/src/acp/assistant_tools.rs index 10608d4b53..c14e366a67 100644 --- a/src-tauri/src/acp/assistant_tools.rs +++ b/src-tauri/src/acp/assistant_tools.rs @@ -6,11 +6,28 @@ pub trait AssistantToolAccess: Send + Sync { async fn is_assistant_connection(&self, conn_id: &str) -> bool; async fn list_sessions(&self, exclude_conn_id: &str) -> AssistantSessionList; async fn focus_session(&self, session_id: i64) -> AssistantActionResult; - async fn send_to_session(&self, session_id: i64, text: String) -> AssistantActionResult; - async fn cancel_session(&self, session_id: i64) -> AssistantActionResult; - async fn answer_permission(&self, session_id: i64, decision: String) -> AssistantActionResult; + /// `requester_conn_id` is the assistant's own connection id, used to + /// register the confirmation card on its conversation. + async fn send_to_session( + &self, + requester_conn_id: &str, + session_id: i64, + text: String, + ) -> AssistantActionResult; + async fn cancel_session( + &self, + requester_conn_id: &str, + session_id: i64, + ) -> AssistantActionResult; + async fn answer_permission( + &self, + requester_conn_id: &str, + session_id: i64, + decision: String, + ) -> AssistantActionResult; async fn start_session( &self, + requester_conn_id: &str, folder_id: i64, agent_type: String, task: String, diff --git a/src-tauri/src/acp/connection.rs b/src-tauri/src/acp/connection.rs index cf7d566cb5..5d8da295df 100644 --- a/src-tauri/src/acp/connection.rs +++ b/src-tauri/src/acp/connection.rs @@ -5644,6 +5644,20 @@ async fn run_connection( Ok(()) => return Ok(()), Err(responder) => responder, }; + // An approval gating one of codeg's assistant workspace-action + // tools on the assistant connection is also redundant: codeg's + // own confirmation card is the user-facing gate. + let owner_label = state_inner.read().await.owner_window_label.clone(); + let responder = match try_auto_allow_codeg_assistant_tool( + &owner_label, + &req, + responder, + ) + .await + { + Ok(()) => return Ok(()), + Err(responder) => responder, + }; // pi asks the user a question THROUGH this channel (see // `try_bridge_pi_select_ask`); route it to the interactive // question card instead of an approval card. Every reject @@ -7038,6 +7052,62 @@ async fn handle_grok_ask_user_question( /// durable permission rule into the user's own agent settings — a decision that /// outlives this turn and this connection, so it stays theirs to make. With no /// such option (an agent that offers only "always"), `None` keeps today's card. +/// Find the `allow_once` option id to auto-select when a permission request is +/// gating one of codeg's own assistant workspace-action tools. Mirrors +/// [`codeg_ask_auto_allow_option`]: the confirmation card codeg itself shows +/// IS the user's consent, so asking the user to also approve the raw MCP +/// tool call is a spurious second dialog. +/// +/// Checked only when the caller already knows the connection carries +/// `ASSISTANT_OWNER_LABEL` (see [`try_auto_allow_codeg_assistant_tool`]). +fn codeg_assistant_auto_allow_option(req: &RequestPermissionRequest) -> Option { + let permission_title = req + .meta + .as_ref() + .and_then(|m| m.get("permission")) + .and_then(|p| p.get("title")) + .and_then(serde_json::Value::as_str); + let is_assistant_tool = [req.tool_call.fields.title.as_deref(), permission_title] + .into_iter() + .flatten() + .any(crate::acp::question::is_codeg_assistant_tool_name); + if !is_assistant_tool { + return None; + } + req.options + .iter() + .find(|opt| opt.kind == PermissionOptionKind::AllowOnce) + .map(|opt| opt.option_id.to_string()) +} + +/// Auto-allow a permission request that gates one of codeg's own assistant +/// mutating tools on an ASSISTANT_OWNER_LABEL connection. The codeg-authored +/// confirmation card is the actual user-facing gate; the raw tool-call +/// approval would be a redundant second dialog. +/// +/// `Err(responder)` returns the request to the normal permission path. +#[allow(clippy::result_large_err)] +async fn try_auto_allow_codeg_assistant_tool( + owner_window_label: &str, + req: &RequestPermissionRequest, + responder: Responder, +) -> Result<(), Responder> { + if owner_window_label != crate::commands::assistant::ASSISTANT_OWNER_LABEL { + return Err(responder); + } + let Some(option_id) = codeg_assistant_auto_allow_option(req) else { + return Err(responder); + }; + tracing::debug!( + "[ACP] auto-allowing assistant-tool permission on the assistant connection \ + (option {option_id}); the codeg confirmation card is the actual gate" + ); + let _ = responder.respond(RequestPermissionResponse::new( + RequestPermissionOutcome::Selected(SelectedPermissionOutcome::new(option_id)), + )); + Ok(()) +} + fn codeg_ask_auto_allow_option(req: &RequestPermissionRequest) -> Option { let permission_title = req .meta @@ -18095,6 +18165,42 @@ mod tests { assert!(codeg_ask_auto_allow_option(&always_only).is_none()); } + #[test] + fn codeg_assistant_auto_allow_option_picks_allow_once_for_assistant_tools() { + for tool_name in [ + "mcp__codeg-mcp__send_to_session", + "mcp__codeg-mcp__cancel_session", + "mcp__codeg-mcp__answer_permission", + "mcp__codeg-mcp__start_session", + ] { + let req = + claude_mcp_permission_request(tool_name, claude_permission_options()); + assert_eq!( + codeg_assistant_auto_allow_option(&req).as_deref(), + Some("allow-once"), + "{tool_name} should auto-allow" + ); + } + } + + #[test] + fn codeg_assistant_auto_allow_option_rejects_non_assistant_tools() { + for tool_name in [ + "mcp__other-server__send_to_session", + "mcp__codeg-mcp__ask_user_question", + "mcp__codeg-mcp__list_sessions", + "mcp__codeg-mcp__focus_session", + "Bash", + ] { + let req = + claude_mcp_permission_request(tool_name, claude_permission_options()); + assert!( + codeg_assistant_auto_allow_option(&req).is_none(), + "{tool_name} must keep its approval card" + ); + } + } + #[test] fn codex_retry_indicator_extracts_message_and_object_http_status() { // codex-acp #289: object-variant `codexErrorInfo` carries an inner diff --git a/src-tauri/src/acp/delegation/listener.rs b/src-tauri/src/acp/delegation/listener.rs index 88deae4b84..a760bc3fbc 100644 --- a/src-tauri/src/acp/delegation/listener.rs +++ b/src-tauri/src/acp/delegation/listener.rs @@ -516,7 +516,11 @@ impl DelegationListener { } let res = self .assistant - .send_to_session(req.session_id, req.text) + .send_to_session( + &entry.parent_connection_id, + req.session_id, + req.text, + ) .await; BrokerResponse { outcome: serde_json::to_value(res).unwrap_or_else(|_| serde_json::Value::Null), @@ -547,7 +551,10 @@ impl DelegationListener { .await?; return Ok(()); } - let res = self.assistant.cancel_session(req.session_id).await; + let res = self + .assistant + .cancel_session(&entry.parent_connection_id, req.session_id) + .await; BrokerResponse { outcome: serde_json::to_value(res).unwrap_or_else(|_| serde_json::Value::Null), } @@ -579,7 +586,11 @@ impl DelegationListener { } let res = self .assistant - .answer_permission(req.session_id, req.decision) + .answer_permission( + &entry.parent_connection_id, + req.session_id, + req.decision, + ) .await; BrokerResponse { outcome: serde_json::to_value(res).unwrap_or_else(|_| serde_json::Value::Null), @@ -612,7 +623,12 @@ impl DelegationListener { } let res = self .assistant - .start_session(req.folder_id, req.agent_type, req.task) + .start_session( + &entry.parent_connection_id, + req.folder_id, + req.agent_type, + req.task, + ) .await; BrokerResponse { outcome: serde_json::to_value(res).unwrap_or_else(|_| serde_json::Value::Null), @@ -1961,6 +1977,7 @@ mod tests { } async fn send_to_session( &self, + _requester_conn_id: &str, _session_id: i64, _text: String, ) -> crate::acp::delegation::transport::AssistantActionResult { @@ -1971,6 +1988,7 @@ mod tests { } async fn cancel_session( &self, + _requester_conn_id: &str, _session_id: i64, ) -> crate::acp::delegation::transport::AssistantActionResult { crate::acp::delegation::transport::AssistantActionResult { @@ -1980,6 +1998,7 @@ mod tests { } async fn answer_permission( &self, + _requester_conn_id: &str, _session_id: i64, _decision: String, ) -> crate::acp::delegation::transport::AssistantActionResult { @@ -1990,6 +2009,7 @@ mod tests { } async fn start_session( &self, + _requester_conn_id: &str, _folder_id: i64, _agent_type: String, _task: String, @@ -2039,32 +2059,36 @@ mod tests { } async fn send_to_session( &self, + requester_conn_id: &str, session_id: i64, text: String, ) -> crate::acp::delegation::transport::AssistantActionResult { - StubAssistant.send_to_session(session_id, text).await + StubAssistant.send_to_session(requester_conn_id, session_id, text).await } async fn cancel_session( &self, + requester_conn_id: &str, session_id: i64, ) -> crate::acp::delegation::transport::AssistantActionResult { - StubAssistant.cancel_session(session_id).await + StubAssistant.cancel_session(requester_conn_id, session_id).await } async fn answer_permission( &self, + requester_conn_id: &str, session_id: i64, decision: String, ) -> crate::acp::delegation::transport::AssistantActionResult { - StubAssistant.answer_permission(session_id, decision).await + StubAssistant.answer_permission(requester_conn_id, session_id, decision).await } async fn start_session( &self, + requester_conn_id: &str, folder_id: i64, agent_type: String, task: String, ) -> crate::acp::delegation::transport::AssistantActionResult { StubAssistant - .start_session(folder_id, agent_type, task) + .start_session(requester_conn_id, folder_id, agent_type, task) .await } } diff --git a/src-tauri/src/acp/delegation/service.rs b/src-tauri/src/acp/delegation/service.rs index 5778ff103e..59a0967069 100644 --- a/src-tauri/src/acp/delegation/service.rs +++ b/src-tauri/src/acp/delegation/service.rs @@ -459,6 +459,7 @@ mod tests { } async fn send_to_session( &self, + _requester_conn_id: &str, _session_id: i64, _text: String, ) -> crate::acp::delegation::transport::AssistantActionResult { @@ -469,6 +470,7 @@ mod tests { } async fn cancel_session( &self, + _requester_conn_id: &str, _session_id: i64, ) -> crate::acp::delegation::transport::AssistantActionResult { crate::acp::delegation::transport::AssistantActionResult { @@ -478,6 +480,7 @@ mod tests { } async fn answer_permission( &self, + _requester_conn_id: &str, _session_id: i64, _decision: String, ) -> crate::acp::delegation::transport::AssistantActionResult { @@ -488,6 +491,7 @@ mod tests { } async fn start_session( &self, + _requester_conn_id: &str, _folder_id: i64, _agent_type: String, _task: String, diff --git a/src-tauri/src/acp/question.rs b/src-tauri/src/acp/question.rs index eccedaec67..eafbdc9e8a 100644 --- a/src-tauri/src/acp/question.rs +++ b/src-tauri/src/acp/question.rs @@ -704,6 +704,38 @@ pub fn is_codeg_ask_tool_name(name: &str) -> bool { normalized.ends_with("ask_user_question") && normalized.contains("codeg_mcp") } +/// True when `name` is one of codeg's own assistant mutating tool names as +/// exposed by `codeg-mcp` (the six that the assistant connection's companion +/// advertises). Used by [`crate::acp::connection`] to auto-allow permission +/// requests from ASSISTANT_OWNER_LABEL connections so the confirmation card +/// codeg itself registers is the only thing the user ever sees. +/// +/// Normalises the same way as [`is_codeg_ask_tool_name`]: every agent host +/// mangles MCP tool names slightly, so we strip punctuation to a canonical +/// `codeg_mcp__` shape before comparing. We check both the `codeg_mcp` +/// server token AND the tool name so a same-named tool on a different server +/// never matches. +pub fn is_codeg_assistant_tool_name(name: &str) -> bool { + let normalized = name + .trim() + .to_ascii_lowercase() + .replace(['-', ' ', '.', '/', ':'], "_"); + if !normalized.contains("codeg_mcp") { + return false; + } + matches!( + normalized + .split("codeg_mcp") + .last() + .unwrap_or("") + .trim_matches('_'), + "send_to_session" + | "cancel_session" + | "answer_permission" + | "start_session" + ) +} + /// Serialize a resolved [`QuestionOutcome`] into grok's `AskUserQuestionExtResponse` /// — the reply to a `_x.ai/ask_user_question` ext request. Verified against grok /// 0.2.101 on a real run: the response is internally tagged by `outcome`; the @@ -3161,6 +3193,40 @@ mod tests { } } + #[test] + fn is_codeg_assistant_tool_name_accepts_mutating_assistant_tools() { + for spelling in [ + "mcp__codeg-mcp__send_to_session", + "codeg-mcp/cancel_session", + "mcp__codeg-mcp__answer_permission", + "codeg-mcp: start_session", + " MCP__Codeg-MCP__Send_To_Session ", + ] { + assert!( + is_codeg_assistant_tool_name(spelling), + "{spelling} is a codeg assistant tool" + ); + } + } + + #[test] + fn is_codeg_assistant_tool_name_rejects_non_assistant_and_wrong_server() { + for other in [ + "mcp__other-server__send_to_session", + "send_to_session", + "mcp__codeg-mcp__ask_user_question", + "mcp__codeg-mcp__list_sessions", + "mcp__codeg-mcp__focus_session", + "mcp__codeg-mcp__delegate_to_agent", + "", + ] { + assert!( + !is_codeg_assistant_tool_name(other), + "{other} must not match as assistant tool" + ); + } + } + #[test] fn pi_select_option_id_maps_pick_and_cancels_what_pi_cannot_take() { let ask = parse_pi_select_ask(&pi_select_tool_call(), &pi_select_options()).unwrap(); diff --git a/src-tauri/src/bin/codeg_server.rs b/src-tauri/src/bin/codeg_server.rs index abab236c1b..f50b328b4c 100644 --- a/src-tauri/src/bin/codeg_server.rs +++ b/src-tauri/src/bin/codeg_server.rs @@ -399,6 +399,11 @@ async fn async_main() -> ExitCode { conn: state.db.conn.clone(), }), emitter: Arc::new(state.emitter.clone()), + questions: Arc::new( + codeg_lib::commands::assistant_tools::ManagerQuestions( + Arc::new(state.connection_manager.clone_ref()), + ), + ), }, ), ); diff --git a/src-tauri/src/commands/assistant_tools.rs b/src-tauri/src/commands/assistant_tools.rs index a07a30b073..856816bbf2 100644 --- a/src-tauri/src/commands/assistant_tools.rs +++ b/src-tauri/src/commands/assistant_tools.rs @@ -5,7 +5,9 @@ use async_trait::async_trait; use crate::acp::assistant_tools::AssistantToolAccess; use crate::acp::delegation::transport::{AssistantActionResult, AssistantSessionList}; use crate::acp::manager::ConnectionManager; -use crate::commands::assistant::ASSISTANT_OWNER_LABEL; +use crate::acp::question::{QuestionOption, QuestionSpec, RegisteredQuestion, SessionQuestionAccess}; +use crate::acp::types::PromptInputBlock; +use crate::commands::assistant::{assistant_get_settings_core, ASSISTANT_OWNER_LABEL}; use crate::db::AppDatabase; use crate::web::event_bridge::emit_event; use crate::web::event_bridge::EventEmitter; @@ -14,6 +16,115 @@ pub struct DbAssistantToolAccess { pub manager: Arc, pub db: Arc, pub emitter: Arc, + pub questions: Arc, +} + +pub struct ManagerQuestions(pub Arc); + +#[async_trait] +impl SessionQuestionAccess for ManagerQuestions { + async fn register_question( + &self, + parent: &str, + questions: Vec, + ) -> Option { + self.0.register_question(parent, questions).await + } + async fn cancel_question(&self, parent: &str, id: &str) { + self.0.cancel_question(parent, id).await + } + async fn cancel_questions_by_parent(&self, parent: &str) { + self.0.cancel_questions_by_parent(parent).await + } +} + +struct ConfirmLabels { + confirm: &'static str, + cancel: &'static str, +} + +fn confirm_labels_for(locale: crate::models::system::AppLocale) -> ConfirmLabels { + use crate::models::system::AppLocale; + match locale { + AppLocale::ZhCn => ConfirmLabels { confirm: "确认", cancel: "取消" }, + AppLocale::ZhTw => ConfirmLabels { confirm: "確認", cancel: "取消" }, + AppLocale::Ja => ConfirmLabels { confirm: "確認", cancel: "キャンセル" }, + AppLocale::Ko => ConfirmLabels { confirm: "확인", cancel: "취소" }, + AppLocale::Es => ConfirmLabels { confirm: "Confirmar", cancel: "Cancelar" }, + AppLocale::De => ConfirmLabels { confirm: "Bestätigen", cancel: "Abbrechen" }, + AppLocale::Fr => ConfirmLabels { confirm: "Confirmer", cancel: "Annuler" }, + AppLocale::Pt => ConfirmLabels { confirm: "Confirmar", cancel: "Cancelar" }, + AppLocale::Ar => ConfirmLabels { confirm: "تأكيد", cancel: "إلغاء" }, + AppLocale::En => ConfirmLabels { confirm: "Confirm", cancel: "Cancel" }, + } +} + +async fn load_confirm_labels(db: &sea_orm::DatabaseConnection) -> ConfirmLabels { + let locale = crate::commands::system_settings::load_system_language_settings(db) + .await + .map(|s| s.language) + .unwrap_or_default(); + confirm_labels_for(locale) +} + +fn make_confirm_spec( + question: impl Into, + header: impl Into, + labels: &ConfirmLabels, +) -> QuestionSpec { + QuestionSpec { + id: uuid::Uuid::new_v4().to_string(), + question: question.into(), + header: header.into(), + multi_select: false, + options: vec![ + QuestionOption { label: labels.confirm.to_string(), description: String::new() }, + QuestionOption { label: labels.cancel.to_string(), description: String::new() }, + ], + is_secret: false, + } +} + +async fn ask_confirm( + questions: &dyn SessionQuestionAccess, + requester_conn_id: &str, + spec: QuestionSpec, +) -> bool { + let confirm_label = spec.options[0].label.clone(); + let Some(RegisteredQuestion { answer_rx, .. }) = + questions.register_question(requester_conn_id, vec![spec]).await + else { + return false; + }; + let Ok(outcome) = answer_rx.await else { + return false; + }; + if outcome.declined { + return false; + } + outcome + .answers + .first() + .map(|a| a.selected.first().map(|s| s == &confirm_label).unwrap_or(false)) + .unwrap_or(false) +} + +/// Truncate `s` to at most `max_chars` Unicode scalar values, appending `…` +/// when truncated. +fn truncate_chars(s: &str, max_chars: usize) -> String { + let mut chars = s.chars(); + let prefix: String = chars.by_ref().take(max_chars).collect(); + if chars.next().is_some() { + format!("{prefix}…") + } else { + prefix + } +} + +impl DbAssistantToolAccess { + fn question_access(&self) -> &dyn SessionQuestionAccess { + self.questions.as_ref() + } } #[async_trait] @@ -33,8 +144,11 @@ impl AssistantToolAccess for DbAssistantToolAccess { } async fn focus_session(&self, session_id: i64) -> AssistantActionResult { - match crate::db::service::conversation_service::get_by_id(&self.db.conn, session_id as i32) - .await + match crate::db::service::conversation_service::get_by_id( + &self.db.conn, + session_id as i32, + ) + .await { Ok(conv) => { let folder_id = conv.folder_id; @@ -75,40 +189,1289 @@ impl AssistantToolAccess for DbAssistantToolAccess { } } - async fn send_to_session(&self, _session_id: i64, _text: String) -> AssistantActionResult { - AssistantActionResult { - outcome: "disabled".to_string(), - message: "not implemented yet".to_string(), + async fn send_to_session( + &self, + requester_conn_id: &str, + session_id: i64, + text: String, + ) -> AssistantActionResult { + let settings = match assistant_get_settings_core(&self.db.conn).await { + Ok(s) => s, + Err(e) => return AssistantActionResult { + outcome: "disabled".to_string(), + message: e.to_string(), + }, + }; + if !settings.allow_session_control { + return AssistantActionResult { + outcome: "disabled".to_string(), + message: "session control is disabled in assistant settings".to_string(), + }; + } + + let Some(conn_id) = self + .manager + .find_connection_by_conversation_id(session_id as i32) + .await + else { + return AssistantActionResult { + outcome: "not_running".to_string(), + message: "session is not currently running".to_string(), + }; + }; + + // Check for an in-flight turn on the TARGET session. + if let Some(state_arc) = self.manager.get_state(&conn_id).await { + if state_arc.read().await.turn_in_flight { + return AssistantActionResult { + outcome: "busy".to_string(), + message: "session has a turn in flight".to_string(), + }; + } + } + + let (agent_label, session_title) = self + .session_label_and_title(session_id as i32, &conn_id) + .await; + + let labels = load_confirm_labels(&self.db.conn).await; + let preview = truncate_chars(&text, 300); + let question = format!("{agent_label} · {session_title}: {preview}"); + let spec = make_confirm_spec(question, "Confirm", &labels); + if !ask_confirm(self.question_access(), requester_conn_id, spec).await { + return AssistantActionResult { + outcome: "declined".to_string(), + message: "user declined".to_string(), + }; + } + + let blocks = vec![PromptInputBlock::Text { text }]; + match self + .manager + .send_prompt_linked_with_message_id(&self.db, &conn_id, blocks, None, None, None, None) + .await + { + Ok(_) => AssistantActionResult { + outcome: "ok".to_string(), + message: "message sent".to_string(), + }, + Err(e) => AssistantActionResult { + outcome: "busy".to_string(), + message: e.to_string(), + }, } } - async fn cancel_session(&self, _session_id: i64) -> AssistantActionResult { - AssistantActionResult { - outcome: "disabled".to_string(), - message: "not implemented yet".to_string(), + async fn cancel_session( + &self, + requester_conn_id: &str, + session_id: i64, + ) -> AssistantActionResult { + let settings = match assistant_get_settings_core(&self.db.conn).await { + Ok(s) => s, + Err(e) => return AssistantActionResult { + outcome: "disabled".to_string(), + message: e.to_string(), + }, + }; + if !settings.allow_session_control { + return AssistantActionResult { + outcome: "disabled".to_string(), + message: "session control is disabled in assistant settings".to_string(), + }; + } + + let Some(conn_id) = self + .manager + .find_connection_by_conversation_id(session_id as i32) + .await + else { + return AssistantActionResult { + outcome: "not_running".to_string(), + message: "session is not currently running".to_string(), + }; + }; + + let (agent_label, session_title) = self + .session_label_and_title(session_id as i32, &conn_id) + .await; + + let labels = load_confirm_labels(&self.db.conn).await; + let question = format!("{agent_label} · {session_title}: stop the current turn"); + let spec = make_confirm_spec(question, "Confirm", &labels); + if !ask_confirm(self.question_access(), requester_conn_id, spec).await { + return AssistantActionResult { + outcome: "declined".to_string(), + message: "user declined".to_string(), + }; + } + + match self.manager.cancel(&self.db.conn, &conn_id).await { + Ok(()) => AssistantActionResult { + outcome: "ok".to_string(), + message: "session cancelled".to_string(), + }, + Err(e) => AssistantActionResult { + outcome: "busy".to_string(), + message: e.to_string(), + }, } } async fn answer_permission( &self, - _session_id: i64, - _decision: String, + requester_conn_id: &str, + session_id: i64, + decision: String, ) -> AssistantActionResult { - AssistantActionResult { - outcome: "disabled".to_string(), - message: "not implemented yet".to_string(), + let settings = match assistant_get_settings_core(&self.db.conn).await { + Ok(s) => s, + Err(e) => return AssistantActionResult { + outcome: "disabled".to_string(), + message: e.to_string(), + }, + }; + if !settings.allow_permission_answers { + return AssistantActionResult { + outcome: "disabled".to_string(), + message: "permission answering is disabled in assistant settings".to_string(), + }; + } + + // The MCP schema advertises "approve"/"deny"; we also accept the + // internal "allow_once"/"reject_once" spellings so callers that read + // the option kind directly still work. + let want_allow = match decision.as_str() { + "approve" | "allow_once" => true, + "deny" | "reject_once" => false, + _ => { + return AssistantActionResult { + outcome: "unsupported".to_string(), + message: format!( + "decision must be 'approve' or 'deny', got '{decision}'" + ), + }; + } + }; + + let Some(conn_id) = self + .manager + .find_connection_by_conversation_id(session_id as i32) + .await + else { + return AssistantActionResult { + outcome: "not_running".to_string(), + message: "session is not currently running".to_string(), + }; + }; + + // Read the pending permission to find the option_id, request_id, and + // question action text (command ?? title). All in one read-lock so we + // don't race between the check and the capture. + let (request_id, option_id, action_text, agent_label) = { + let Some(state_arc) = self.manager.get_state(&conn_id).await else { + return AssistantActionResult { + outcome: "not_running".to_string(), + message: "session is not currently running".to_string(), + }; + }; + let state = state_arc.read().await; + let Some(ref p) = state.pending_permission else { + return AssistantActionResult { + outcome: "no_pending_permission".to_string(), + message: "session has no pending permission request".to_string(), + }; + }; + + // Map the user-facing decision to the concrete option_id. + // Never select allow_always / reject_always — those write durable + // rules and belong to the user's own click. + let target_kind = if want_allow { "allow_once" } else { "reject_once" }; + let Some(opt) = p.options.iter().find(|o| o.kind == target_kind) else { + return AssistantActionResult { + outcome: "unsupported".to_string(), + message: format!( + "no '{target_kind}' option available; the permission card stays for a click" + ), + }; + }; + + // Build the question text: command ?? title from the tool_call JSON. + let action = p + .tool_call + .get("command") + .or_else(|| p.tool_call.get("title")) + .and_then(|v| v.as_str()) + .unwrap_or("unknown action") + .to_string(); + + let agent = state.agent_type.to_string(); + + (p.request_id.clone(), opt.option_id.clone(), action, agent) + }; + + let session_title = crate::db::service::conversation_service::get_by_id( + &self.db.conn, + session_id as i32, + ) + .await + .ok() + .and_then(|c| c.title) + .unwrap_or_else(|| format!("session {session_id}")); + + let labels = load_confirm_labels(&self.db.conn).await; + let verb = if want_allow { "approve" } else { "deny" }; + let question = format!( + "{agent_label} · {session_title}: {verb}: {action_text}" + ); + let spec = make_confirm_spec(question, "Confirm", &labels); + if !ask_confirm(self.question_access(), requester_conn_id, spec).await { + return AssistantActionResult { + outcome: "declined".to_string(), + message: "user declined".to_string(), + }; + } + + // re-check: still the same request_id? + // target session's own card while this confirmation was open. + let still_pending = { + let Some(state_arc) = self.manager.get_state(&conn_id).await else { + return AssistantActionResult { + outcome: "not_running".to_string(), + message: "session ended while waiting for confirmation".to_string(), + }; + }; + let state = state_arc.read().await; + state + .pending_permission + .as_ref() + .map(|p| p.request_id == request_id) + .unwrap_or(false) + }; + if !still_pending { + return AssistantActionResult { + outcome: "no_pending_permission".to_string(), + message: "permission was already answered".to_string(), + }; + } + + match self + .manager + .respond_permission(&conn_id, &request_id, &option_id) + .await + { + Ok(()) => AssistantActionResult { + outcome: "ok".to_string(), + message: format!("permission answered with option '{option_id}'"), + }, + Err(e) => AssistantActionResult { + outcome: "busy".to_string(), + message: e.to_string(), + }, } } async fn start_session( &self, - _folder_id: i64, - _agent_type: String, - _task: String, + requester_conn_id: &str, + folder_id: i64, + agent_type: String, + task: String, ) -> AssistantActionResult { + let settings = match assistant_get_settings_core(&self.db.conn).await { + Ok(s) => s, + Err(e) => return AssistantActionResult { + outcome: "disabled".to_string(), + message: e.to_string(), + }, + }; + if !settings.allow_session_control { + return AssistantActionResult { + outcome: "disabled".to_string(), + message: "session control is disabled in assistant settings".to_string(), + }; + } + + let agent_type_parsed = + match serde_json::from_str::(&format!("\"{agent_type}\"")) { + Ok(a) => a, + Err(_) => { + return AssistantActionResult { + outcome: "unsupported".to_string(), + message: format!("unknown agent type: {agent_type}"), + } + } + }; + + let folder = match crate::db::service::folder_service::get_folder_by_id( + &self.db.conn, + folder_id as i32, + ) + .await + { + Ok(Some(f)) => f, + Ok(None) => { + return AssistantActionResult { + outcome: "not_found".to_string(), + message: format!("folder {folder_id} not found"), + } + } + Err(e) => { + return AssistantActionResult { + outcome: "not_found".to_string(), + message: e.to_string(), + } + } + }; + + let labels = load_confirm_labels(&self.db.conn).await; + let task_preview = truncate_chars(&task, 300); + let question = format!( + "{} · {}: {task_preview}", + agent_type_parsed, folder.name + ); + let spec = make_confirm_spec(question, "Confirm", &labels); + if !ask_confirm(self.question_access(), requester_conn_id, spec).await { + return AssistantActionResult { + outcome: "declined".to_string(), + message: "user declined".to_string(), + }; + } + + let conv = match crate::db::service::conversation_service::create( + &self.db.conn, + folder_id as i32, + agent_type_parsed, + None, + None, + ) + .await + { + Ok(c) => c, + Err(e) => { + return AssistantActionResult { + outcome: "busy".to_string(), + message: e.to_string(), + } + } + }; + + let conversation_id = conv.id; + let data_dir = crate::paths::codeg_home_dir(); + let runtime_env = match crate::commands::acp::build_session_runtime_env( + &self.db, + agent_type_parsed, + None, + &data_dir, + ) + .await + { + Ok(env) => env, + Err(e) => { + return AssistantActionResult { + outcome: "busy".to_string(), + message: e.to_string(), + } + } + }; + + let conn_id = match self + .manager + .spawn_agent( + agent_type_parsed, + Some(folder.path.clone()), + None, + runtime_env, + "main".to_string(), + (*self.emitter).clone(), + None, + std::collections::BTreeMap::new(), + ) + .await + { + Ok(id) => id, + Err(e) => { + return AssistantActionResult { + outcome: "busy".to_string(), + message: e.to_string(), + } + } + }; + + let blocks = vec![PromptInputBlock::Text { text: task }]; + if let Err(e) = self + .manager + .send_prompt_linked_with_message_id( + &self.db, + &conn_id, + blocks, + Some(folder_id as i32), + Some(conversation_id), + None, + None, + ) + .await + { + return AssistantActionResult { + outcome: "busy".to_string(), + message: e.to_string(), + }; + } + + let agent_str = serde_json::to_value(agent_type_parsed) + .ok() + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_default(); + #[derive(serde::Serialize)] + #[serde(rename_all = "camelCase")] + struct FocusPayload { + folder_id: i32, + conversation_id: i32, + agent: String, + } + emit_event( + &self.emitter, + "workspace://focus-conversation", + FocusPayload { folder_id: folder_id as i32, conversation_id, agent: agent_str }, + ); + AssistantActionResult { - outcome: "disabled".to_string(), - message: "not implemented yet".to_string(), + outcome: "ok".to_string(), + message: format!("started session {conversation_id}"), + } + } +} + +impl DbAssistantToolAccess { + /// Read the target connection's agent display label and the DB conversation + /// title in one pass. Used to build confirmation card question text. + async fn session_label_and_title(&self, session_id: i32, conn_id: &str) -> (String, String) { + let agent_label = if let Some(state_arc) = self.manager.get_state(conn_id).await { + state_arc.read().await.agent_type.to_string() + } else { + String::new() + }; + let session_title = crate::db::service::conversation_service::get_by_id( + &self.db.conn, + session_id, + ) + .await + .ok() + .and_then(|c| c.title) + .unwrap_or_else(|| format!("session {session_id}")); + (agent_label, session_title) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::Arc; + + use async_trait::async_trait; + use tokio::sync::oneshot; + + use crate::acp::question::{ + QuestionAnsweredItem, QuestionOutcome, QuestionSpec, RegisteredQuestion, + SessionQuestionAccess, + }; + + // --------------------------------------------------------------------------- + // Manual fake questions (pop_sender pattern for join! tests) + // --------------------------------------------------------------------------- + + #[derive(Default)] + struct ManualFakeQuestions { + counter: AtomicUsize, + pending: tokio::sync::Mutex< + HashMap)>, + >, + } + + impl ManualFakeQuestions { + async fn pop_sender(&self) -> Option> { + let mut map = self.pending.lock().await; + let key = map.keys().next().cloned()?; + Some(map.remove(&key).unwrap().1) + } + } + + #[async_trait] + impl SessionQuestionAccess for ManualFakeQuestions { + async fn register_question( + &self, + parent_connection_id: &str, + _questions: Vec, + ) -> Option { + let id = format!("q{}", self.counter.fetch_add(1, Ordering::SeqCst) + 1); + let (tx, rx) = oneshot::channel(); + self.pending + .lock() + .await + .insert(id.clone(), (parent_connection_id.to_string(), tx)); + Some(RegisteredQuestion { question_id: id, answer_rx: rx }) + } + + async fn cancel_question(&self, _parent: &str, id: &str) { + self.pending.lock().await.remove(id); + } + + async fn cancel_questions_by_parent(&self, parent: &str) { + self.pending + .lock() + .await + .retain(|_, (pid, _)| pid != parent); + } + } + + use super::{ask_confirm, confirm_labels_for, make_confirm_spec, truncate_chars, ConfirmLabels}; + + fn en_labels() -> ConfirmLabels { + ConfirmLabels { confirm: "Confirm", cancel: "Cancel" } + } + + // --------------------------------------------------------------------------- + // Existing ask_confirm / label tests (preserved) + // --------------------------------------------------------------------------- + + #[tokio::test] + async fn send_to_session_confirm_returns_true_on_confirm_choice() { + let q = Arc::new(ManualFakeQuestions::default()); + let spec = make_confirm_spec("Send message?", "Confirm", &en_labels()); + let confirm_label = spec.options[0].label.clone(); + let q_clone = Arc::clone(&q); + let (confirmed, ()) = tokio::join!( + ask_confirm(q.as_ref(), "assistant-conn", spec), + async move { + let tx = loop { + if let Some(tx) = q_clone.pop_sender().await { break tx; } + tokio::task::yield_now().await; + }; + let _ = tx.send(QuestionOutcome { + declined: false, + answers: vec![QuestionAnsweredItem { + question: "Send message?".into(), + header: "Confirm".into(), + multi_select: false, + selected: vec![confirm_label], + }], + }); + } + ); + assert!(confirmed, "should return true when user confirms"); + } + + #[tokio::test] + async fn cancel_session_confirm_returns_false_on_cancel_choice() { + let q = Arc::new(ManualFakeQuestions::default()); + let spec = make_confirm_spec("Cancel session?", "Confirm", &en_labels()); + let cancel_label = spec.options[1].label.clone(); + let q_clone = Arc::clone(&q); + let (confirmed, ()) = tokio::join!( + ask_confirm(q.as_ref(), "assistant-conn", spec), + async move { + let tx = loop { + if let Some(tx) = q_clone.pop_sender().await { break tx; } + tokio::task::yield_now().await; + }; + let _ = tx.send(QuestionOutcome { + declined: false, + answers: vec![QuestionAnsweredItem { + question: "Cancel session?".into(), + header: "Confirm".into(), + multi_select: false, + selected: vec![cancel_label], + }], + }); + } + ); + assert!(!confirmed, "should return false when user cancels"); + } + + #[tokio::test] + async fn answer_permission_confirm_returns_false_on_dismissed_card() { + let q = Arc::new(ManualFakeQuestions::default()); + let spec = make_confirm_spec("Allow action?", "Confirm", &en_labels()); + let q_clone = Arc::clone(&q); + let (confirmed, ()) = tokio::join!( + ask_confirm(q.as_ref(), "assistant-conn", spec), + async move { + let tx = loop { + if let Some(tx) = q_clone.pop_sender().await { break tx; } + tokio::task::yield_now().await; + }; + let _ = tx.send(QuestionOutcome { declined: true, answers: vec![] }); + } + ); + assert!(!confirmed, "should return false when user dismisses"); + } + + #[tokio::test] + async fn start_session_confirm_returns_false_when_channel_dropped() { + let q = Arc::new(ManualFakeQuestions::default()); + let spec = make_confirm_spec("Start session?", "Confirm", &en_labels()); + let q_clone = Arc::clone(&q); + let (confirmed, ()) = tokio::join!( + ask_confirm(q.as_ref(), "assistant-conn", spec), + async move { + let tx = loop { + if let Some(tx) = q_clone.pop_sender().await { break tx; } + tokio::task::yield_now().await; + }; + drop(tx); + } + ); + assert!(!confirmed, "should return false when answer channel is dropped"); + } + + #[tokio::test] + async fn ask_confirm_returns_false_when_no_connection() { + struct NullQuestions; + #[async_trait] + impl SessionQuestionAccess for NullQuestions { + async fn register_question( + &self, + _parent: &str, + _questions: Vec, + ) -> Option { + None + } + async fn cancel_question(&self, _parent: &str, _id: &str) {} + async fn cancel_questions_by_parent(&self, _parent: &str) {} + } + let spec = make_confirm_spec("Test?", "Confirm", &en_labels()); + let result = ask_confirm(&NullQuestions, "some-conn", spec).await; + assert!(!result); + } + + #[test] + fn confirm_labels_for_en_returns_english() { + use crate::models::system::AppLocale; + let labels = confirm_labels_for(AppLocale::En); + assert_eq!(labels.confirm, "Confirm"); + assert_eq!(labels.cancel, "Cancel"); + } + + #[test] + fn confirm_labels_for_zh_cn_returns_chinese() { + use crate::models::system::AppLocale; + let labels = confirm_labels_for(AppLocale::ZhCn); + assert_eq!(labels.confirm, "确认"); + assert_eq!(labels.cancel, "取消"); + } + + // --------------------------------------------------------------------------- + // truncate_chars + // --------------------------------------------------------------------------- + + #[test] + fn truncate_chars_short_string_passes_through() { + assert_eq!(truncate_chars("hello", 300), "hello"); + } + + #[test] + fn truncate_chars_exactly_at_limit_passes_through() { + let s: String = "x".repeat(300); + assert_eq!(truncate_chars(&s, 300), s); + } + + #[test] + fn truncate_chars_over_limit_appends_ellipsis() { + let s: String = "x".repeat(301); + let t = truncate_chars(&s, 300); + assert!(t.ends_with('…')); + assert_eq!(t.chars().count(), 301); // 300 x + ellipsis + } + + // --------------------------------------------------------------------------- + // DbAssistantToolAccess unit tests using real in-memory DB + // --------------------------------------------------------------------------- + + use crate::acp::assistant_tools::AssistantToolAccess; + use crate::acp::manager::ConnectionManager; + use crate::acp::session_state::PendingPermissionState; + use crate::acp::types::PermissionOptionInfo; + use crate::commands::assistant::{assistant_set_settings_core, AssistantSettings}; + use crate::db::test_helpers::fresh_in_memory_db; + use crate::models::AgentType; + use crate::web::event_bridge::EventEmitter; + + use super::DbAssistantToolAccess; + + /// Track whether `register_question` was called. + #[derive(Default)] + struct TrackingFakeQuestions { + called: AtomicBool, + answer: Option, // Some(true) = confirm, Some(false) = cancel, None = busy + } + + impl TrackingFakeQuestions { + fn confirming() -> Arc { + Arc::new(Self { answer: Some(true), ..Default::default() }) + } + fn cancelling() -> Arc { + Arc::new(Self { answer: Some(false), ..Default::default() }) + } + + fn was_called(&self) -> bool { + self.called.load(Ordering::SeqCst) + } + } + + #[async_trait] + impl SessionQuestionAccess for TrackingFakeQuestions { + async fn register_question( + &self, + _parent: &str, + questions: Vec, + ) -> Option { + self.called.store(true, Ordering::SeqCst); + let choice = self.answer?; + let spec = questions.into_iter().next()?; + let label = if choice { + spec.options[0].label.clone() + } else { + spec.options[1].label.clone() + }; + let (tx, rx) = oneshot::channel(); + let _ = tx.send(QuestionOutcome { + declined: false, + answers: vec![QuestionAnsweredItem { + question: spec.question.clone(), + header: spec.header.clone(), + multi_select: false, + selected: vec![label], + }], + }); + Some(RegisteredQuestion { + question_id: "test-q".to_string(), + answer_rx: rx, + }) + } + async fn cancel_question(&self, _: &str, _: &str) {} + async fn cancel_questions_by_parent(&self, _: &str) {} + } + + /// Builds a `DbAssistantToolAccess` wired to the given manager, db, and a + /// fake question access. + fn make_access( + manager: Arc, + db: Arc, + questions: Arc, + ) -> DbAssistantToolAccess { + DbAssistantToolAccess { + manager, + db, + emitter: Arc::new(EventEmitter::Noop), + questions, + } + } + + /// Seeds a live test connection on the manager and returns its id. + async fn seed_connection( + manager: &ConnectionManager, + id: &str, + agent_type: AgentType, + conv_id: Option, + owner: &str, + ) { + manager + .insert_test_connection(id, agent_type, None, EventEmitter::Noop) + .await; + let mut connections = manager.connections.lock().await; + let conn = connections.get_mut(id).unwrap(); + conn.owner_window_label = owner.to_string(); + if let Some(cid) = conv_id { + conn.state.write().await.conversation_id = Some(cid); + } + } + + /// Enables assistant settings (allow_session_control + allow_permission_answers). + async fn enable_settings(db: &crate::db::AppDatabase) { + assistant_set_settings_core( + &db.conn, + AssistantSettings { + agent_type: None, + allow_session_control: true, + allow_permission_answers: true, + }, + ) + .await + .unwrap(); + } + + // ---- send_to_session gates ---- + + #[tokio::test] + async fn send_to_session_disabled_when_setting_off() { + let db = Arc::new(fresh_in_memory_db().await); + // settings OFF (default) + let manager = Arc::new(ConnectionManager::new()); + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.send_to_session("asst", 1, "hello".into()).await; + assert_eq!(result.outcome, "disabled"); + assert!(!questions.was_called(), "no card shown when setting is off"); + } + + #[tokio::test] + async fn send_to_session_not_running_when_no_connection() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.send_to_session("asst", 999, "hello".into()).await; + assert_eq!(result.outcome, "not_running"); + assert!(!questions.was_called()); + } + + #[tokio::test] + async fn send_to_session_busy_when_turn_in_flight() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + + // Seed a conversation row. + let folder_id = crate::db::test_helpers::seed_folder(&db, "/tmp/test").await; + let conv_id = + crate::db::test_helpers::seed_conversation(&db, folder_id, AgentType::Codex).await; + + seed_connection(&manager, "target", AgentType::Codex, Some(conv_id), "main").await; + // Mark the turn as in-flight. + { + let conns = manager.connections.lock().await; + conns["target"].state.write().await.turn_in_flight = true; + } + + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.send_to_session("asst", conv_id as i64, "hello".into()).await; + assert_eq!(result.outcome, "busy"); + assert!(!questions.was_called(), "no card shown when turn in flight"); + } + + #[tokio::test] + async fn send_to_session_confirm_proceeds() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + + let folder_id = crate::db::test_helpers::seed_folder(&db, "/tmp/test").await; + let conv_id = + crate::db::test_helpers::seed_conversation(&db, folder_id, AgentType::Codex).await; + + seed_connection(&manager, "target", AgentType::Codex, Some(conv_id), "main").await; + // Note: the cmd receiver is dropped by insert_test_connection, so send + // will get a `ProcessExited` error — we assert it was at least attempted + // (outcome is "busy", not "declined"). + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.send_to_session("asst", conv_id as i64, "do something".into()).await; + assert!(questions.was_called(), "card must be shown"); + // The cmd tx is dropped so send_prompt fails → "busy", not "declined". + assert_ne!(result.outcome, "declined", "user confirmed; must not be declined"); + } + + #[tokio::test] + async fn send_to_session_cancel_returns_declined() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + + let folder_id = crate::db::test_helpers::seed_folder(&db, "/tmp/test").await; + let conv_id = + crate::db::test_helpers::seed_conversation(&db, folder_id, AgentType::Codex).await; + + seed_connection(&manager, "target", AgentType::Codex, Some(conv_id), "main").await; + let questions = TrackingFakeQuestions::cancelling(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.send_to_session("asst", conv_id as i64, "do something".into()).await; + assert!(questions.was_called()); + assert_eq!(result.outcome, "declined"); + } + + // ---- cancel_session gates ---- + + #[tokio::test] + async fn cancel_session_disabled_when_setting_off() { + let db = Arc::new(fresh_in_memory_db().await); + let manager = Arc::new(ConnectionManager::new()); + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.cancel_session("asst", 1).await; + assert_eq!(result.outcome, "disabled"); + assert!(!questions.was_called()); + } + + #[tokio::test] + async fn cancel_session_not_running_when_no_connection() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.cancel_session("asst", 999).await; + assert_eq!(result.outcome, "not_running"); + assert!(!questions.was_called()); + } + + #[tokio::test] + async fn cancel_session_confirm_shows_card() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + + let folder_id = crate::db::test_helpers::seed_folder(&db, "/tmp/test").await; + let conv_id = + crate::db::test_helpers::seed_conversation(&db, folder_id, AgentType::Codex).await; + + seed_connection(&manager, "target", AgentType::Codex, Some(conv_id), "main").await; + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.cancel_session("asst", conv_id as i64).await; + assert!(questions.was_called(), "card must be shown"); + // cmd tx is dropped → cancel fails internally; it's not "declined" + assert_ne!(result.outcome, "declined"); + } + + #[tokio::test] + async fn cancel_session_cancel_returns_declined() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + + let folder_id = crate::db::test_helpers::seed_folder(&db, "/tmp/test").await; + let conv_id = + crate::db::test_helpers::seed_conversation(&db, folder_id, AgentType::Codex).await; + + seed_connection(&manager, "target", AgentType::Codex, Some(conv_id), "main").await; + let questions = TrackingFakeQuestions::cancelling(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.cancel_session("asst", conv_id as i64).await; + assert!(questions.was_called()); + assert_eq!(result.outcome, "declined"); + } + + // ---- answer_permission gates ---- + + #[tokio::test] + async fn answer_permission_disabled_when_setting_off() { + let db = Arc::new(fresh_in_memory_db().await); + // settings OFF by default + let manager = Arc::new(ConnectionManager::new()); + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.answer_permission("asst", 1, "approve".into()).await; + assert_eq!(result.outcome, "disabled"); + assert!(!questions.was_called()); + } + + #[tokio::test] + async fn answer_permission_not_running_when_no_connection() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.answer_permission("asst", 999, "approve".into()).await; + assert_eq!(result.outcome, "not_running"); + assert!(!questions.was_called()); + } + + #[tokio::test] + async fn answer_permission_no_pending_permission() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + + let folder_id = crate::db::test_helpers::seed_folder(&db, "/tmp/test").await; + let conv_id = + crate::db::test_helpers::seed_conversation(&db, folder_id, AgentType::Codex).await; + + seed_connection(&manager, "target", AgentType::Codex, Some(conv_id), "main").await; + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.answer_permission("asst", conv_id as i64, "approve".into()).await; + assert_eq!(result.outcome, "no_pending_permission"); + assert!(!questions.was_called()); + } + + /// Seeds a pending permission on a connection. Returns the request_id. + async fn seed_pending_permission( + manager: &ConnectionManager, + conn_id: &str, + options: Vec, + ) -> String { + let request_id = "req-001".to_string(); + let conns = manager.connections.lock().await; + let mut state = conns[conn_id].state.write().await; + state.pending_permission = Some(PendingPermissionState { + request_id: request_id.clone(), + tool_call_id: "tc-001".to_string(), + tool_call: serde_json::json!({ "title": "Run bash command", "command": "ls -la" }), + options, + created_at: chrono::Utc::now(), + queued: 0, + }); + request_id + } + + fn allow_once_option() -> PermissionOptionInfo { + PermissionOptionInfo { + option_id: "opt-allow-once".to_string(), + name: "Allow once".to_string(), + kind: "allow_once".to_string(), + meta: None, + } + } + + fn reject_once_option() -> PermissionOptionInfo { + PermissionOptionInfo { + option_id: "opt-reject-once".to_string(), + name: "Reject once".to_string(), + kind: "reject_once".to_string(), + meta: None, } } + + fn allow_always_option() -> PermissionOptionInfo { + PermissionOptionInfo { + option_id: "opt-allow-always".to_string(), + name: "Allow always".to_string(), + kind: "allow_always".to_string(), + meta: None, + } + } + + #[tokio::test] + async fn answer_permission_unsupported_when_only_allow_always_offered() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + + let folder_id = crate::db::test_helpers::seed_folder(&db, "/tmp/test").await; + let conv_id = + crate::db::test_helpers::seed_conversation(&db, folder_id, AgentType::Codex).await; + + seed_connection(&manager, "target", AgentType::Codex, Some(conv_id), "main").await; + // Only allow_always offered → unsupported, no card + seed_pending_permission(&manager, "target", vec![allow_always_option()]).await; + + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.answer_permission("asst", conv_id as i64, "approve".into()).await; + assert_eq!(result.outcome, "unsupported"); + assert!(!questions.was_called(), "no card when unsupported"); + } + + #[tokio::test] + async fn answer_permission_picks_allow_once_id_even_when_allow_always_listed_first() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + + let folder_id = crate::db::test_helpers::seed_folder(&db, "/tmp/test").await; + let conv_id = + crate::db::test_helpers::seed_conversation(&db, folder_id, AgentType::Codex).await; + + seed_connection(&manager, "target", AgentType::Codex, Some(conv_id), "main").await; + // allow_always listed FIRST, allow_once second + seed_pending_permission( + &manager, + "target", + vec![allow_always_option(), allow_once_option()], + ) + .await; + + // Track which option_id is passed to respond_permission. + // The cmd tx is dropped, so respond_permission will error — we verify + // the code got past the option selection step by checking it asked + // the confirmation card. + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.answer_permission("asst", conv_id as i64, "approve".into()).await; + // Card WAS shown (option was found). + assert!(questions.was_called(), "card must be shown when allow_once is available"); + // The permission was still pending when we confirmed (no other actor + // cleared it), so respond_permission was called. The cmd tx is dead → + // responds with an error, giving "busy" outcome. NOT "unsupported". + assert_ne!(result.outcome, "unsupported"); + assert_ne!(result.outcome, "declined"); + } + + #[tokio::test] + async fn answer_permission_deny_picks_reject_once_option_id() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + + let folder_id = crate::db::test_helpers::seed_folder(&db, "/tmp/test").await; + let conv_id = + crate::db::test_helpers::seed_conversation(&db, folder_id, AgentType::Codex).await; + + seed_connection(&manager, "target", AgentType::Codex, Some(conv_id), "main").await; + seed_pending_permission( + &manager, + "target", + vec![allow_once_option(), reject_once_option()], + ) + .await; + + let questions = TrackingFakeQuestions::confirming(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + // "deny" → must look for reject_once option + let result = access.answer_permission("asst", conv_id as i64, "deny".into()).await; + assert!(questions.was_called()); + assert_ne!(result.outcome, "unsupported"); + assert_ne!(result.outcome, "declined"); + } + + #[tokio::test] + async fn answer_permission_cancel_returns_declined() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + + let folder_id = crate::db::test_helpers::seed_folder(&db, "/tmp/test").await; + let conv_id = + crate::db::test_helpers::seed_conversation(&db, folder_id, AgentType::Codex).await; + + seed_connection(&manager, "target", AgentType::Codex, Some(conv_id), "main").await; + seed_pending_permission( + &manager, + "target", + vec![allow_once_option(), reject_once_option()], + ) + .await; + + let questions = TrackingFakeQuestions::cancelling(); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), Arc::clone(&questions) as Arc); + let result = access.answer_permission("asst", conv_id as i64, "approve".into()).await; + assert!(questions.was_called()); + assert_eq!(result.outcome, "declined"); + } + + #[tokio::test] + async fn answer_permission_stale_request_id_after_confirmation_no_respond() { + let db = Arc::new(fresh_in_memory_db().await); + enable_settings(&db).await; + let manager = Arc::new(ConnectionManager::new()); + + let folder_id = crate::db::test_helpers::seed_folder(&db, "/tmp/test").await; + let conv_id = + crate::db::test_helpers::seed_conversation(&db, folder_id, AgentType::Codex).await; + + seed_connection(&manager, "target", AgentType::Codex, Some(conv_id), "main").await; + seed_pending_permission( + &manager, + "target", + vec![allow_once_option(), reject_once_option()], + ) + .await; + + // After the user confirms the card, clear the pending_permission to + // simulate the user clicking the real card on the target session. + // We use a custom FakeQuestions that clears the pending permission + // from the target session's state before resolving the answer. + struct ClearOnConfirm { + manager: Arc, + conn_id: String, + } + #[async_trait] + impl SessionQuestionAccess for ClearOnConfirm { + async fn register_question( + &self, + _parent: &str, + questions: Vec, + ) -> Option { + let spec = questions.into_iter().next()?; + let label = spec.options[0].label.clone(); + // Clear the pending permission BEFORE delivering the answer. + { + let conns = self.manager.connections.lock().await; + if let Some(conn) = conns.get(&self.conn_id) { + conn.state.write().await.pending_permission = None; + } + } + let (tx, rx) = oneshot::channel(); + let _ = tx.send(QuestionOutcome { + declined: false, + answers: vec![QuestionAnsweredItem { + question: spec.question, + header: spec.header, + multi_select: false, + selected: vec![label], + }], + }); + Some(RegisteredQuestion { question_id: "q1".into(), answer_rx: rx }) + } + async fn cancel_question(&self, _: &str, _: &str) {} + async fn cancel_questions_by_parent(&self, _: &str) {} + } + + let questions: Arc = Arc::new(ClearOnConfirm { + manager: Arc::clone(&manager), + conn_id: "target".to_string(), + }); + let access = make_access(Arc::clone(&manager), Arc::clone(&db), questions); + let result = access.answer_permission("asst", conv_id as i64, "approve".into()).await; + // The re-check sees no pending permission → no_pending_permission + assert_eq!( + result.outcome, "no_pending_permission", + "stale request id → must not call respond_permission" + ); + } + + // ---- auto-allow / is_codeg_assistant_tool_name ---- + + #[test] + fn is_codeg_assistant_tool_name_accepts_mutating_tools() { + use crate::acp::question::is_codeg_assistant_tool_name; + for name in [ + "codeg_mcp__send_to_session", + "codeg-mcp__cancel_session", + "codeg_mcp__answer_permission", + "codeg_mcp__start_session", + // with server prefix spacing variants + "codeg mcp cancel_session", + ] { + assert!( + is_codeg_assistant_tool_name(name), + "{name} should be recognized as a codeg assistant tool" + ); + } + } + + #[test] + fn is_codeg_assistant_tool_name_rejects_non_assistant_tools() { + use crate::acp::question::is_codeg_assistant_tool_name; + for name in [ + "codeg_mcp__list_sessions", // list/focus are not mutating + "codeg_mcp__focus_session", + "send_to_session", // missing server prefix + "other_mcp__send_to_session", + "ask_user_question", + ] { + assert!( + !is_codeg_assistant_tool_name(name), + "{name} should NOT be recognized as a codeg assistant tool" + ); + } + } + + #[tokio::test] + async fn auto_allow_not_applied_for_non_assistant_owner() { + use crate::acp::question::is_codeg_assistant_tool_name; + use crate::commands::assistant::ASSISTANT_OWNER_LABEL; + let non_assistant_owner = "main"; + assert_ne!(non_assistant_owner, ASSISTANT_OWNER_LABEL); + assert!(is_codeg_assistant_tool_name("codeg_mcp__send_to_session"), + "tool is recognized by name"); + let is_auto_allowed = non_assistant_owner == ASSISTANT_OWNER_LABEL + && is_codeg_assistant_tool_name("codeg_mcp__send_to_session"); + assert!(!is_auto_allowed, + "auto-allow requires ASSISTANT_OWNER_LABEL; 'main' must not trigger it"); + } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index bb8ac59092..57340d710c 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1077,6 +1077,11 @@ mod tauri_app { app.handle().clone(), ), ), + questions: std::sync::Arc::new( + crate::commands::assistant_tools::ManagerQuestions( + std::sync::Arc::new(cm_state.clone_ref()), + ), + ), }, ), ); diff --git a/src-tauri/tests/delegation_e2e_uds.rs b/src-tauri/tests/delegation_e2e_uds.rs index 88f7b3ce40..c86dfd6f10 100644 --- a/src-tauri/tests/delegation_e2e_uds.rs +++ b/src-tauri/tests/delegation_e2e_uds.rs @@ -153,14 +153,24 @@ impl AssistantToolAccess for NoAssistant { async fn focus_session(&self, _session_id: i64) -> AssistantActionResult { no_assistant() } - async fn send_to_session(&self, _session_id: i64, _text: String) -> AssistantActionResult { + async fn send_to_session( + &self, + _requester_conn_id: &str, + _session_id: i64, + _text: String, + ) -> AssistantActionResult { no_assistant() } - async fn cancel_session(&self, _session_id: i64) -> AssistantActionResult { + async fn cancel_session( + &self, + _requester_conn_id: &str, + _session_id: i64, + ) -> AssistantActionResult { no_assistant() } async fn answer_permission( &self, + _requester_conn_id: &str, _session_id: i64, _decision: String, ) -> AssistantActionResult { @@ -168,6 +178,7 @@ impl AssistantToolAccess for NoAssistant { } async fn start_session( &self, + _requester_conn_id: &str, _folder_id: i64, _agent_type: String, _task: String,