diff --git a/src/components/message/message-list-view.tsx b/src/components/message/message-list-view.tsx
index c41aa440da..ae1bd2f0fe 100644
--- a/src/components/message/message-list-view.tsx
+++ b/src/components/message/message-list-view.tsx
@@ -981,6 +981,7 @@ const HistoricalMessageGroup = memo(function HistoricalMessageGroup({
previousUserIndex={previousUserIndex}
isResponseComplete={isResponseComplete}
copyText={extractTextFromParts(group.parts)}
+ speechId={group.id}
completedAt={group.completed_at}
forkDisabled={forkDisabled || forkPointUnnamed}
forkDisabledReason={forkPointUnnamed ? "unnamed" : "busy"}
diff --git a/src/components/message/turn-stats.test.tsx b/src/components/message/turn-stats.test.tsx
index 00fffb6936..86cf20eae5 100644
--- a/src/components/message/turn-stats.test.tsx
+++ b/src/components/message/turn-stats.test.tsx
@@ -10,7 +10,24 @@ vi.mock("./use-create-task-from-message", () => ({
useCreateTaskFromMessage: () => () => {},
}))
+const playerSpies = vi.hoisted(() => ({
+ speak: vi.fn(),
+ stopSpeech: vi.fn(),
+}))
+vi.mock("@/lib/speech-player", async (importOriginal) => {
+ const actual = await importOriginal
()
+ playerSpies.speak.mockImplementation(actual.speak)
+ playerSpies.stopSpeech.mockImplementation(actual.stopSpeech)
+ return { ...actual, ...playerSpies }
+})
+
import { TurnStats } from "./turn-stats"
+import { resetSpeechPlayerForTests } from "@/lib/speech-player"
+import {
+ DEFAULT_SPEECH_PREFS,
+ resetSpeechPrefsCacheForTests,
+ saveSpeechPrefs,
+} from "@/lib/speech-prefs"
import { MessageScrollProvider } from "./message-scroll-context"
import { ModelLabelProvider } from "./model-label-context"
import type { ModelLabelResolver } from "@/hooks/use-model-labels"
@@ -241,3 +258,70 @@ describe("TurnStats zeroed counters", () => {
expect(screen.getByLabelText(tokenStatsLabel)).toBeInTheDocument()
})
})
+
+describe("TurnStats read aloud", () => {
+ const labels = enMessages.Folder.chat.messageList
+
+ function setup(enabled: boolean) {
+ localStorage.clear()
+ resetSpeechPrefsCacheForTests()
+ resetSpeechPlayerForTests()
+ playerSpies.speak.mockClear()
+ playerSpies.stopSpeech.mockClear()
+ vi.stubGlobal("speechSynthesis", {
+ getVoices: () => [{ voiceURI: "v", lang: "en-US" }],
+ speak: vi.fn(),
+ cancel: vi.fn(),
+ })
+ vi.stubGlobal(
+ "SpeechSynthesisUtterance",
+ class {
+ constructor(public text: string) {}
+ }
+ )
+ saveSpeechPrefs({ output: { ...DEFAULT_SPEECH_PREFS.output, enabled } })
+ }
+
+ it("is hidden while read aloud is disabled", () => {
+ setup(false)
+ renderStats()
+ expect(screen.queryByLabelText(labels.readAloud)).not.toBeInTheDocument()
+ vi.unstubAllGlobals()
+ })
+
+ it("is hidden without text or on an unfinished reply", () => {
+ setup(true)
+ renderStats()
+ renderStats(
+
+ )
+ expect(screen.queryByLabelText(labels.readAloud)).not.toBeInTheDocument()
+ vi.unstubAllGlobals()
+ })
+
+ it("speaks the reply text, then stops on the second click", async () => {
+ setup(true)
+ const user = userEvent.setup()
+ renderStats()
+
+ await user.click(screen.getByLabelText(labels.readAloud))
+ expect(playerSpies.speak).toHaveBeenCalledWith(
+ "turn-1",
+ "Hello **there**",
+ expect.objectContaining({ language: "en-US" })
+ )
+ const active = screen.getByRole("button", { pressed: true })
+ expect([labels.readAloudLoading, labels.stopReading]).toContain(
+ active.getAttribute("aria-label")
+ )
+
+ await user.click(active)
+ expect(playerSpies.stopSpeech).toHaveBeenCalled()
+ expect(screen.getByLabelText(labels.readAloud)).toBeInTheDocument()
+ vi.unstubAllGlobals()
+ })
+})
diff --git a/src/components/message/turn-stats.tsx b/src/components/message/turn-stats.tsx
index 77e2d59637..b454b43f70 100644
--- a/src/components/message/turn-stats.tsx
+++ b/src/components/message/turn-stats.tsx
@@ -8,9 +8,13 @@ import {
Coins,
CopyIcon,
ListTodo,
+ Loader2,
Split,
+ Square,
+ Volume2,
} from "lucide-react"
import { useLocale, useTranslations } from "next-intl"
+import { toast } from "sonner"
import {
Tooltip,
TooltipContent,
@@ -20,6 +24,9 @@ import {
import { useMessageScroll } from "@/components/message/message-scroll-context"
import { useModelLabel } from "@/components/message/model-label-context"
import { useCreateTaskFromMessage } from "./use-create-task-from-message"
+import { resolveSpeechLanguage } from "@/lib/speech-capabilities"
+import { speak, stopSpeech, useSpeechPlayer } from "@/lib/speech-player"
+import { useSpeechPrefs } from "@/lib/speech-prefs"
import { formatTokenCount } from "@/lib/token-format"
import { cn, copyTextToClipboard } from "@/lib/utils"
import type { TurnUsage } from "@/lib/types"
@@ -46,6 +53,8 @@ interface TurnStatsProps {
* name the backend can resolve yet (`unnamed` — the post-turn reparse fills
* it in a moment later). Only read while `forkDisabled`. */
forkDisabledReason?: "busy" | "unnamed"
+ /** Stable id of this reply; enables the read-aloud action when set. */
+ speechId?: string
}
const iconButtonClass =
@@ -63,6 +72,7 @@ export function TurnStats({
onForkFromHere,
forkDisabled = false,
forkDisabledReason = "busy",
+ speechId,
}: TurnStatsProps) {
const locale = useLocale()
const t = useTranslations("Folder.chat.messageList")
@@ -110,6 +120,16 @@ export function TurnStats({
(id) => modelLabel(id) ?? id
)
const hasCopy = copyText.trim().length > 0
+ const speechPrefs = useSpeechPrefs()
+ const readAloudEnabled = speechPrefs.output.enabled
+ const player = useSpeechPlayer()
+ const hasReadAloud = readAloudEnabled && hasCopy && Boolean(speechId)
+ const isThisSpeaking = hasReadAloud && player.playingId === speechId
+ const readAloudLabel = isThisSpeaking
+ ? player.status === "loading"
+ ? t("readAloudLoading")
+ : t("stopReading")
+ : t("readAloud")
const hasUsage = Boolean(usage)
// An all-zero usage means "nobody said", not "nothing was spent": a reply
// that exists cannot have cost zero tokens. Qoder zeroes every counter for
@@ -157,6 +177,24 @@ export function TurnStats({
timeoutRef.current = window.setTimeout(() => setIsCopied(false), 2000)
}, [copyText, hasCopy, isCopied])
+ const handleReadAloud = useCallback(() => {
+ if (!speechId) return
+ if (isThisSpeaking) {
+ stopSpeech()
+ return
+ }
+ speak(speechId, copyText, {
+ language: resolveSpeechLanguage(speechPrefs.input, locale),
+ labels: {
+ codeOmitted: t("speechCodeOmitted"),
+ tableOmitted: t("speechTableOmitted"),
+ },
+ onError: () => {
+ toast.error(t("readAloudFailed"))
+ },
+ })
+ }, [copyText, isThisSpeaking, locale, speechId, speechPrefs.input, t])
+
useEffect(
() => () => {
window.clearTimeout(timeoutRef.current)
@@ -194,6 +232,31 @@ export function TurnStats({
)}
+ {hasReadAloud && (
+
+
+
+
+ {readAloudLabel}
+
+ )}
{hasCopy && (
diff --git a/src/components/settings/settings-shell.tsx b/src/components/settings/settings-shell.tsx
index 977646d6da..1a108b882d 100644
--- a/src/components/settings/settings-shell.tsx
+++ b/src/components/settings/settings-shell.tsx
@@ -8,6 +8,7 @@ import {
type ReactNode,
} from "react"
import {
+ AudioLines,
Bot,
BookOpenText,
Boxes,
@@ -45,6 +46,7 @@ interface SettingsNavItem {
href: string
labelKey:
| "general"
+ | "speech"
| "appearance"
| "agents"
| "model_providers"
@@ -74,6 +76,11 @@ const SETTINGS_NAV_ITEMS: SettingsNavItem[] = [
labelKey: "general",
icon: SlidersHorizontal,
},
+ {
+ href: "/settings/speech",
+ labelKey: "speech",
+ icon: AudioLines,
+ },
{
href: "/settings/mcp",
labelKey: "mcp",
diff --git a/src/components/settings/speech-settings.test.tsx b/src/components/settings/speech-settings.test.tsx
new file mode 100644
index 0000000000..1d4fdf3fab
--- /dev/null
+++ b/src/components/settings/speech-settings.test.tsx
@@ -0,0 +1,251 @@
+import { cleanup, render, screen, waitFor } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { NextIntlClientProvider } from "next-intl"
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"
+
+import enMessages from "@/i18n/messages/en.json"
+import type { SpeechCapabilities } from "@/lib/speech-capabilities"
+import {
+ getSpeechPrefs,
+ resetSpeechPrefsCacheForTests,
+ saveSpeechPrefs,
+} from "@/lib/speech-prefs"
+import type { SpeechCloudSettings, SpeechCloudSettingsView } from "@/lib/types"
+
+const getSettings = vi.fn<() => Promise>()
+const updateSettings =
+ vi.fn<
+ (
+ s: SpeechCloudSettings,
+ k: string | null
+ ) => Promise
+ >()
+const toastError = vi.fn()
+const toastSuccess = vi.fn()
+let caps: SpeechCapabilities
+
+vi.mock("@/lib/api", () => ({
+ speechGetSettings: () => getSettings(),
+ speechUpdateSettings: (s: SpeechCloudSettings, k: string | null) =>
+ updateSettings(s, k),
+}))
+vi.mock("@/lib/speech-capabilities", async (importOriginal) => ({
+ ...(await importOriginal()),
+ detectSpeechCapabilities: () => caps,
+}))
+vi.mock("sonner", () => ({
+ toast: {
+ error: (m: string) => toastError(m),
+ success: (m: string) => toastSuccess(m),
+ },
+}))
+
+import { SpeechSettings } from "./speech-settings"
+
+const CLOUD: SpeechCloudSettings = {
+ baseUrl: "https://api.openai.com/v1",
+ sttModel: "whisper-1",
+ ttsModel: "tts-1",
+ ttsVoice: "alloy",
+}
+
+function view(apiKeySet: boolean, settings = CLOUD): SpeechCloudSettingsView {
+ return { settings, apiKeySet }
+}
+
+function renderPage() {
+ return render(
+
+
+
+ )
+}
+
+function enableInput(engine: "auto" | "browser" | "cloud" = "auto") {
+ saveSpeechPrefs({ input: { enabled: true, engine, language: "" } })
+}
+
+beforeEach(() => {
+ localStorage.clear()
+ resetSpeechPrefsCacheForTests()
+ caps = { browserStt: true, mediaCapture: true, secureContext: true }
+ getSettings.mockReset()
+ updateSettings.mockReset()
+ toastError.mockClear()
+ toastSuccess.mockClear()
+ getSettings.mockResolvedValue(view(false))
+})
+afterEach(() => cleanup())
+
+describe("SpeechSettings", () => {
+ it("persists the voice input switch to prefs", async () => {
+ const user = userEvent.setup()
+ renderPage()
+
+ await user.click(await screen.findByRole("switch", { name: "Voice input" }))
+
+ expect(getSpeechPrefs().input.enabled).toBe(true)
+ expect(await screen.findByText("Recognition engine")).toBeInTheDocument()
+ })
+
+ it("persists the engine choice", async () => {
+ const user = userEvent.setup()
+ enableInput()
+ renderPage()
+
+ await user.click(
+ await screen.findByRole("combobox", { name: "Recognition engine" })
+ )
+ await user.click(await screen.findByRole("option", { name: "Cloud" }))
+
+ expect(getSpeechPrefs().input.engine).toBe("cloud")
+ })
+
+ it("sends no key when the key field is untouched", async () => {
+ const user = userEvent.setup()
+ getSettings.mockResolvedValue(view(true))
+ updateSettings.mockResolvedValue(view(true))
+ renderPage()
+
+ await user.click(await screen.findByRole("button", { name: "Save" }))
+
+ await waitFor(() => expect(updateSettings).toHaveBeenCalledTimes(1))
+ expect(updateSettings.mock.calls[0][1]).toBeNull()
+ expect(toastSuccess).toHaveBeenCalledWith("Speech settings saved")
+ })
+
+ it("sends a typed key and then shows it as saved", async () => {
+ const user = userEvent.setup()
+ updateSettings.mockResolvedValue(view(true))
+ renderPage()
+
+ await user.type(await screen.findByLabelText("API key"), "sk-test")
+ await user.click(screen.getByRole("button", { name: "Save" }))
+
+ await waitFor(() =>
+ expect(updateSettings).toHaveBeenCalledWith(CLOUD, "sk-test")
+ )
+ const keyInput = await screen.findByLabelText("API key")
+ expect(keyInput).toHaveValue("")
+ expect(keyInput).toHaveAttribute("placeholder", "Saved")
+ })
+
+ it("Remove key sends an empty key", async () => {
+ const user = userEvent.setup()
+ getSettings.mockResolvedValue(view(true))
+ updateSettings.mockResolvedValue(view(false))
+ renderPage()
+
+ await user.click(await screen.findByRole("button", { name: "Remove key" }))
+
+ await waitFor(() => expect(updateSettings).toHaveBeenCalledWith(CLOUD, ""))
+ expect(
+ screen.queryByRole("button", { name: "Remove key" })
+ ).not.toBeInTheDocument()
+ })
+
+ it("reports a rejected save and keeps the key state", async () => {
+ const user = userEvent.setup()
+ updateSettings.mockRejectedValue(new Error("invalid base url"))
+ renderPage()
+
+ const baseUrl = await screen.findByLabelText("Base URL")
+ await user.clear(baseUrl)
+ await user.type(baseUrl, "ftp://x")
+ await user.click(screen.getByRole("button", { name: "Save" }))
+
+ await waitFor(() => expect(toastError).toHaveBeenCalledTimes(1))
+ expect(toastError.mock.calls[0][0]).toContain("invalid base url")
+ expect(toastSuccess).not.toHaveBeenCalled()
+ })
+
+ it("status line names the unavailable reason", async () => {
+ caps = { browserStt: false, mediaCapture: true, secureContext: true }
+ enableInput("auto")
+ renderPage()
+
+ expect(await screen.findByTestId("speech-engine-status")).toHaveTextContent(
+ "Unavailable: the cloud service has no API key yet."
+ )
+ })
+
+ it("status line names the engine in use", async () => {
+ caps = { browserStt: false, mediaCapture: true, secureContext: true }
+ getSettings.mockResolvedValue(view(true))
+ enableInput("auto")
+ renderPage()
+
+ expect(await screen.findByTestId("speech-engine-status")).toHaveTextContent(
+ "Using: Cloud"
+ )
+ })
+})
+
+describe("SpeechSettings read aloud", () => {
+ const m = enMessages.SpeechSettings
+
+ afterEach(() => vi.unstubAllGlobals())
+
+ it("enables read aloud and shows its controls", async () => {
+ vi.stubGlobal("speechSynthesis", {
+ getVoices: () => [
+ { voiceURI: "fr", name: "Amelie", lang: "fr-FR" },
+ { voiceURI: "en", name: "Samantha", lang: "en-US" },
+ ],
+ })
+ const user = userEvent.setup()
+ renderPage()
+ await user.click(await screen.findByLabelText(m.outputTitle))
+ expect(getSpeechPrefs().output.enabled).toBe(true)
+ expect(await screen.findByTestId("speech-output-status")).toHaveTextContent(
+ "Using: Browser"
+ )
+ expect(screen.getByLabelText(m.rateLabel)).toBeInTheDocument()
+ expect(screen.getByLabelText(m.voiceLabel)).toBeInTheDocument()
+
+ await user.click(screen.getByLabelText(m.autoReadLabel))
+ expect(getSpeechPrefs().output.autoRead).toBe(true)
+
+ await user.click(screen.getByLabelText(m.outputTitle))
+ expect(getSpeechPrefs().output).toMatchObject({
+ enabled: false,
+ autoRead: false,
+ })
+ })
+
+ it("reports missing voices and key when nothing can speak", async () => {
+ vi.stubGlobal("speechSynthesis", undefined)
+ saveSpeechPrefs({
+ output: {
+ enabled: true,
+ engine: "auto",
+ browserVoiceUri: "",
+ rate: 1,
+ autoRead: false,
+ },
+ })
+ renderPage()
+ expect(await screen.findByTestId("speech-output-status")).toHaveTextContent(
+ m.reasonCloudNotConfigured
+ )
+ expect(screen.queryByLabelText(m.voiceLabel)).not.toBeInTheDocument()
+ })
+
+ it("saves the TTS model and voice with the cloud settings", async () => {
+ updateSettings.mockResolvedValue(view(false))
+ const user = userEvent.setup()
+ renderPage()
+ const model = await screen.findByLabelText(m.ttsModel)
+ await user.clear(model)
+ await user.type(model, "gpt-4o-mini-tts")
+ const voice = screen.getByLabelText(m.ttsVoice)
+ await user.clear(voice)
+ await user.type(voice, "nova")
+ await user.click(screen.getByRole("button", { name: m.save }))
+ await waitFor(() => expect(updateSettings).toHaveBeenCalled())
+ expect(updateSettings.mock.calls[0][0]).toMatchObject({
+ ttsModel: "gpt-4o-mini-tts",
+ ttsVoice: "nova",
+ })
+ })
+})
diff --git a/src/components/settings/speech-settings.tsx b/src/components/settings/speech-settings.tsx
new file mode 100644
index 0000000000..61d2bbefb8
--- /dev/null
+++ b/src/components/settings/speech-settings.tsx
@@ -0,0 +1,610 @@
+"use client"
+
+import {
+ useCallback,
+ useEffect,
+ useMemo,
+ useState,
+ useSyncExternalStore,
+} from "react"
+import {
+ AudioLines,
+ Cloud,
+ Cpu,
+ KeyRound,
+ Languages,
+ Link,
+ Gauge,
+ Loader2,
+ Mic,
+ Speaker,
+ Volume2,
+ Wand2,
+} from "lucide-react"
+import { useLocale, useTranslations } from "next-intl"
+import { toast } from "sonner"
+
+import { SettingCard, SettingRow } from "@/components/shared/setting-card"
+import {
+ SettingsError,
+ SettingsSaveBar,
+ SettingsSection,
+} from "@/components/shared/settings-section"
+import { Button } from "@/components/ui/button"
+import { Input } from "@/components/ui/input"
+import { ScrollArea } from "@/components/ui/scroll-area"
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select"
+import { Slider } from "@/components/ui/slider"
+import { Switch } from "@/components/ui/switch"
+import { speechGetSettings, speechUpdateSettings } from "@/lib/api"
+import { toErrorMessage } from "@/lib/app-error"
+import {
+ LOCALE_TO_BCP47,
+ detectSpeechCapabilities,
+ resolveInputEngine,
+ resolveOutputEngine,
+ resolveSpeechLanguage,
+ waitForVoices,
+} from "@/lib/speech-capabilities"
+import {
+ MAX_SPEECH_RATE,
+ MIN_SPEECH_RATE,
+ saveSpeechPrefs,
+ useSpeechPrefs,
+ type SpeechEnginePreference,
+} from "@/lib/speech-prefs"
+import type { SpeechCloudSettings } from "@/lib/types"
+
+const LANGUAGE_FOLLOW_APP = "follow-app"
+const VOICE_DEFAULT = "default"
+const LANGUAGE_CUSTOM = "custom"
+const LANGUAGE_TAGS = Array.from(new Set(Object.values(LOCALE_TO_BCP47)))
+
+const REASON_KEYS = {
+ "no-mic": "reasonNoMic",
+ "insecure-context": "reasonInsecure",
+ "no-engine": "reasonNoEngine",
+ "cloud-not-configured": "reasonCloudNotConfigured",
+} as const
+
+const OUTPUT_REASON_KEYS = {
+ "no-engine": "reasonNoVoices",
+ "cloud-not-configured": "reasonCloudNotConfigured",
+} as const
+
+const subscribeNever = () => () => {}
+const onClient = () => true
+const onServer = () => false
+
+function languageName(tag: string, locale: string): string {
+ try {
+ return new Intl.DisplayNames([locale], { type: "language" }).of(tag) ?? tag
+ } catch {
+ return tag
+ }
+}
+
+export function SpeechSettings() {
+ const t = useTranslations("SpeechSettings")
+ const locale = useLocale()
+ const prefs = useSpeechPrefs()
+
+ const [loading, setLoading] = useState(true)
+ const [loadError, setLoadError] = useState(null)
+ const [cloud, setCloud] = useState(null)
+ const [apiKeySet, setApiKeySet] = useState(false)
+ const [apiKeyDraft, setApiKeyDraft] = useState("")
+ const [saving, setSaving] = useState(false)
+ const [customLanguageMode, setCustomLanguageMode] = useState(false)
+ const [voices, setVoices] = useState(null)
+
+ const mounted = useSyncExternalStore(subscribeNever, onClient, onServer)
+ const caps = useMemo(
+ () => (mounted ? detectSpeechCapabilities() : null),
+ [mounted]
+ )
+
+ useEffect(() => {
+ let alive = true
+ speechGetSettings().then(
+ (view) => {
+ if (!alive) return
+ setCloud(view.settings)
+ setApiKeySet(view.apiKeySet)
+ setLoading(false)
+ },
+ (err) => {
+ if (!alive) return
+ setLoadError(toErrorMessage(err))
+ setLoading(false)
+ }
+ )
+ return () => {
+ alive = false
+ }
+ }, [])
+
+ const input = prefs.input
+ const updateInput = useCallback(
+ (patch: Partial) => {
+ saveSpeechPrefs({ input: { ...input, ...patch } })
+ },
+ [input]
+ )
+
+ const output = prefs.output
+ const updateOutput = useCallback(
+ (patch: Partial) => {
+ saveSpeechPrefs({ output: { ...output, ...patch } })
+ },
+ [output]
+ )
+
+ const outputOn = output.enabled
+ useEffect(() => {
+ if (!outputOn) return
+ let alive = true
+ void waitForVoices().then((list) => {
+ if (alive) setVoices(list)
+ })
+ return () => {
+ alive = false
+ }
+ }, [outputOn])
+
+ const speechLanguage = resolveSpeechLanguage(input, locale)
+ const sortedVoices = useMemo(() => {
+ if (!voices) return []
+ const base = speechLanguage.toLowerCase().split("-")[0]
+ const matches = (voice: SpeechSynthesisVoice) =>
+ voice.lang.toLowerCase().split("-")[0] === base
+ return [
+ ...voices.filter(matches),
+ ...voices.filter((voice) => !matches(voice)),
+ ]
+ }, [speechLanguage, voices])
+
+ const outputStatus = useMemo(() => {
+ if (voices === null) return null
+ const resolution = resolveOutputEngine(
+ output,
+ { browserTts: voices.length > 0 },
+ apiKeySet
+ )
+ if (resolution.engine === null) {
+ return t(OUTPUT_REASON_KEYS[resolution.reason])
+ }
+ return t("engineUsing", {
+ engine: t(
+ resolution.engine === "browser" ? "engineBrowser" : "engineCloud"
+ ),
+ })
+ }, [apiKeySet, output, t, voices])
+
+ const engineStatus = useMemo(() => {
+ if (!caps) return null
+ const resolution = resolveInputEngine(input, caps, apiKeySet)
+ if (resolution.engine === null) {
+ return t(REASON_KEYS[resolution.reason])
+ }
+ return t("engineUsing", {
+ engine: t(
+ resolution.engine === "browser" ? "engineBrowser" : "engineCloud"
+ ),
+ })
+ }, [apiKeySet, caps, input, t])
+
+ const languageSelection =
+ customLanguageMode ||
+ (input.language !== "" && !LANGUAGE_TAGS.includes(input.language))
+ ? LANGUAGE_CUSTOM
+ : input.language || LANGUAGE_FOLLOW_APP
+
+ const onLanguageSelect = useCallback(
+ (value: string) => {
+ if (value === LANGUAGE_CUSTOM) {
+ setCustomLanguageMode(true)
+ return
+ }
+ setCustomLanguageMode(false)
+ updateInput({ language: value === LANGUAGE_FOLLOW_APP ? "" : value })
+ },
+ [updateInput]
+ )
+
+ const persistCloud = useCallback(
+ async (apiKey: string | null, successMessage: string) => {
+ if (!cloud) return
+ setSaving(true)
+ try {
+ const view = await speechUpdateSettings(cloud, apiKey)
+ setCloud(view.settings)
+ setApiKeySet(view.apiKeySet)
+ setApiKeyDraft("")
+ toast.success(successMessage)
+ } catch (err) {
+ toast.error(t("saveFailed", { message: toErrorMessage(err) }))
+ } finally {
+ setSaving(false)
+ }
+ },
+ [cloud, t]
+ )
+
+ if (loading) {
+ return (
+
+
+ {t("loading")}
+
+ )
+ }
+
+ return (
+
+
+
+ {t("sectionTitle")}
+
+ {t("sectionDescription")}
+
+
+
+
updateInput({ enabled })}
+ />
+ }
+ >
+ {input.enabled && (
+
+
+ {engineStatus}
+
+ ) : undefined
+ }
+ htmlFor="speech-input-engine"
+ control={
+
+ }
+ />
+
+
+
+
+
+
+ {t("languageFollowApp")}
+
+ {LANGUAGE_TAGS.map((tag) => (
+
+ {languageName(tag, locale)}
+
+ ))}
+
+ {t("languageCustom")}
+
+
+
+ }
+ >
+ {languageSelection === LANGUAGE_CUSTOM && (
+
+ updateInput({ language: e.target.value.trim() })
+ }
+ spellCheck={false}
+ />
+ )}
+
+
+ )}
+
+
+
+ updateOutput(
+ enabled ? { enabled } : { enabled, autoRead: false }
+ )
+ }
+ />
+ }
+ >
+ {output.enabled && (
+
+
+ {outputStatus}
+
+ ) : undefined
+ }
+ htmlFor="speech-output-engine"
+ control={
+
+ }
+ />
+ {output.engine !== "cloud" && sortedVoices.length > 0 && (
+
+ updateOutput({
+ browserVoiceUri: uri === VOICE_DEFAULT ? "" : uri,
+ })
+ }
+ >
+
+
+
+
+
+ {t("voiceDefault")}
+
+ {sortedVoices.map((voice) => (
+
+ {voice.name} ({voice.lang})
+
+ ))}
+
+
+ }
+ />
+ )}
+
+ {output.rate.toFixed(2)}x
+
+ }
+ >
+ updateOutput({ rate })}
+ />
+
+ updateOutput({ autoRead })}
+ />
+ }
+ />
+
+ )}
+
+
+
+ {loadError && (
+
+ {t("loadFailed", { message: loadError })}
+
+ )}
+ {cloud && (
+
+
+
+ setCloud({ ...cloud, baseUrl: e.target.value })
+ }
+ spellCheck={false}
+ />
+
+ void persistCloud("", t("keyRemoved"))}
+ >
+ {t("removeKey")}
+
+ ) : undefined
+ }
+ >
+ setApiKeyDraft(e.target.value)}
+ placeholder={
+ apiKeySet ? t("apiKeySaved") : t("apiKeyPlaceholder")
+ }
+ autoComplete="new-password"
+ />
+
+
+
+ setCloud({ ...cloud, sttModel: e.target.value })
+ }
+ spellCheck={false}
+ />
+
+
+
+ setCloud({ ...cloud, ttsModel: e.target.value })
+ }
+ spellCheck={false}
+ />
+
+
+
+ setCloud({ ...cloud, ttsVoice: e.target.value })
+ }
+ spellCheck={false}
+ />
+
+
+ void persistCloud(
+ apiKeyDraft ? apiKeyDraft : null,
+ t("saved")
+ )
+ }
+ saving={saving}
+ label={t("save")}
+ savingLabel={t("saving")}
+ />
+
+ )}
+
+
raw
\n\n---\n\n[^1]: note"
+ expect(toSpeakableText(md, labels)).toBe("A bold and gone word.")
+ })
+})
+
+describe("chunkSpeakableText", () => {
+ it("packs whole sentences up to maxLen", () => {
+ expect(chunkSpeakableText("One. Two. Three.", 10)).toEqual([
+ "One. Two.",
+ " Three.",
+ ])
+ })
+
+ it("splits CJK sentences on full-width punctuation and hard-splits long runs", () => {
+ const text = "你好。今天天气很好！" + "长".repeat(12)
+ const chunks = chunkSpeakableText(text, 5)
+ expect(chunks.every((c) => c.length <= 5)).toBe(true)
+ expect(chunks.join("")).toBe(text)
+ expect(chunks[0]).toBe("你好。")
+ })
+
+ it("hard-splits a long Latin sentence at the last space", () => {
+ const chunks = chunkSpeakableText("alpha beta gamma delta", 12)
+ expect(chunks).toEqual(["alpha beta ", "gamma delta"])
+ })
+
+ it("keeps every chunk within maxLen and loses nothing on a 10 000-char input", () => {
+ const sentence = "The quick brown fox jumps over the lazy dog. "
+ const text = sentence.repeat(Math.ceil(10_000 / sentence.length)).trim()
+ const chunks = chunkSpeakableText(text, 220)
+ expect(text.length).toBeGreaterThanOrEqual(10_000)
+ expect(chunks.every((c) => c.length <= 220)).toBe(true)
+ expect(chunks.join("")).toBe(text)
+ })
+})
diff --git a/src/lib/speakable-text.ts b/src/lib/speakable-text.ts
new file mode 100644
index 0000000000..016840ca5f
--- /dev/null
+++ b/src/lib/speakable-text.ts
@@ -0,0 +1,132 @@
+import type { Nodes, Root } from "mdast"
+import remarkGfm from "remark-gfm"
+import remarkParse from "remark-parse"
+import { unified } from "unified"
+
+export interface SpeakableLabels {
+ codeOmitted: string
+ tableOmitted: string
+}
+
+const BARE_URL = /https?:\/\/\S+/g
+const SENTENCE_END = /[.!?:;。！？：；]$/
+const CJK = /[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\uf900-\ufaff]/
+
+const parser = unified().use(remarkParse).use(remarkGfm)
+
+function collapse(text: string): string {
+ return text.replace(/\s+/g, " ").trim()
+}
+
+function asSentence(text: string): string {
+ const clean = collapse(text)
+ if (!clean) return ""
+ return SENTENCE_END.test(clean) ? clean : `${clean}.`
+}
+
+function inlineText(node: Nodes, labels: SpeakableLabels): string {
+ switch (node.type) {
+ case "text":
+ return node.value.replace(BARE_URL, "")
+ case "inlineCode":
+ return node.value
+ case "break":
+ return " "
+ case "image":
+ case "imageReference":
+ case "html":
+ case "footnoteReference":
+ return ""
+ default:
+ if ("children" in node) {
+ return (node.children as Nodes[])
+ .map((child) => inlineText(child, labels))
+ .join("")
+ }
+ return ""
+ }
+}
+
+function blockSentences(node: Nodes, labels: SpeakableLabels): string[] {
+ switch (node.type) {
+ case "code":
+ return [asSentence(labels.codeOmitted)]
+ case "table":
+ return [asSentence(labels.tableOmitted)]
+ case "html":
+ case "thematicBreak":
+ case "footnoteDefinition":
+ case "definition":
+ case "yaml":
+ return []
+ case "heading":
+ case "paragraph":
+ return [asSentence(inlineText(node, labels))]
+ case "root":
+ case "blockquote":
+ case "list":
+ case "listItem":
+ return node.children.flatMap((child) =>
+ blockSentences(child as Nodes, labels)
+ )
+ default:
+ return [asSentence(inlineText(node, labels))]
+ }
+}
+
+export function toSpeakableText(
+ markdown: string,
+ labels: SpeakableLabels
+): string {
+ const tree = parser.parse(markdown) as Root
+ return blockSentences(tree, labels).filter(Boolean).join(" ")
+}
+
+function splitSentences(text: string): string[] {
+ const sentences: string[] = []
+ let current = ""
+ for (const char of text) {
+ current += char
+ if (/[.!?。！？\n]/.test(char)) {
+ sentences.push(current)
+ current = ""
+ }
+ }
+ if (current) sentences.push(current)
+ return sentences
+}
+
+function hardSplit(sentence: string, maxLen: number): string[] {
+ const pieces: string[] = []
+ let rest = sentence
+ while (rest.length > maxLen) {
+ const window = rest.slice(0, maxLen)
+ const space = window.lastIndexOf(" ")
+ const cut =
+ space > 0 && !CJK.test(window.charAt(maxLen - 1)) ? space + 1 : maxLen
+ pieces.push(rest.slice(0, cut))
+ rest = rest.slice(cut)
+ }
+ if (rest) pieces.push(rest)
+ return pieces
+}
+
+/**
+ * Packs whole sentences greedily into chunks of at most `maxLen` characters.
+ * Chunks keep their original spacing, so `chunks.join("")` is the input.
+ */
+export function chunkSpeakableText(text: string, maxLen: number): string[] {
+ const chunks: string[] = []
+ let current = ""
+ for (const sentence of splitSentences(text)) {
+ for (const piece of hardSplit(sentence, maxLen)) {
+ if (current.length + piece.length > maxLen && current) {
+ chunks.push(current)
+ current = ""
+ }
+ current += piece
+ }
+ }
+ if (current) chunks.push(current)
+ return chunks.filter((chunk) => chunk.trim().length > 0)
+}
diff --git a/src/lib/speech-capabilities.test.ts b/src/lib/speech-capabilities.test.ts
new file mode 100644
index 0000000000..9f59ed6c82
--- /dev/null
+++ b/src/lib/speech-capabilities.test.ts
@@ -0,0 +1,293 @@
+import { describe, expect, it, vi } from "vitest"
+
+import {
+ detectSpeechCapabilities,
+ hasBrowserTts,
+ resolveInputEngine,
+ resolveOutputEngine,
+ waitForVoices,
+ resolveSpeechLanguage,
+ type SpeechCapabilities,
+} from "./speech-capabilities"
+
+describe("speech capabilities detection", () => {
+ it("detects browser STT and media capture in a standard browser", () => {
+ const caps = detectSpeechCapabilities({
+ isDesktop: false,
+ platform: "macos",
+ hasSpeechRecognition: true,
+ hasMediaDevices: true,
+ isSecureContext: true,
+ })
+ expect(caps).toEqual({
+ browserStt: true,
+ mediaCapture: true,
+ secureContext: true,
+ })
+ })
+
+ it("supports webkitSpeechRecognition prefix", () => {
+ const caps = detectSpeechCapabilities({
+ isDesktop: false,
+ platform: "macos",
+ webkitSpeechRecognition: function () {},
+ hasMediaDevices: true,
+ isSecureContext: true,
+ })
+ expect(caps.browserStt).toBe(true)
+ })
+
+ it("permits browser STT on desktop macOS", () => {
+ const caps = detectSpeechCapabilities({
+ isDesktop: true,
+ platform: "macos",
+ hasSpeechRecognition: true,
+ hasMediaDevices: true,
+ isSecureContext: true,
+ })
+ expect(caps.browserStt).toBe(true)
+ })
+
+ it("excludes browser STT on desktop Windows due to WebView2 limitations", () => {
+ const caps = detectSpeechCapabilities({
+ isDesktop: true,
+ platform: "windows",
+ hasSpeechRecognition: true,
+ hasMediaDevices: true,
+ isSecureContext: true,
+ })
+ expect(caps.browserStt).toBe(false)
+ })
+
+ it("excludes browser STT on desktop Linux due to WebKitGTK limitations", () => {
+ const caps = detectSpeechCapabilities({
+ isDesktop: true,
+ platform: "linux",
+ hasSpeechRecognition: true,
+ hasMediaDevices: true,
+ isSecureContext: true,
+ })
+ expect(caps.browserStt).toBe(false)
+ })
+
+ it("disables mediaCapture when not in a secure context", () => {
+ const caps = detectSpeechCapabilities({
+ isDesktop: false,
+ platform: "macos",
+ hasSpeechRecognition: true,
+ hasMediaDevices: true,
+ isSecureContext: false,
+ })
+ expect(caps.browserStt).toBe(false)
+ expect(caps.mediaCapture).toBe(false)
+ expect(caps.secureContext).toBe(false)
+ })
+
+ it("disables mediaCapture when getUserMedia is missing", () => {
+ const caps = detectSpeechCapabilities({
+ isDesktop: false,
+ platform: "macos",
+ hasSpeechRecognition: true,
+ hasMediaDevices: false,
+ isSecureContext: true,
+ })
+ expect(caps.mediaCapture).toBe(false)
+ expect(caps.secureContext).toBe(true)
+ })
+})
+
+describe("resolveInputEngine decision table", () => {
+ const fullCaps: SpeechCapabilities = {
+ browserStt: true,
+ mediaCapture: true,
+ secureContext: true,
+ }
+
+ const noBrowserCaps: SpeechCapabilities = {
+ browserStt: false,
+ mediaCapture: true,
+ secureContext: true,
+ }
+
+ const noMicCaps: SpeechCapabilities = {
+ browserStt: false,
+ mediaCapture: false,
+ secureContext: true,
+ }
+
+ const insecureCaps: SpeechCapabilities = {
+ browserStt: false,
+ mediaCapture: false,
+ secureContext: false,
+ }
+
+ it("resolves explicit browser preference when available", () => {
+ expect(resolveInputEngine({ engine: "browser" }, fullCaps, false)).toEqual({
+ engine: "browser",
+ })
+ })
+
+ it("returns no-engine when explicit browser preference is unavailable", () => {
+ expect(
+ resolveInputEngine({ engine: "browser" }, noBrowserCaps, true)
+ ).toEqual({
+ engine: null,
+ reason: "no-engine",
+ })
+ })
+
+ it("resolves explicit cloud preference when media capture and cloud config exist", () => {
+ expect(
+ resolveInputEngine({ engine: "cloud" }, noBrowserCaps, true)
+ ).toEqual({
+ engine: "cloud",
+ })
+ })
+
+ it("returns insecure-context for explicit cloud preference when insecure", () => {
+ expect(resolveInputEngine({ engine: "cloud" }, insecureCaps, true)).toEqual(
+ {
+ engine: null,
+ reason: "insecure-context",
+ }
+ )
+ })
+
+ it("returns no-mic for explicit cloud preference when mic is missing", () => {
+ expect(resolveInputEngine({ engine: "cloud" }, noMicCaps, true)).toEqual({
+ engine: null,
+ reason: "no-mic",
+ })
+ })
+
+ it("returns cloud-not-configured for explicit cloud preference when unconfigured", () => {
+ expect(
+ resolveInputEngine({ engine: "cloud" }, noBrowserCaps, false)
+ ).toEqual({
+ engine: null,
+ reason: "cloud-not-configured",
+ })
+ })
+
+ it("resolves auto preference to browser when browser STT is available", () => {
+ expect(resolveInputEngine({ engine: "auto" }, fullCaps, false)).toEqual({
+ engine: "browser",
+ })
+ })
+
+ it("resolves auto preference to cloud when browser is unavailable but cloud is ready", () => {
+ expect(resolveInputEngine({ engine: "auto" }, noBrowserCaps, true)).toEqual(
+ {
+ engine: "cloud",
+ }
+ )
+ })
+
+ it("prioritizes insecure-context for auto fallback", () => {
+ expect(resolveInputEngine({ engine: "auto" }, insecureCaps, false)).toEqual(
+ {
+ engine: null,
+ reason: "insecure-context",
+ }
+ )
+ })
+
+ it("prioritizes cloud-not-configured over no-engine for auto fallback", () => {
+ expect(
+ resolveInputEngine({ engine: "auto" }, noBrowserCaps, false)
+ ).toEqual({
+ engine: null,
+ reason: "cloud-not-configured",
+ })
+ })
+
+ it("returns no-mic for auto fallback when cloud is configured but mic missing", () => {
+ expect(resolveInputEngine({ engine: "auto" }, noMicCaps, true)).toEqual({
+ engine: null,
+ reason: "no-mic",
+ })
+ })
+})
+
+describe("resolveSpeechLanguage mapping", () => {
+ it("uses custom preference language when non-empty", () => {
+ expect(resolveSpeechLanguage({ language: "fr-CA" }, "en")).toBe("fr-CA")
+ expect(resolveSpeechLanguage("de-AT", "zh-CN")).toBe("de-AT")
+ })
+
+ it("maps next-intl UI locales to BCP-47 tags when preference language is empty", () => {
+ expect(resolveSpeechLanguage({ language: "" }, "en")).toBe("en-US")
+ expect(resolveSpeechLanguage("", "zh-CN")).toBe("zh-CN")
+ expect(resolveSpeechLanguage("", "zh-TW")).toBe("zh-TW")
+ expect(resolveSpeechLanguage("", "ja")).toBe("ja-JP")
+ expect(resolveSpeechLanguage("", "ko")).toBe("ko-KR")
+ expect(resolveSpeechLanguage("", "es")).toBe("es-ES")
+ expect(resolveSpeechLanguage("", "de")).toBe("de-DE")
+ expect(resolveSpeechLanguage("", "fr")).toBe("fr-FR")
+ expect(resolveSpeechLanguage("", "pt")).toBe("pt-BR")
+ expect(resolveSpeechLanguage("", "ar")).toBe("ar-SA")
+ })
+
+ it("falls back to raw non-empty locale or en-US for unmapped or empty locales", () => {
+ expect(resolveSpeechLanguage("", "it")).toBe("it")
+ expect(resolveSpeechLanguage("", "")).toBe("en-US")
+ })
+})
+
+describe("read-aloud capabilities", () => {
+ const voice = { voiceURI: "v", lang: "en-US" } as SpeechSynthesisVoice
+
+ it("reports browser TTS only when voices exist", () => {
+ expect(hasBrowserTts(undefined)).toBe(false)
+ expect(hasBrowserTts({ getVoices: () => [] })).toBe(false)
+ expect(hasBrowserTts({ getVoices: () => [voice] })).toBe(true)
+ })
+
+ it("waitForVoices resolves on voiceschanged", async () => {
+ let voices: SpeechSynthesisVoice[] = []
+ const target = new EventTarget()
+ const synth = {
+ getVoices: () => voices,
+ addEventListener: target.addEventListener.bind(target),
+ removeEventListener: target.removeEventListener.bind(target),
+ } as unknown as SpeechSynthesis
+ const pending = waitForVoices(synth, 60_000)
+ voices = [voice]
+ target.dispatchEvent(new Event("voiceschanged"))
+ await expect(pending).resolves.toEqual([voice])
+ })
+
+ it("waitForVoices resolves empty after the timeout", async () => {
+ vi.useFakeTimers()
+ try {
+ const target = new EventTarget()
+ const synth = {
+ getVoices: () => [],
+ addEventListener: target.addEventListener.bind(target),
+ removeEventListener: target.removeEventListener.bind(target),
+ } as unknown as SpeechSynthesis
+ const pending = waitForVoices(synth, 1500)
+ vi.advanceTimersByTime(1500)
+ await expect(pending).resolves.toEqual([])
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it.each([
+ ["browser", true, false, { engine: "browser" }],
+ ["browser", false, true, { engine: null, reason: "no-engine" }],
+ ["cloud", true, true, { engine: "cloud" }],
+ ["cloud", true, false, { engine: null, reason: "cloud-not-configured" }],
+ ["auto", true, true, { engine: "browser" }],
+ ["auto", false, true, { engine: "cloud" }],
+ ["auto", false, false, { engine: null, reason: "cloud-not-configured" }],
+ ] as const)(
+ "resolveOutputEngine(%s, tts=%s, cloud=%s)",
+ (engine, browserTts, cloudConfigured, expected) => {
+ expect(
+ resolveOutputEngine({ engine }, { browserTts }, cloudConfigured)
+ ).toEqual(expected)
+ }
+ )
+})
diff --git a/src/lib/speech-capabilities.ts b/src/lib/speech-capabilities.ts
new file mode 100644
index 0000000000..18278c7232
--- /dev/null
+++ b/src/lib/speech-capabilities.ts
@@ -0,0 +1,220 @@
+import { detectPlatform } from "@/hooks/use-platform"
+import { isDesktop as isDesktopRuntime } from "./platform"
+import type { SpeechInputPrefs } from "./speech-prefs"
+
+export interface SpeechCapabilities {
+ browserStt: boolean
+ mediaCapture: boolean
+ secureContext: boolean
+}
+
+export type InputEngineResolution =
+ | { engine: "browser" | "cloud" }
+ | {
+ engine: null
+ reason:
+ | "no-mic"
+ | "insecure-context"
+ | "no-engine"
+ | "cloud-not-configured"
+ }
+
+export interface SpeechCapabilitiesEnv {
+ isDesktop?: boolean
+ platform?: "macos" | "windows" | "linux" | "unknown"
+ SpeechRecognition?: unknown
+ webkitSpeechRecognition?: unknown
+ hasSpeechRecognition?: boolean
+ hasMediaDevices?: boolean
+ getUserMedia?: unknown
+ isSecureContext?: boolean
+}
+
+export const LOCALE_TO_BCP47: Record = {
+ en: "en-US",
+ "zh-CN": "zh-CN",
+ "zh-TW": "zh-TW",
+ ja: "ja-JP",
+ ko: "ko-KR",
+ es: "es-ES",
+ de: "de-DE",
+ fr: "fr-FR",
+ pt: "pt-BR",
+ ar: "ar-SA",
+}
+
+export function detectSpeechCapabilities(
+ env?: SpeechCapabilitiesEnv
+): SpeechCapabilities {
+ const desktop =
+ env?.isDesktop ??
+ (typeof window !== "undefined" ? isDesktopRuntime() : false)
+ const plat =
+ env?.platform ??
+ (typeof window !== "undefined" ? detectPlatform() : "unknown")
+
+ let hasRecognizer = false
+ if (typeof env?.hasSpeechRecognition === "boolean") {
+ hasRecognizer = env.hasSpeechRecognition
+ } else if (
+ env?.SpeechRecognition !== undefined ||
+ env?.webkitSpeechRecognition !== undefined
+ ) {
+ hasRecognizer = Boolean(
+ env.SpeechRecognition || env.webkitSpeechRecognition
+ )
+ } else if (typeof window !== "undefined") {
+ const win = window as unknown as Record
+ hasRecognizer = Boolean(
+ win.SpeechRecognition || win.webkitSpeechRecognition
+ )
+ }
+
+ const isUnsupportedDesktop =
+ desktop && (plat === "windows" || plat === "linux")
+ const secureContext =
+ typeof env?.isSecureContext === "boolean"
+ ? env.isSecureContext
+ : typeof window !== "undefined"
+ ? Boolean(window.isSecureContext)
+ : false
+
+ // Chromium exposes SpeechRecognition on insecure origins too, but it cannot
+ // open the microphone there and fails at once with "audio-capture".
+ const browserStt = hasRecognizer && !isUnsupportedDesktop && secureContext
+
+ let hasGetUserMedia = false
+ if (typeof env?.hasMediaDevices === "boolean") {
+ hasGetUserMedia = env.hasMediaDevices
+ } else if (env?.getUserMedia !== undefined) {
+ hasGetUserMedia = Boolean(env.getUserMedia)
+ } else if (typeof navigator !== "undefined") {
+ hasGetUserMedia = Boolean(navigator.mediaDevices?.getUserMedia)
+ }
+
+ const mediaCapture = hasGetUserMedia && secureContext
+
+ return {
+ browserStt,
+ mediaCapture,
+ secureContext,
+ }
+}
+
+export function resolveInputEngine(
+ pref: SpeechInputPrefs | { engine: "auto" | "browser" | "cloud" },
+ caps: SpeechCapabilities,
+ cloudConfigured: boolean
+): InputEngineResolution {
+ if (pref.engine === "browser") {
+ if (caps.browserStt) {
+ return { engine: "browser" }
+ }
+ return { engine: null, reason: "no-engine" }
+ }
+
+ if (pref.engine === "cloud") {
+ if (caps.mediaCapture && cloudConfigured) {
+ return { engine: "cloud" }
+ }
+ if (!caps.secureContext) {
+ return { engine: null, reason: "insecure-context" }
+ }
+ if (!caps.mediaCapture) {
+ return { engine: null, reason: "no-mic" }
+ }
+ return { engine: null, reason: "cloud-not-configured" }
+ }
+
+ if (caps.browserStt) {
+ return { engine: "browser" }
+ }
+ if (caps.mediaCapture && cloudConfigured) {
+ return { engine: "cloud" }
+ }
+
+ if (!caps.secureContext) {
+ return { engine: null, reason: "insecure-context" }
+ }
+ if (!cloudConfigured) {
+ return { engine: null, reason: "cloud-not-configured" }
+ }
+ if (!caps.mediaCapture) {
+ return { engine: null, reason: "no-mic" }
+ }
+ return { engine: null, reason: "no-engine" }
+}
+
+export type OutputEngineResolution =
+ | { engine: "browser" | "cloud" }
+ | { engine: null; reason: "no-engine" | "cloud-not-configured" }
+
+type VoiceSource = Pick &
+ Partial>
+
+function currentSynth(): VoiceSource | undefined {
+ return typeof window !== "undefined" ? window.speechSynthesis : undefined
+}
+
+export function hasBrowserTts(synth: VoiceSource | undefined = currentSynth()) {
+ return Boolean(synth) && synth!.getVoices().length > 0
+}
+
+/** Chromium fills `getVoices()` asynchronously; resolves once voices exist or the timeout passes. */
+export function waitForVoices(
+ synth: VoiceSource | undefined = currentSynth(),
+ timeoutMs = 1500
+): Promise {
+ if (!synth) return Promise.resolve([])
+ const voices = synth.getVoices()
+ if (voices.length > 0 || !synth.addEventListener) {
+ return Promise.resolve(voices)
+ }
+ return new Promise((resolve) => {
+ const finish = () => {
+ clearTimeout(timer)
+ synth.removeEventListener?.("voiceschanged", finish)
+ resolve(synth.getVoices())
+ }
+ const timer = setTimeout(finish, timeoutMs)
+ synth.addEventListener!("voiceschanged", finish)
+ })
+}
+
+export function resolveOutputEngine(
+ pref: { engine: "auto" | "browser" | "cloud" },
+ caps: { browserTts: boolean },
+ cloudConfigured: boolean
+): OutputEngineResolution {
+ if (pref.engine === "browser") {
+ return caps.browserTts
+ ? { engine: "browser" }
+ : { engine: null, reason: "no-engine" }
+ }
+ if (pref.engine === "cloud") {
+ return cloudConfigured
+ ? { engine: "cloud" }
+ : { engine: null, reason: "cloud-not-configured" }
+ }
+ if (caps.browserTts) return { engine: "browser" }
+ if (cloudConfigured) return { engine: "cloud" }
+ return { engine: null, reason: "cloud-not-configured" }
+}
+
+export function resolveSpeechLanguage(
+ pref: SpeechInputPrefs | { language?: string } | string,
+ uiLocale: string
+): string {
+ const language =
+ typeof pref === "string" ? pref.trim() : (pref.language ?? "").trim()
+
+ if (language.length > 0) {
+ return language
+ }
+
+ if (LOCALE_TO_BCP47[uiLocale]) {
+ return LOCALE_TO_BCP47[uiLocale]
+ }
+
+ return uiLocale.trim().length > 0 ? uiLocale.trim() : "en-US"
+}
diff --git a/src/lib/speech-player.test.ts b/src/lib/speech-player.test.ts
new file mode 100644
index 0000000000..6ec2ed46c0
--- /dev/null
+++ b/src/lib/speech-player.test.ts
@@ -0,0 +1,357 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"
+
+const synthesize = vi.fn()
+const getSettings = vi.fn()
+vi.mock("@/lib/api", () => ({
+ speechSynthesize: (...args: unknown[]) => synthesize(...args),
+ speechGetSettings: () => getSettings(),
+}))
+
+import {
+ getSpeechPlayerState,
+ maybeAutoRead,
+ resetSpeechPlayerForTests,
+ speak,
+ stopSpeech,
+ subscribeSpeechPlayer,
+} from "./speech-player"
+import type { SpeakOptions } from "./speech-player"
+import {
+ DEFAULT_SPEECH_PREFS,
+ resetSpeechPrefsCacheForTests,
+ saveSpeechPrefs,
+} from "./speech-prefs"
+
+const labels = { codeOmitted: "Code omitted", tableOmitted: "Table omitted" }
+
+class FakeUtterance {
+ text: string
+ lang = ""
+ rate = 1
+ voice: unknown = null
+ onstart: (() => void) | null = null
+ onend: (() => void) | null = null
+ onerror: ((event: { error: string }) => void) | null = null
+ constructor(text: string) {
+ this.text = text
+ }
+}
+
+class FakeSynth {
+ queue: FakeUtterance[] = []
+ voices = [{ voiceURI: "v-en", lang: "en-US" }]
+ speak = vi.fn((u: FakeUtterance): void => {
+ this.queue.push(u)
+ })
+ cancel = vi.fn(() => {
+ this.queue = []
+ })
+ getVoices = () => this.voices
+}
+
+class FakeAudio {
+ static instances: FakeAudio[] = []
+ src = ""
+ paused = true
+ onended: (() => void) | null = null
+ onerror: (() => void) | null = null
+ played: string[] = []
+ constructor() {
+ FakeAudio.instances.push(this)
+ }
+ play = vi.fn(() => {
+ this.paused = false
+ this.played.push(this.src)
+ return Promise.resolve()
+ })
+ pause = vi.fn(() => {
+ this.paused = true
+ })
+ removeAttribute = vi.fn((name: string) => {
+ if (name === "src") this.src = ""
+ })
+ load = vi.fn()
+ finish() {
+ this.paused = true
+ this.onended?.()
+ }
+}
+
+function deferred() {
+ let resolve!: (value: T) => void
+ let reject!: (error: unknown) => void
+ const promise = new Promise((res, rej) => {
+ resolve = res
+ reject = rej
+ })
+ return { promise, resolve, reject }
+}
+
+function waitForState(predicate: () => boolean): Promise {
+ if (predicate()) return Promise.resolve()
+ return new Promise((resolve, reject) => {
+ const timer = setTimeout(() => {
+ unsubscribe()
+ reject(
+ new Error(
+ `state never matched: ${JSON.stringify(getSpeechPlayerState())}`
+ )
+ )
+ }, 1000)
+ const unsubscribe = subscribeSpeechPlayer(() => {
+ if (!predicate()) return
+ clearTimeout(timer)
+ unsubscribe()
+ resolve()
+ })
+ })
+}
+
+const audioPayload = { audioBase64: btoa("mp3"), mimeType: "audio/mpeg" }
+
+let synth: FakeSynth
+let urlCount = 0
+const createObjectURL = vi.fn(() => `blob:${++urlCount}`)
+const revokeObjectURL = vi.fn()
+
+const browser: SpeakOptions = { engine: "browser", language: "en-US", labels }
+const cloud: SpeakOptions = { engine: "cloud", language: "en-US", labels }
+
+beforeEach(() => {
+ localStorage.clear()
+ resetSpeechPrefsCacheForTests()
+ synth = new FakeSynth()
+ FakeAudio.instances = []
+ urlCount = 0
+ synthesize.mockReset()
+ getSettings.mockReset()
+ createObjectURL.mockClear()
+ revokeObjectURL.mockClear()
+ vi.stubGlobal("speechSynthesis", synth)
+ vi.stubGlobal("SpeechSynthesisUtterance", FakeUtterance)
+ vi.stubGlobal("Audio", FakeAudio)
+ vi.stubGlobal("URL", Object.assign(URL, { createObjectURL, revokeObjectURL }))
+ resetSpeechPlayerForTests()
+})
+
+afterEach(() => {
+ resetSpeechPlayerForTests()
+ vi.unstubAllGlobals()
+})
+
+describe("browser engine", () => {
+ it("queues one utterance per chunk and tracks state until the last ends", () => {
+ const long = "First sentence here. ".repeat(20)
+ speak("turn-1", long, browser)
+ expect(synth.queue.length).toBeGreaterThan(1)
+ expect(synth.queue.every((u) => u.text.length <= 220)).toBe(true)
+ expect(synth.queue[0].lang).toBe("en-US")
+ expect(synth.queue[0].voice).toEqual(synth.voices[0])
+ expect(getSpeechPlayerState()).toEqual({
+ playingId: "turn-1",
+ status: "loading",
+ })
+
+ synth.queue[0].onstart?.()
+ expect(getSpeechPlayerState().status).toBe("playing")
+ synth.queue[0].onend?.()
+ expect(getSpeechPlayerState().status).toBe("playing")
+ synth.queue[synth.queue.length - 1].onend?.()
+ expect(getSpeechPlayerState()).toEqual({ playingId: null, status: "idle" })
+ })
+
+ it("stops and reports an error, but ignores interruptions", () => {
+ const onError = vi.fn()
+ speak("turn-1", "Hello there.", { ...browser, onError })
+ synth.queue[0].onerror?.({ error: "interrupted" })
+ expect(onError).not.toHaveBeenCalled()
+ synth.queue[0].onerror?.({ error: "synthesis-failed" })
+ expect(onError).toHaveBeenCalledWith("failed")
+ expect(getSpeechPlayerState().status).toBe("idle")
+ })
+
+ it("uses the saved rate and voice", () => {
+ synth.voices.push({ voiceURI: "v-other", lang: "en-GB" })
+ saveSpeechPrefs({
+ output: {
+ ...DEFAULT_SPEECH_PREFS.output,
+ rate: 1.5,
+ browserVoiceUri: "v-other",
+ },
+ })
+ speak("turn-1", "Hello.", browser)
+ expect(synth.queue[0].rate).toBe(1.5)
+ expect(synth.queue[0].voice).toEqual({ voiceURI: "v-other", lang: "en-GB" })
+ })
+
+ it("speak while playing stops the previous playback", () => {
+ speak("turn-1", "One.", browser)
+ speak("turn-2", "Two.", browser)
+ expect(synth.cancel).toHaveBeenCalled()
+ expect(synth.queue.map((u) => u.text)).toEqual(["Two."])
+ expect(getSpeechPlayerState().playingId).toBe("turn-2")
+ synth.queue[0].onend?.()
+ expect(getSpeechPlayerState().status).toBe("idle")
+ })
+})
+
+describe("cloud engine", () => {
+ it("plays chunks in order, prefetches the next and revokes URLs", async () => {
+ const first = deferred()
+ const second = deferred()
+ synthesize
+ .mockReturnValueOnce(first.promise)
+ .mockReturnValueOnce(second.promise)
+ const text = `${"a".repeat(3000)}. ${"b".repeat(3000)}.`
+
+ speak("turn-1", text, cloud)
+ expect(synthesize).toHaveBeenCalledTimes(1)
+ first.resolve(audioPayload)
+ await waitForState(() => getSpeechPlayerState().status === "playing")
+
+ const audio = FakeAudio.instances[0]
+ expect(audio.played).toEqual(["blob:1"])
+ expect(synthesize).toHaveBeenCalledTimes(2)
+ expect(synthesize.mock.calls[1][1]).toBe(1)
+
+ second.resolve(audioPayload)
+ const secondPlay = new Promise((resolve) => {
+ audio.play.mockImplementationOnce(() => {
+ audio.played.push(audio.src)
+ resolve()
+ return Promise.resolve()
+ })
+ })
+ audio.finish()
+ await secondPlay
+ expect(audio.played).toEqual(["blob:1", "blob:2"])
+ expect(revokeObjectURL).toHaveBeenCalledWith("blob:1")
+
+ const idle = waitForState(() => getSpeechPlayerState().status === "idle")
+ audio.finish()
+ await idle
+ expect(revokeObjectURL).toHaveBeenCalledWith("blob:2")
+ expect(FakeAudio.instances).toHaveLength(1)
+ })
+
+ it("stopSpeech mid-fetch discards the late response", async () => {
+ const pending = deferred()
+ synthesize.mockReturnValueOnce(pending.promise)
+ speak("turn-1", "Hello.", cloud)
+ stopSpeech()
+ expect(getSpeechPlayerState().status).toBe("idle")
+ pending.resolve(audioPayload)
+ await pending.promise
+ await Promise.resolve()
+ expect(createObjectURL).not.toHaveBeenCalled()
+ expect(FakeAudio.instances[0].play).not.toHaveBeenCalled()
+ })
+
+ it("stopSpeech while playing pauses, clears src and revokes URLs", async () => {
+ synthesize.mockResolvedValue(audioPayload)
+ speak("turn-1", "Hello.", cloud)
+ await waitForState(() => getSpeechPlayerState().status === "playing")
+ const audio = FakeAudio.instances[0]
+ stopSpeech()
+ expect(audio.pause).toHaveBeenCalled()
+ expect(audio.removeAttribute).toHaveBeenCalledWith("src")
+ expect(audio.load).toHaveBeenCalled()
+ expect(revokeObjectURL).toHaveBeenCalledWith("blob:1")
+ expect(getSpeechPlayerState().status).toBe("idle")
+ })
+
+ it("maps an auth failure to the auth error", async () => {
+ synthesize.mockRejectedValue({
+ code: "authentication_failed",
+ message: "no",
+ })
+ const onError = vi.fn()
+ const failed = new Promise((resolve) =>
+ onError.mockImplementation(() => resolve())
+ )
+ speak("turn-1", "Hello.", { ...cloud, onError })
+ await failed
+ expect(onError).toHaveBeenCalledWith("auth")
+ expect(getSpeechPlayerState().status).toBe("idle")
+ })
+})
+
+describe("engine resolution", () => {
+ const auto: SpeakOptions = { language: "en-US", labels }
+
+ it("uses browser voices when the preference is auto", async () => {
+ const spoken = new Promise((resolve) =>
+ synth.speak.mockImplementationOnce((u: FakeUtterance) => {
+ synth.queue.push(u)
+ resolve()
+ })
+ )
+ speak("turn-1", "Hello.", auto)
+ await spoken
+ expect(synth.queue.map((q) => q.text)).toEqual(["Hello."])
+ expect(getSettings).not.toHaveBeenCalled()
+ })
+
+ it("falls back to cloud when no browser voice exists and a key is set", async () => {
+ synth.voices = []
+ vi.stubGlobal("speechSynthesis", undefined)
+ getSettings.mockResolvedValue({ apiKeySet: true })
+ synthesize.mockResolvedValue(audioPayload)
+ speak("turn-1", "Hello.", auto)
+ await waitForState(() => getSpeechPlayerState().status === "playing")
+ expect(synthesize).toHaveBeenCalledWith("Hello.", 1)
+ })
+
+ it("reports not-configured when neither engine is usable", async () => {
+ vi.stubGlobal("speechSynthesis", undefined)
+ getSettings.mockResolvedValue({ apiKeySet: false })
+ const onError = vi.fn()
+ const failed = new Promise((resolve) =>
+ onError.mockImplementation(() => resolve())
+ )
+ speak("turn-1", "Hello.", { ...auto, onError })
+ await failed
+ expect(onError).toHaveBeenCalledWith("not-configured")
+ expect(getSpeechPlayerState().status).toBe("idle")
+ })
+})
+
+describe("maybeAutoRead", () => {
+ const ctx = {
+ contextKey: "tab-1",
+ activeId: "tab-1",
+ visibility: "visible" as DocumentVisibilityState,
+ }
+
+ function enableAutoRead(autoRead = true) {
+ saveSpeechPrefs({
+ output: { ...DEFAULT_SPEECH_PREFS.output, enabled: true, autoRead },
+ })
+ }
+
+ it("does nothing when read aloud or auto-read is off", () => {
+ expect(maybeAutoRead(ctx, "Hi.", browser)).toBe(false)
+ enableAutoRead(false)
+ expect(maybeAutoRead(ctx, "Hi.", browser)).toBe(false)
+ expect(synth.speak).not.toHaveBeenCalled()
+ })
+
+ it("skips background tabs, hidden documents and empty text", () => {
+ enableAutoRead()
+ expect(maybeAutoRead({ ...ctx, activeId: "tab-2" }, "Hi.", browser)).toBe(
+ false
+ )
+ expect(
+ maybeAutoRead({ ...ctx, visibility: "hidden" }, "Hi.", browser)
+ ).toBe(false)
+ expect(maybeAutoRead(ctx, " \n ", browser)).toBe(false)
+ expect(synth.speak).not.toHaveBeenCalled()
+ })
+
+ it("speaks the active visible tab's reply", () => {
+ enableAutoRead()
+ expect(maybeAutoRead(ctx, "All done.", browser)).toBe(true)
+ expect(synth.queue.map((u) => u.text)).toEqual(["All done."])
+ expect(getSpeechPlayerState().playingId).toBe("auto:tab-1")
+ })
+})
diff --git a/src/lib/speech-player.ts b/src/lib/speech-player.ts
new file mode 100644
index 0000000000..4ec9f4801e
--- /dev/null
+++ b/src/lib/speech-player.ts
@@ -0,0 +1,301 @@
+"use client"
+
+import { useSyncExternalStore } from "react"
+
+import { speechGetSettings, speechSynthesize } from "@/lib/api"
+import { extractAppCommandError } from "@/lib/app-error"
+import { chunkSpeakableText, toSpeakableText } from "@/lib/speakable-text"
+import type { SpeakableLabels } from "@/lib/speakable-text"
+import {
+ resolveOutputEngine,
+ waitForVoices,
+ type OutputEngineResolution,
+} from "@/lib/speech-capabilities"
+import { getSpeechPrefs } from "@/lib/speech-prefs"
+
+export type SpeechPlayerStatus = "idle" | "loading" | "playing"
+
+export interface SpeechPlayerState {
+ playingId: string | null
+ status: SpeechPlayerStatus
+}
+
+export type SpeechPlaybackError =
+ | "auth"
+ | "not-configured"
+ | "unavailable"
+ | "failed"
+
+export interface SpeakOptions {
+ /** Omitted: resolved from the saved preference and what this device supports. */
+ engine?: "browser" | "cloud"
+ language: string
+ labels: SpeakableLabels
+ onError?: (error: SpeechPlaybackError) => void
+}
+
+const BROWSER_CHUNK = 220
+const CLOUD_CHUNK = 4000
+const IDLE: SpeechPlayerState = { playingId: null, status: "idle" }
+
+let state: SpeechPlayerState = IDLE
+let generation = 0
+let audio: HTMLAudioElement | null = null
+const objectUrls = new Set()
+const listeners = new Set<() => void>()
+
+function setState(next: SpeechPlayerState) {
+ if (next.playingId === state.playingId && next.status === state.status) {
+ return
+ }
+ state = next
+ for (const listener of listeners) listener()
+}
+
+export function getSpeechPlayerState(): SpeechPlayerState {
+ return state
+}
+
+export function subscribeSpeechPlayer(listener: () => void): () => void {
+ listeners.add(listener)
+ return () => {
+ listeners.delete(listener)
+ }
+}
+
+function getServerState(): SpeechPlayerState {
+ return IDLE
+}
+
+export function useSpeechPlayer(): SpeechPlayerState {
+ return useSyncExternalStore(
+ subscribeSpeechPlayer,
+ getSpeechPlayerState,
+ getServerState
+ )
+}
+
+function revokeAll() {
+ for (const url of objectUrls) URL.revokeObjectURL(url)
+ objectUrls.clear()
+}
+
+export function stopSpeech(): void {
+ generation += 1
+ if (typeof window !== "undefined" && window.speechSynthesis) {
+ window.speechSynthesis.cancel()
+ }
+ if (audio) {
+ audio.onended = null
+ audio.onerror = null
+ audio.pause()
+ audio.removeAttribute("src")
+ audio.load()
+ }
+ revokeAll()
+ setState(IDLE)
+}
+
+function classify(error: unknown): SpeechPlaybackError {
+ switch (extractAppCommandError(error)?.code) {
+ case "authentication_failed":
+ return "auth"
+ case "configuration_missing":
+ return "not-configured"
+ default:
+ return "failed"
+ }
+}
+
+function pickVoice(
+ voices: SpeechSynthesisVoice[],
+ uri: string,
+ language: string
+): SpeechSynthesisVoice | null {
+ if (uri) {
+ const chosen = voices.find((voice) => voice.voiceURI === uri)
+ if (chosen) return chosen
+ }
+ const lang = language.toLowerCase()
+ const base = lang.split("-")[0]
+ return (
+ voices.find((voice) => voice.lang.toLowerCase() === lang) ??
+ voices.find((voice) => voice.lang.toLowerCase().startsWith(base)) ??
+ null
+ )
+}
+
+function speakBrowser(
+ id: string,
+ chunks: string[],
+ options: SpeakOptions,
+ run: number
+) {
+ const synth = window.speechSynthesis
+ const { rate, browserVoiceUri } = getSpeechPrefs().output
+ const voice = pickVoice(synth.getVoices(), browserVoiceUri, options.language)
+ chunks.forEach((chunk, index) => {
+ const utterance = new SpeechSynthesisUtterance(chunk)
+ utterance.lang = options.language
+ utterance.rate = rate
+ if (voice) utterance.voice = voice
+ if (index === 0) {
+ utterance.onstart = () => {
+ if (run === generation) setState({ playingId: id, status: "playing" })
+ }
+ }
+ if (index === chunks.length - 1) {
+ utterance.onend = () => {
+ if (run === generation) setState(IDLE)
+ }
+ }
+ utterance.onerror = (event) => {
+ if (run !== generation) return
+ if (event.error === "interrupted" || event.error === "canceled") return
+ stopSpeech()
+ options.onError?.("failed")
+ }
+ synth.speak(utterance)
+ })
+}
+
+async function fetchChunk(
+ chunk: string,
+ rate: number,
+ run: number
+): Promise {
+ const { audioBase64, mimeType } = await speechSynthesize(chunk, rate)
+ if (run !== generation) return null
+ const bytes = Uint8Array.from(atob(audioBase64), (c) => c.charCodeAt(0))
+ const url = URL.createObjectURL(new Blob([bytes], { type: mimeType }))
+ objectUrls.add(url)
+ return url
+}
+
+async function speakCloud(
+ id: string,
+ chunks: string[],
+ options: SpeakOptions,
+ run: number
+) {
+ const { rate } = getSpeechPrefs().output
+ audio ??= new Audio()
+ const player = audio
+ let next = fetchChunk(chunks[0], rate, run)
+ try {
+ for (let index = 0; index < chunks.length; index += 1) {
+ const url = await next
+ if (url === null || run !== generation) return
+ if (index + 1 < chunks.length) {
+ next = fetchChunk(chunks[index + 1], rate, run)
+ next.catch(() => {})
+ }
+ await new Promise((resolve, reject) => {
+ player.onended = () => resolve()
+ player.onerror = () => reject(new Error("audio playback failed"))
+ player.src = url
+ player.play().then(() => {
+ if (run === generation) setState({ playingId: id, status: "playing" })
+ }, reject)
+ })
+ URL.revokeObjectURL(url)
+ objectUrls.delete(url)
+ if (run !== generation) return
+ }
+ setState(IDLE)
+ } catch (error) {
+ if (run !== generation) return
+ stopSpeech()
+ options.onError?.(classify(error))
+ }
+}
+
+async function resolvePlaybackEngine(): Promise {
+ const pref = getSpeechPrefs().output
+ const voices = pref.engine === "cloud" ? [] : await waitForVoices()
+ const browserTts = voices.length > 0
+ let cloudConfigured = false
+ if (pref.engine === "cloud" || (pref.engine === "auto" && !browserTts)) {
+ cloudConfigured = (await speechGetSettings()).apiKeySet
+ }
+ return resolveOutputEngine(pref, { browserTts }, cloudConfigured)
+}
+
+function start(
+ id: string,
+ markdown: string,
+ engine: "browser" | "cloud",
+ options: SpeakOptions,
+ run: number
+) {
+ const text = toSpeakableText(markdown, options.labels)
+ const chunks = chunkSpeakableText(
+ text,
+ engine === "cloud" ? CLOUD_CHUNK : BROWSER_CHUNK
+ )
+ if (chunks.length === 0) {
+ setState(IDLE)
+ return
+ }
+ if (engine === "browser") {
+ speakBrowser(id, chunks, options, run)
+ } else {
+ void speakCloud(id, chunks, options, run)
+ }
+}
+
+export function speak(id: string, markdown: string, options: SpeakOptions) {
+ stopSpeech()
+ const run = generation
+ setState({ playingId: id, status: "loading" })
+ if (options.engine) {
+ start(id, markdown, options.engine, options, run)
+ return
+ }
+ resolvePlaybackEngine().then(
+ (resolution) => {
+ if (run !== generation) return
+ if (resolution.engine === null) {
+ stopSpeech()
+ options.onError?.(
+ resolution.reason === "cloud-not-configured"
+ ? "not-configured"
+ : "unavailable"
+ )
+ return
+ }
+ start(id, markdown, resolution.engine, options, run)
+ },
+ (error: unknown) => {
+ if (run !== generation) return
+ stopSpeech()
+ options.onError?.(classify(error))
+ }
+ )
+}
+
+export interface AutoReadContext {
+ contextKey: string
+ activeId: string | null
+ visibility: DocumentVisibilityState
+}
+
+export function maybeAutoRead(
+ { contextKey, activeId, visibility }: AutoReadContext,
+ text: string,
+ options: SpeakOptions
+): boolean {
+ const { enabled, autoRead } = getSpeechPrefs().output
+ if (!enabled || !autoRead) return false
+ if (contextKey !== activeId || visibility !== "visible") return false
+ if (!text.trim()) return false
+ speak(`auto:${contextKey}`, text, options)
+ return true
+}
+
+export function resetSpeechPlayerForTests(): void {
+ stopSpeech()
+ audio = null
+ listeners.clear()
+ state = IDLE
+}
diff --git a/src/lib/speech-prefs.test.ts b/src/lib/speech-prefs.test.ts
new file mode 100644
index 0000000000..a5a33cacc6
--- /dev/null
+++ b/src/lib/speech-prefs.test.ts
@@ -0,0 +1,161 @@
+import { beforeEach, describe, expect, it } from "vitest"
+
+import {
+ DEFAULT_SPEECH_PREFS,
+ getSpeechPrefs,
+ loadSpeechPrefs,
+ parseSpeechPrefs,
+ resetSpeechPrefsCacheForTests,
+ saveSpeechPrefs,
+ subscribeSpeechPrefs,
+} from "./speech-prefs"
+
+describe("speech preferences", () => {
+ beforeEach(() => {
+ localStorage.clear()
+ resetSpeechPrefsCacheForTests()
+ })
+
+ it("defaults to disabled with auto engine and empty language", () => {
+ expect(DEFAULT_SPEECH_PREFS).toEqual({
+ input: {
+ enabled: false,
+ engine: "auto",
+ language: "",
+ },
+ output: {
+ enabled: false,
+ engine: "auto",
+ browserVoiceUri: "",
+ rate: 1,
+ autoRead: false,
+ },
+ })
+ expect(loadSpeechPrefs()).toEqual(DEFAULT_SPEECH_PREFS)
+ })
+
+ it("returns a fresh object copy on load to avoid mutating default", () => {
+ const loaded = loadSpeechPrefs()
+ loaded.input.enabled = true
+ expect(DEFAULT_SPEECH_PREFS.input.enabled).toBe(false)
+ })
+
+ it("round-trips valid preference changes through save and load", () => {
+ const custom = {
+ input: {
+ enabled: true,
+ engine: "cloud" as const,
+ language: "zh-CN",
+ },
+ output: {
+ enabled: true,
+ engine: "browser" as const,
+ browserVoiceUri: "Google US English",
+ rate: 1.5,
+ autoRead: true,
+ },
+ }
+ saveSpeechPrefs(custom)
+ expect(loadSpeechPrefs()).toEqual(custom)
+ })
+
+ it("saving one section keeps the other", () => {
+ saveSpeechPrefs({
+ output: { ...DEFAULT_SPEECH_PREFS.output, enabled: true, rate: 1.25 },
+ })
+ saveSpeechPrefs({
+ input: { enabled: true, engine: "browser", language: "" },
+ })
+ const loaded = loadSpeechPrefs()
+ expect(loaded.input.enabled).toBe(true)
+ expect(loaded.output).toEqual({
+ ...DEFAULT_SPEECH_PREFS.output,
+ enabled: true,
+ rate: 1.25,
+ })
+ })
+
+ it("parses output per field and clamps the rate", () => {
+ expect(
+ parseSpeechPrefs({
+ output: {
+ enabled: 1,
+ engine: "loud",
+ browserVoiceUri: null,
+ rate: "fast",
+ autoRead: "yes",
+ },
+ }).output
+ ).toEqual(DEFAULT_SPEECH_PREFS.output)
+ expect(parseSpeechPrefs({ output: { rate: 9 } }).output.rate).toBe(2)
+ expect(parseSpeechPrefs({ output: { rate: 0.1 } }).output.rate).toBe(0.5)
+ expect(parseSpeechPrefs({ output: { rate: Number.NaN } }).output.rate).toBe(
+ 1
+ )
+ })
+
+ it("falls back per-field for invalid or missing values", () => {
+ const parsed = parseSpeechPrefs({
+ input: {
+ enabled: "yes",
+ engine: "invalid-engine",
+ language: 12345,
+ },
+ })
+ expect(parsed).toEqual(DEFAULT_SPEECH_PREFS)
+
+ const partial = parseSpeechPrefs({
+ input: {
+ enabled: true,
+ engine: "browser",
+ },
+ })
+ expect(partial.input.enabled).toBe(true)
+ expect(partial.input.engine).toBe("browser")
+ expect(partial.input.language).toBe("")
+ })
+
+ it("falls back to default on corrupt storage JSON", () => {
+ localStorage.setItem("settings:speech:v1", "corrupt{json")
+ expect(loadSpeechPrefs()).toEqual(DEFAULT_SPEECH_PREFS)
+ })
+
+ it("notifies same-window subscribers and invalidates snapshot on save", () => {
+ const changes: boolean[] = []
+ const unsubscribe = subscribeSpeechPrefs(() => {
+ changes.push(getSpeechPrefs().input.enabled)
+ })
+
+ saveSpeechPrefs({
+ input: {
+ enabled: true,
+ engine: "auto",
+ language: "en-US",
+ },
+ })
+
+ unsubscribe()
+ expect(changes).toEqual([true])
+ })
+
+ it("invalidates memoized snapshot on storage event", () => {
+ const first = getSpeechPrefs()
+ expect(getSpeechPrefs()).toBe(first)
+
+ localStorage.setItem(
+ "settings:speech:v1",
+ JSON.stringify({
+ input: { enabled: true, engine: "cloud", language: "ja-JP" },
+ })
+ )
+ window.dispatchEvent(
+ new StorageEvent("storage", { key: "settings:speech:v1" })
+ )
+
+ const second = getSpeechPrefs()
+ expect(second).not.toBe(first)
+ expect(second.input.enabled).toBe(true)
+ expect(second.input.engine).toBe("cloud")
+ expect(second.input.language).toBe("ja-JP")
+ })
+})
diff --git a/src/lib/speech-prefs.ts b/src/lib/speech-prefs.ts
new file mode 100644
index 0000000000..45ba6c9182
--- /dev/null
+++ b/src/lib/speech-prefs.ts
@@ -0,0 +1,218 @@
+"use client"
+
+/**
+ * Speech preferences: dictation (input) and read-aloud (output) settings.
+ *
+ * Stored in localStorage rather than the backend because speech input configuration
+ * (microphone access, engine preferences) is per-device. Uses the same reactive
+ * pattern as `notification-sound-prefs.ts`: a custom event for the current window
+ * plus the native `storage` event for cross-window/tab sync.
+ */
+
+import { useSyncExternalStore } from "react"
+
+const PREFS_KEY = "settings:speech:v1"
+const PREFS_EVENT = "codeg:speech-prefs-changed"
+
+export type SpeechEnginePreference = "auto" | "browser" | "cloud"
+
+export interface SpeechInputPrefs {
+ /** Master switch for speech input. Off by default. */
+ enabled: boolean
+ /** STT engine preference. */
+ engine: SpeechEnginePreference
+ /** BCP-47 tag or locale string; empty string means follow UI locale. */
+ language: string
+}
+
+export interface SpeechOutputPrefs {
+ enabled: boolean
+ engine: SpeechEnginePreference
+ /** `SpeechSynthesisVoice.voiceURI`; empty means the default voice for the language. */
+ browserVoiceUri: string
+ rate: number
+ autoRead: boolean
+}
+
+export interface SpeechPrefs {
+ input: SpeechInputPrefs
+ output: SpeechOutputPrefs
+}
+
+export const MIN_SPEECH_RATE = 0.5
+export const MAX_SPEECH_RATE = 2
+
+export const DEFAULT_SPEECH_PREFS: SpeechPrefs = {
+ input: {
+ enabled: false,
+ engine: "auto",
+ language: "",
+ },
+ output: {
+ enabled: false,
+ engine: "auto",
+ browserVoiceUri: "",
+ rate: 1,
+ autoRead: false,
+ },
+}
+
+function defaultPrefs(): SpeechPrefs {
+ return {
+ input: { ...DEFAULT_SPEECH_PREFS.input },
+ output: { ...DEFAULT_SPEECH_PREFS.output },
+ }
+}
+
+function isSpeechEnginePreference(
+ value: unknown
+): value is SpeechEnginePreference {
+ return value === "auto" || value === "browser" || value === "cloud"
+}
+
+function asRecord(value: unknown): Record | null {
+ return value && typeof value === "object"
+ ? (value as Record)
+ : null
+}
+
+function parseInput(raw: unknown): SpeechInputPrefs {
+ const defaults = DEFAULT_SPEECH_PREFS.input
+ const source = asRecord(raw)
+ if (!source) return { ...defaults }
+ return {
+ enabled:
+ typeof source.enabled === "boolean" ? source.enabled : defaults.enabled,
+ engine: isSpeechEnginePreference(source.engine)
+ ? source.engine
+ : defaults.engine,
+ language:
+ typeof source.language === "string" ? source.language : defaults.language,
+ }
+}
+
+export function clampSpeechRate(rate: number): number {
+ return Math.min(MAX_SPEECH_RATE, Math.max(MIN_SPEECH_RATE, rate))
+}
+
+function parseOutput(raw: unknown): SpeechOutputPrefs {
+ const defaults = DEFAULT_SPEECH_PREFS.output
+ const source = asRecord(raw)
+ if (!source) return { ...defaults }
+ return {
+ enabled:
+ typeof source.enabled === "boolean" ? source.enabled : defaults.enabled,
+ engine: isSpeechEnginePreference(source.engine)
+ ? source.engine
+ : defaults.engine,
+ browserVoiceUri:
+ typeof source.browserVoiceUri === "string"
+ ? source.browserVoiceUri
+ : defaults.browserVoiceUri,
+ rate:
+ typeof source.rate === "number" && Number.isFinite(source.rate)
+ ? clampSpeechRate(source.rate)
+ : defaults.rate,
+ autoRead:
+ typeof source.autoRead === "boolean"
+ ? source.autoRead
+ : defaults.autoRead,
+ }
+}
+
+/**
+ * Merge a stored blob over the defaults, field by field. Every field is
+ * validated independently so a partial write from an older build (or a
+ * hand-edited value) degrades to the default for that one field instead of
+ * discarding the whole preference set.
+ */
+export function parseSpeechPrefs(raw: unknown): SpeechPrefs {
+ const source = asRecord(raw)
+ return {
+ input: parseInput(source?.input),
+ output: parseOutput(source?.output),
+ }
+}
+
+export function loadSpeechPrefs(): SpeechPrefs {
+ if (typeof window === "undefined") return defaultPrefs()
+ try {
+ const raw = localStorage.getItem(PREFS_KEY)
+ if (!raw) return defaultPrefs()
+ return parseSpeechPrefs(JSON.parse(raw))
+ } catch {
+ return defaultPrefs()
+ }
+}
+
+/** Saves a partial update; omitted sections keep their current values. */
+export function saveSpeechPrefs(update: {
+ input?: SpeechInputPrefs
+ output?: SpeechOutputPrefs
+}): void {
+ if (typeof window === "undefined") return
+ const prefs = parseSpeechPrefs({ ...loadSpeechPrefs(), ...update })
+ try {
+ localStorage.setItem(PREFS_KEY, JSON.stringify(prefs))
+ } catch {
+ /* ignore */
+ }
+ window.dispatchEvent(new CustomEvent(PREFS_EVENT, { detail: prefs }))
+}
+
+// ── Shared snapshot ──
+
+let snapshot: SpeechPrefs | null = null
+const listeners = new Set<() => void>()
+let windowBound = false
+
+function bindWindow(): void {
+ if (windowBound || typeof window === "undefined") return
+ windowBound = true
+ const invalidate = () => {
+ snapshot = null
+ for (const listener of listeners) listener()
+ }
+ window.addEventListener(PREFS_EVENT, invalidate)
+ window.addEventListener("storage", invalidate)
+}
+
+/**
+ * Current preferences, memoized. Identity only changes when the stored value
+ * does, so it is safe as a `useSyncExternalStore` snapshot.
+ */
+export function getSpeechPrefs(): SpeechPrefs {
+ bindWindow()
+ if (typeof window === "undefined") return DEFAULT_SPEECH_PREFS
+ snapshot ??= loadSpeechPrefs()
+ return snapshot
+}
+
+/** Subscribe to preference changes from this window or any other. */
+export function subscribeSpeechPrefs(onChange: () => void): () => void {
+ bindWindow()
+ listeners.add(onChange)
+ return () => {
+ listeners.delete(onChange)
+ }
+}
+
+function getServerSpeechPrefs(): SpeechPrefs {
+ return DEFAULT_SPEECH_PREFS
+}
+
+/** Reactive read of speech preferences; live across windows. */
+export function useSpeechPrefs(): SpeechPrefs {
+ return useSyncExternalStore(
+ subscribeSpeechPrefs,
+ getSpeechPrefs,
+ getServerSpeechPrefs
+ )
+}
+
+/** Test seam: forget the memoized snapshot so the next read hits storage. */
+export function resetSpeechPrefsCacheForTests(): void {
+ snapshot = null
+ windowBound = false
+ listeners.clear()
+}
diff --git a/src/lib/types.ts b/src/lib/types.ts
index b7c77f4712..fa83f18191 100644
--- a/src/lib/types.ts
+++ b/src/lib/types.ts
@@ -5074,3 +5074,33 @@ export interface DeepSeekModelCatalog {
* fixed, sessions run on the agent's built-in catalog instead. */
invalid: string | null
}
+export interface SpeechCloudSettings {
+ baseUrl: string
+ sttModel: string
+ ttsModel: string
+ ttsVoice: string
+}
+
+export interface SpeechCloudSettingsView {
+ settings: SpeechCloudSettings
+ apiKeySet: boolean
+}
+
+export interface SpeechAudio {
+ audioBase64: string
+ mimeType: string
+}
+
+export interface AssistantSettings {
+ agentType: AgentType | null
+ allowSessionControl: boolean
+ allowPermissionAnswers: boolean
+}
+
+export interface AssistantSession {
+ connectionId: string
+ conversationId: number
+ folderId: number
+ agentType: AgentType
+ primer: string | null
+}