feat(frontend): voice input and read-aloud UI

- assistant messages get a 🔊 button that fetches TTS audio and plays
  it, toggling to stop
- ChatInput gains a mic button: MediaRecorder capture, base64 upload to
  /voice/stt, recognized text fills the input
- api client: voice.tts (blob) and voice.stt helpers
This commit is contained in:
Aska committed 2026-08-21 18:31:24 +08:00
1 parent a0357a7bbd
commit 2104fc0c6c
3 files changed
+105 -2

No files matched your search

+16
View File
@@ -62,6 +62,22 @@ export const api = {
`/messages/search?q=${encodeURIComponent(q)}`),
},
voice: {
// tts returns the synthesized audio as a Blob (audio/mpeg).
tts: async (text: string): Promise<Blob> => {
const res = await fetch(`${BASE}/voice/tts`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}` } : {}) },
body: JSON.stringify({ text }),
})
if (!res.ok) throw new Error(`tts failed: ${res.status}`)
return res.blob()
},
// stt sends base64-encoded audio and returns the recognized text.
stt: (audioBase64: string) =>
request<{ text: string }>('/voice/stt', { method: 'POST', body: JSON.stringify({ audio_data: audioBase64 }) }),
},
admin: {
dashboard: () => request<{ dashboard: Record<string, unknown> }>('/admin/dashboard'),
},
+49 -1
View File
@@ -1,3 +1,6 @@
import { useRef, useState } from 'react'
import { api } from '../../api/client'
interface ChatInputProps {
value: string
onChange: (v: string) => void
@@ -6,6 +9,9 @@ interface ChatInputProps {
}
export default function ChatInput({ value, onChange, onSend, disabled }: ChatInputProps) {
const [recording, setRecording] = useState(false)
const recorderRef = useRef<MediaRecorder | null>(null)
const handleKeyDown = (e: React.KeyboardEvent) => {
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault()
@@ -13,6 +19,36 @@ export default function ChatInput({ value, onChange, onSend, disabled }: ChatInp
}
}
const toggleRecord = async () => {
if (recording) {
recorderRef.current?.stop()
return
}
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
const recorder = new MediaRecorder(stream)
const chunks: Blob[] = []
recorder.ondataavailable = (e) => chunks.push(e.data)
recorder.onstop = async () => {
stream.getTracks().forEach((t) => t.stop())
setRecording(false)
const blob = new Blob(chunks, { type: recorder.mimeType })
const dataUrl = await new Promise<string>((resolve) => {
const reader = new FileReader()
reader.onloadend = () => resolve(reader.result as string)
reader.readAsDataURL(blob)
})
try {
const { text } = await api.voice.stt(dataUrl.split(',')[1] || '')
if (text.trim()) onChange(value ? `${value} ${text}` : text)
} catch { /* STT unavailable */ }
}
recorderRef.current = recorder
recorder.start()
setRecording(true)
} catch { /* mic permission denied */ }
}
return (
<div className="flex items-end gap-2 bg-[var(--mx-surface)] rounded-[20px] border border-[var(--mx-outline)] focus-within:border-[var(--mx-primary)] transition-colors p-1.5">
<input
@@ -21,9 +57,21 @@ export default function ChatInput({ value, onChange, onSend, disabled }: ChatInp
onChange={(e) => onChange(e.target.value)}
onKeyDown={handleKeyDown}
disabled={disabled}
placeholder="输入消息..."
placeholder={recording ? '正在录音,点击 ⏹ 结束…' : '输入消息...'}
className="flex-1 bg-transparent px-3 py-1.5 text-sm text-[var(--mx-text)] placeholder:text-[var(--mx-text-dim)] outline-none disabled:opacity-40"
/>
<button
onClick={toggleRecord}
disabled={disabled}
title={recording ? '结束录音并识别' : '语音输入'}
className={`w-9 h-9 rounded-full flex items-center justify-center shrink-0 transition-all duration-200 ${
recording
? 'bg-red-500 animate-pulse text-white'
: 'text-[var(--mx-text-dim)] hover:text-[var(--mx-text)]'
}`}
>
{recording ? '⏹' : '🎤'}
</button>
<button
onClick={onSend}
disabled={disabled || !value.trim()}
@@ -1,4 +1,6 @@
import { useRef, useState } from 'react'
import type { Message } from '../../types/chat'
import { api } from '../../api/client'
export default function MessageList({ messages }: { messages: Message[] }) {
return (
@@ -10,6 +12,40 @@ export default function MessageList({ messages }: { messages: Message[] }) {
)
}
// SpeakButton fetches TTS audio for a message and plays/stops it.
function SpeakButton({ text }: { text: string }) {
const [state, setState] = useState<'idle' | 'loading' | 'playing'>('idle')
const audioRef = useRef<HTMLAudioElement | null>(null)
const toggle = async () => {
if (state === 'playing') {
audioRef.current?.pause()
audioRef.current = null
setState('idle')
return
}
setState('loading')
try {
const blob = await api.voice.tts(text)
const audio = new Audio(URL.createObjectURL(blob))
audioRef.current = audio
audio.onended = () => { setState('idle'); audioRef.current = null }
await audio.play()
setState('playing')
} catch {
setState('idle')
}
}
return (
<button onClick={toggle}
className="ml-2 text-xs opacity-40 hover:opacity-100 transition-opacity align-middle"
title={state === 'playing' ? '停止' : '朗读'}>
{state === 'loading' ? '⏳' : state === 'playing' ? '⏹' : '🔊'}
</button>
)
}
function MessageBubble({ message }: { message: Message }) {
const isUser = message.role === 'user'
const isAction = message.role === 'action'
@@ -57,7 +93,10 @@ function MessageBubble({ message }: { message: Message }) {
<span className="w-2 h-2 rounded-full bg-[#404040] animate-bounce" style={{animationDelay:'300ms'}} />
</span>
) : (
message.content
<>
{message.content}
{!isUser && message.role === 'assistant' && <SpeakButton text={message.content} />}
</>
)}
</div>