feat(frontend): voice input and read-aloud UI
- assistant messages get a 🔊 button that fetches TTS audio and plays
it, toggling to stop
- ChatInput gains a mic button: MediaRecorder capture, base64 upload to
/voice/stt, recognized text fills the input
- api client: voice.tts (blob) and voice.stt helpers
This commit is contained in:
1 parent
a0357a7bbd
commit
2104fc0c6c
3 files changed
+105
-2
No files matched your search
@@ -62,6 +62,22 @@ export const api = {
|
||||
`/messages/search?q=${encodeURIComponent(q)}`),
|
||||
},
|
||||
|
||||
voice: {
|
||||
// tts returns the synthesized audio as a Blob (audio/mpeg).
|
||||
tts: async (text: string): Promise<Blob> => {
|
||||
const res = await fetch(`${BASE}/voice/tts`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}` } : {}) },
|
||||
body: JSON.stringify({ text }),
|
||||
})
|
||||
if (!res.ok) throw new Error(`tts failed: ${res.status}`)
|
||||
return res.blob()
|
||||
},
|
||||
// stt sends base64-encoded audio and returns the recognized text.
|
||||
stt: (audioBase64: string) =>
|
||||
request<{ text: string }>('/voice/stt', { method: 'POST', body: JSON.stringify({ audio_data: audioBase64 }) }),
|
||||
},
|
||||
|
||||
admin: {
|
||||
dashboard: () => request<{ dashboard: Record<string, unknown> }>('/admin/dashboard'),
|
||||
},
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { useRef, useState } from 'react'
|
||||
import { api } from '../../api/client'
|
||||
|
||||
interface ChatInputProps {
|
||||
value: string
|
||||
onChange: (v: string) => void
|
||||
@@ -6,6 +9,9 @@ interface ChatInputProps {
|
||||
}
|
||||
|
||||
export default function ChatInput({ value, onChange, onSend, disabled }: ChatInputProps) {
|
||||
const [recording, setRecording] = useState(false)
|
||||
const recorderRef = useRef<MediaRecorder | null>(null)
|
||||
|
||||
const handleKeyDown = (e: React.KeyboardEvent) => {
|
||||
if (e.key === 'Enter' && !e.shiftKey) {
|
||||
e.preventDefault()
|
||||
@@ -13,6 +19,36 @@ export default function ChatInput({ value, onChange, onSend, disabled }: ChatInp
|
||||
}
|
||||
}
|
||||
|
||||
const toggleRecord = async () => {
|
||||
if (recording) {
|
||||
recorderRef.current?.stop()
|
||||
return
|
||||
}
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
const recorder = new MediaRecorder(stream)
|
||||
const chunks: Blob[] = []
|
||||
recorder.ondataavailable = (e) => chunks.push(e.data)
|
||||
recorder.onstop = async () => {
|
||||
stream.getTracks().forEach((t) => t.stop())
|
||||
setRecording(false)
|
||||
const blob = new Blob(chunks, { type: recorder.mimeType })
|
||||
const dataUrl = await new Promise<string>((resolve) => {
|
||||
const reader = new FileReader()
|
||||
reader.onloadend = () => resolve(reader.result as string)
|
||||
reader.readAsDataURL(blob)
|
||||
})
|
||||
try {
|
||||
const { text } = await api.voice.stt(dataUrl.split(',')[1] || '')
|
||||
if (text.trim()) onChange(value ? `${value} ${text}` : text)
|
||||
} catch { /* STT unavailable */ }
|
||||
}
|
||||
recorderRef.current = recorder
|
||||
recorder.start()
|
||||
setRecording(true)
|
||||
} catch { /* mic permission denied */ }
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex items-end gap-2 bg-[var(--mx-surface)] rounded-[20px] border border-[var(--mx-outline)] focus-within:border-[var(--mx-primary)] transition-colors p-1.5">
|
||||
<input
|
||||
@@ -21,9 +57,21 @@ export default function ChatInput({ value, onChange, onSend, disabled }: ChatInp
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
onKeyDown={handleKeyDown}
|
||||
disabled={disabled}
|
||||
placeholder="输入消息..."
|
||||
placeholder={recording ? '正在录音,点击 ⏹ 结束…' : '输入消息...'}
|
||||
className="flex-1 bg-transparent px-3 py-1.5 text-sm text-[var(--mx-text)] placeholder:text-[var(--mx-text-dim)] outline-none disabled:opacity-40"
|
||||
/>
|
||||
<button
|
||||
onClick={toggleRecord}
|
||||
disabled={disabled}
|
||||
title={recording ? '结束录音并识别' : '语音输入'}
|
||||
className={`w-9 h-9 rounded-full flex items-center justify-center shrink-0 transition-all duration-200 ${
|
||||
recording
|
||||
? 'bg-red-500 animate-pulse text-white'
|
||||
: 'text-[var(--mx-text-dim)] hover:text-[var(--mx-text)]'
|
||||
}`}
|
||||
>
|
||||
{recording ? '⏹' : '🎤'}
|
||||
</button>
|
||||
<button
|
||||
onClick={onSend}
|
||||
disabled={disabled || !value.trim()}
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { useRef, useState } from 'react'
|
||||
import type { Message } from '../../types/chat'
|
||||
import { api } from '../../api/client'
|
||||
|
||||
export default function MessageList({ messages }: { messages: Message[] }) {
|
||||
return (
|
||||
@@ -10,6 +12,40 @@ export default function MessageList({ messages }: { messages: Message[] }) {
|
||||
)
|
||||
}
|
||||
|
||||
// SpeakButton fetches TTS audio for a message and plays/stops it.
|
||||
function SpeakButton({ text }: { text: string }) {
|
||||
const [state, setState] = useState<'idle' | 'loading' | 'playing'>('idle')
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null)
|
||||
|
||||
const toggle = async () => {
|
||||
if (state === 'playing') {
|
||||
audioRef.current?.pause()
|
||||
audioRef.current = null
|
||||
setState('idle')
|
||||
return
|
||||
}
|
||||
setState('loading')
|
||||
try {
|
||||
const blob = await api.voice.tts(text)
|
||||
const audio = new Audio(URL.createObjectURL(blob))
|
||||
audioRef.current = audio
|
||||
audio.onended = () => { setState('idle'); audioRef.current = null }
|
||||
await audio.play()
|
||||
setState('playing')
|
||||
} catch {
|
||||
setState('idle')
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<button onClick={toggle}
|
||||
className="ml-2 text-xs opacity-40 hover:opacity-100 transition-opacity align-middle"
|
||||
title={state === 'playing' ? '停止' : '朗读'}>
|
||||
{state === 'loading' ? '⏳' : state === 'playing' ? '⏹' : '🔊'}
|
||||
</button>
|
||||
)
|
||||
}
|
||||
|
||||
function MessageBubble({ message }: { message: Message }) {
|
||||
const isUser = message.role === 'user'
|
||||
const isAction = message.role === 'action'
|
||||
@@ -57,7 +93,10 @@ function MessageBubble({ message }: { message: Message }) {
|
||||
<span className="w-2 h-2 rounded-full bg-[#404040] animate-bounce" style={{animationDelay:'300ms'}} />
|
||||
</span>
|
||||
) : (
|
||||
message.content
|
||||
<>
|
||||
{message.content}
|
||||
{!isUser && message.role === 'assistant' && <SpeakButton text={message.content} />}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
|
||||
Reference in new issue
Block a user