feat: 语音流式输入管线 + VAD前端集成 + 插件-工具合并清理
- 前端: VAD语音检测(@ricky0123/vad-web) + useVoiceInput双模式(流式WS/REST) - Gateway: VoiceStreamManager代理WS流式STT到voice-service - Voice-service: DashScope REST → Realtime WS → Whisper三级引擎 + ffmpeg转码 - 共享模块: pkg/audio(音频转换) + pkg/dashscope(ASR REST客户端) - 清理: 移除旧plugin-manager和pkg/plugins，完成插件→工具合并 - 文档: 完善gateway-api.md和voice-service.md语音API文档 - 工具: scripts/voice/ 语音转换脚本集 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1,11 +1,13 @@
|
||||
import { useState, useRef, useCallback, useEffect } from 'react';
|
||||
import type { ChatMode, MessageAttachment } from '@/types/chat';
|
||||
import { useSpeechRecognition } from '@/hooks/useSpeechRecognition';
|
||||
import { useVoiceInput } from '@/hooks/useVoiceInput';
|
||||
import { uploadFile } from '@/api/files';
|
||||
import { useChatStore } from '@/store/chatStore';
|
||||
|
||||
interface ChatInputProps {
|
||||
onSend: (content: string, mode: ChatMode, attachments?: MessageAttachment[]) => void;
|
||||
onSendVoiceStream?: (msg: import('@/types/chat').WSClientMessage) => void;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
@@ -19,7 +21,7 @@ const MAX_IMAGE_SIZE = 10 * 1024 * 1024; // 10MB
|
||||
const SUPPORTED_IMAGE_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp', 'image/bmp'];
|
||||
const MAX_IMAGES = 5;
|
||||
|
||||
export function ChatInput({ onSend, disabled }: ChatInputProps) {
|
||||
export function ChatInput({ onSend, onSendVoiceStream, disabled }: ChatInputProps) {
|
||||
const [content, setContent] = useState('');
|
||||
const [mode, setMode] = useState<ChatMode>('text');
|
||||
const [pendingImages, setPendingImages] = useState<PendingImage[]>([]);
|
||||
@@ -30,27 +32,50 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
|
||||
const isTyping = useChatStore((s) => s.isTyping);
|
||||
|
||||
const {
|
||||
isListening,
|
||||
isSupported,
|
||||
isListening: isSRListening,
|
||||
isSupported: isSRSpported,
|
||||
isFallbackMode,
|
||||
interimText,
|
||||
finalText,
|
||||
error,
|
||||
startListening,
|
||||
stopListening,
|
||||
error: srError,
|
||||
startListening: startSR,
|
||||
stopListening: stopSR,
|
||||
resetText,
|
||||
} = useSpeechRecognition();
|
||||
|
||||
// 当 finalText 更新时，追加到输入框
|
||||
// VAD-based voice input (primary when supported)
|
||||
const {
|
||||
isListening: isVADListening,
|
||||
isSpeaking: isVADSpeaking,
|
||||
isSupported: isVADSupported,
|
||||
interimText: vadInterimText,
|
||||
finalText: vadFinalText,
|
||||
error: vadError,
|
||||
startListening: startVAD,
|
||||
stopListening: stopVAD,
|
||||
} = useVoiceInput({
|
||||
onTranscription: (text: string) => {
|
||||
setContent((prev) => {
|
||||
const trimmed = prev.trimEnd();
|
||||
return (trimmed ? trimmed + ' ' : '') + text;
|
||||
});
|
||||
},
|
||||
sendMessage: onSendVoiceStream,
|
||||
});
|
||||
|
||||
const isListening = isVADSupported ? isVADListening : isSRListening;
|
||||
const voiceError = isVADSupported ? vadError : srError;
|
||||
|
||||
// 当 SR finalText 更新时，追加到输入框 (仅非 VAD 模式)
|
||||
useEffect(() => {
|
||||
if (finalText) {
|
||||
if (!isVADSupported && finalText) {
|
||||
setContent((prev) => {
|
||||
const trimmed = prev.trimEnd();
|
||||
return (trimmed ? trimmed + ' ' : '') + finalText;
|
||||
});
|
||||
resetText();
|
||||
}
|
||||
}, [finalText, resetText]);
|
||||
}, [isVADSupported, finalText, resetText]);
|
||||
|
||||
const handleSend = useCallback(async () => {
|
||||
const trimmed = content.trim();
|
||||
@@ -121,13 +146,13 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
|
||||
if (e.key === 'V' && e.ctrlKey && e.shiftKey) {
|
||||
e.preventDefault();
|
||||
if (isListening) {
|
||||
stopListening();
|
||||
isVADSupported ? stopVAD() : stopSR();
|
||||
} else {
|
||||
startListening();
|
||||
isVADSupported ? startVAD() : startSR();
|
||||
}
|
||||
}
|
||||
},
|
||||
[handleSend, isListening, startListening, stopListening]
|
||||
[handleSend, isListening, isVADSupported, startVAD, stopVAD, startSR, stopSR]
|
||||
);
|
||||
|
||||
// 粘贴图片
|
||||
@@ -260,11 +285,11 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
|
||||
|
||||
const handleVoiceToggle = useCallback(() => {
|
||||
if (isListening) {
|
||||
stopListening();
|
||||
isVADSupported ? stopVAD() : stopSR();
|
||||
} else {
|
||||
startListening();
|
||||
isVADSupported ? startVAD() : startSR();
|
||||
}
|
||||
}, [isListening, startListening, stopListening]);
|
||||
}, [isListening, isVADSupported, startVAD, stopVAD, startSR, stopSR]);
|
||||
|
||||
return (
|
||||
<div
|
||||
@@ -305,8 +330,17 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* 实时识别文本提示 */}
|
||||
{isListening && interimText && (
|
||||
{/* VAD 语音状态提示 */}
|
||||
{isVADSupported && isVADListening && (
|
||||
<div className="text-sm text-pink-500 dark:text-pink-400 italic px-1" aria-live="polite">
|
||||
{isVADSpeaking
|
||||
? (vadInterimText || '检测到语音，正在识别...')
|
||||
: '正在聆听...'}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* 实时识别文本提示 (仅非 VAD 模式) */}
|
||||
{!isVADSupported && isListening && interimText && (
|
||||
<div
|
||||
className="interim-text text-sm text-pink-500 dark:text-pink-400 italic px-1"
|
||||
aria-live="polite"
|
||||
@@ -317,12 +351,12 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
|
||||
)}
|
||||
|
||||
{/* 错误提示 */}
|
||||
{error && (
|
||||
{voiceError && (
|
||||
<div
|
||||
className="text-xs text-red-500 dark:text-red-400 px-1"
|
||||
role="alert"
|
||||
>
|
||||
⚠️ {error}
|
||||
⚠️ {voiceError}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -434,18 +468,20 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
|
||||
className="flex-1 resize-none rounded-xl border border-pink-200 dark:border-pink-800 bg-white dark:bg-gray-800 px-4 py-2 text-sm text-gray-700 dark:text-gray-200 placeholder-gray-400 focus:outline-none focus:ring-2 focus:ring-pink-400 focus:border-transparent disabled:opacity-50"
|
||||
/>
|
||||
|
||||
{/* 语音输入按钮 (仅浏览器支持时显示) */}
|
||||
{isSupported && (
|
||||
{/* 语音输入按钮 (VAD 或浏览器 SpeechRecognition/MediaRecorder 支持时显示) */}
|
||||
{(isVADSupported || isSRSpported) && (
|
||||
<button
|
||||
onClick={handleVoiceToggle}
|
||||
disabled={disabled || uploading}
|
||||
aria-label={isListening ? '停止语音输入' : '开始语音输入'}
|
||||
aria-pressed={isListening}
|
||||
title={isListening ? '停止聆听 (Ctrl+Shift+V)' : '语音输入 (Ctrl+Shift+V)'}
|
||||
title={isListening ? '停止聆听 (Ctrl+Shift+V)' : isVADSupported ? '语音输入 (自动检测说话)' : '语音输入 (Ctrl+Shift+V)'}
|
||||
className={`p-2 rounded-xl transition-all flex-shrink-0 border-2 ${
|
||||
isListening
|
||||
? 'voice-btn-active bg-red-500 border-red-500 text-white'
|
||||
: 'bg-gray-100 dark:bg-gray-700 border-gray-200 dark:border-gray-600 text-gray-500 hover:text-red-500 hover:border-red-300'
|
||||
: isVADSpeaking
|
||||
? 'bg-yellow-400 border-yellow-400 text-white'
|
||||
: 'bg-gray-100 dark:bg-gray-700 border-gray-200 dark:border-gray-600 text-gray-500 hover:text-red-500 hover:border-red-300'
|
||||
} disabled:opacity-40 disabled:cursor-not-allowed`}
|
||||
>
|
||||
<svg
|
||||
@@ -461,7 +497,7 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
|
||||
)}
|
||||
|
||||
{/* 不支持时显示禁用按钮 */}
|
||||
{!isSupported && (
|
||||
{!isVADSupported && !isSRSpported && (
|
||||
<button
|
||||
disabled
|
||||
title="您的浏览器不支持语音识别"
|
||||
@@ -508,7 +544,10 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
|
||||
{/* 语音输入状态提示 */}
|
||||
{isListening && (
|
||||
<p className="text-xs text-red-400 text-center animate-pulse">
|
||||
{isFallbackMode ? '🎤 后端语音识别中...' : '🎤 正在聆听...'}
|
||||
{isVADSupported
|
||||
? (isVADSpeaking ? '🔊 检测到语音，正在识别...' : '🎤 正在聆听...')
|
||||
: (isFallbackMode ? '🎤 后端语音识别中...' : '🎤 正在聆听...')
|
||||
}
|
||||
<span className="text-gray-400 ml-2">(Ctrl+Shift+V 停止)</span>
|
||||
</p>
|
||||
)}
|
||||
|
||||
Reference in New Issue
Block a user