AskaEth/Cyrene
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: 语音流式输入管线 + VAD前端集成 + 插件-工具合并清理

Browse Source 
- 前端: VAD语音检测(@ricky0123/vad-web) + useVoiceInput双模式(流式WS/REST)
- Gateway: VoiceStreamManager代理WS流式STT到voice-service
- Voice-service: DashScope REST → Realtime WS → Whisper三级引擎 + ffmpeg转码
- 共享模块: pkg/audio(音频转换) + pkg/dashscope(ASR REST客户端)
- 清理: 移除旧plugin-manager和pkg/plugins，完成插件→工具合并
- 文档: 完善gateway-api.md和voice-service.md语音API文档
- 工具: scripts/voice/ 语音转换脚本集

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
AskaEth
2026-06-06 11:50:40 +08:00
parent 258cf81b25
commit 6ef9e082a6
91 changed files with 4091 additions and 3929 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/web/src/App.tsx
+9 -8
View File
@@ -36,7 +36,7 @@ function setHashSessionId(sessionId: string | null) {
export default function App() {
const { isLoggedIn, login, register, loading: authLoading, userId } = useAuth();
const { send } = useChat();
const { send, sendVoiceStreamMessage } = useChat();
const { loadSessionsFromServer, ensureMainSession, setCurrentSessionId, setMessages, loadMessagesFromServer, sessions, currentSessionId } = useSessionStore();
const [authMode, setAuthMode] = useState<'login' | 'register'>('login');
@@ -330,41 +330,42 @@ export default function App() {
return (
<ErrorBoundary>
<AppLayout>
<PageRouter onSend={send} />
<PageRouter onSend={send} onSendVoiceStream={sendVoiceStreamMessage} />
</AppLayout>
</ErrorBoundary>
);
}
type SendFn = (content: string, mode?: import('@/types/chat').ChatMode, attachments?: import('@/types/chat').MessageAttachment[]) => void;
type SendVoiceStreamFn = (msg: import('@/types/chat').WSClientMessage) => void;
function PageRouter({ onSend }: { onSend: SendFn }) {
function PageRouter({ onSend, onSendVoiceStream }: { onSend: SendFn; onSendVoiceStream: SendVoiceStreamFn }) {
const currentPage = usePageStore((s) => s.currentPage);
const isAdmin = isAdminUser(localStorage.getItem('user_id') || '');
switch (currentPage) {
case 'admin-models':
if (!isAdmin) return <ChatPage onSend={onSend} />;
if (!isAdmin) return <ChatPage onSend={onSend} onSendVoiceStream={onSendVoiceStream} />;
return <ModelsAdminPage />;
case 'admin-dashboard':
if (!isAdmin) return <ChatPage onSend={onSend} />;
if (!isAdmin) return <ChatPage onSend={onSend} onSendVoiceStream={onSendVoiceStream} />;
return <AdminDashboard />;
case 'profile':
return <ProfilePage />;
case 'chat':
default:
return <ChatPage onSend={onSend} />;
return <ChatPage onSend={onSend} onSendVoiceStream={onSendVoiceStream} />;
}
}
function ChatPage({ onSend }: { onSend: SendFn }) {
function ChatPage({ onSend, onSendVoiceStream }: { onSend: SendFn; onSendVoiceStream: SendVoiceStreamFn }) {
return (
<div className="flex flex-col h-full overflow-hidden">
<div className="flex-1 min-h-0 overflow-hidden">
<ChatContainer />
</div>
<div className="flex-shrink-0">
<ChatInput onSend={onSend} />
<ChatInput onSend={onSend} onSendVoiceStream={onSendVoiceStream} />
</div>
</div>
);
frontend/web/src/components/chat/ChatInput.tsx
+64 -25
View File
@@ -1,11 +1,13 @@
import { useState, useRef, useCallback, useEffect } from 'react';
import type { ChatMode, MessageAttachment } from '@/types/chat';
import { useSpeechRecognition } from '@/hooks/useSpeechRecognition';
import { useVoiceInput } from '@/hooks/useVoiceInput';
import { uploadFile } from '@/api/files';
import { useChatStore } from '@/store/chatStore';
interface ChatInputProps {
onSend: (content: string, mode: ChatMode, attachments?: MessageAttachment[]) => void;
onSendVoiceStream?: (msg: import('@/types/chat').WSClientMessage) => void;
disabled?: boolean;
}
@@ -19,7 +21,7 @@ const MAX_IMAGE_SIZE = 10 * 1024 * 1024; // 10MB
const SUPPORTED_IMAGE_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp', 'image/bmp'];
const MAX_IMAGES = 5;
export function ChatInput({ onSend, disabled }: ChatInputProps) {
export function ChatInput({ onSend, onSendVoiceStream, disabled }: ChatInputProps) {
const [content, setContent] = useState('');
const [mode, setMode] = useState<ChatMode>('text');
const [pendingImages, setPendingImages] = useState<PendingImage[]>([]);
@@ -30,27 +32,50 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
const isTyping = useChatStore((s) => s.isTyping);
const {
isListening,
isSupported,
isListening: isSRListening,
isSupported: isSRSpported,
isFallbackMode,
interimText,
finalText,
error,
startListening,
stopListening,
error: srError,
startListening: startSR,
stopListening: stopSR,
resetText,
} = useSpeechRecognition();
// 当 finalText 更新时，追加到输入框
// VAD-based voice input (primary when supported)
const {
isListening: isVADListening,
isSpeaking: isVADSpeaking,
isSupported: isVADSupported,
interimText: vadInterimText,
finalText: vadFinalText,
error: vadError,
startListening: startVAD,
stopListening: stopVAD,
} = useVoiceInput({
onTranscription: (text: string) => {
setContent((prev) => {
const trimmed = prev.trimEnd();
return (trimmed ? trimmed + ' ' : '') + text;
});
},
sendMessage: onSendVoiceStream,
});
const isListening = isVADSupported ? isVADListening : isSRListening;
const voiceError = isVADSupported ? vadError : srError;
// 当 SR finalText 更新时，追加到输入框 (仅非 VAD 模式)
useEffect(() => {
if (finalText) {
if (!isVADSupported && finalText) {
setContent((prev) => {
const trimmed = prev.trimEnd();
return (trimmed ? trimmed + ' ' : '') + finalText;
});
resetText();
}
}, [finalText, resetText]);
}, [isVADSupported, finalText, resetText]);
const handleSend = useCallback(async () => {
const trimmed = content.trim();
@@ -121,13 +146,13 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
if (e.key === 'V' && e.ctrlKey && e.shiftKey) {
e.preventDefault();
if (isListening) {
stopListening();
isVADSupported ? stopVAD() : stopSR();
} else {
startListening();
isVADSupported ? startVAD() : startSR();
}
}
},
[handleSend, isListening, startListening, stopListening]
[handleSend, isListening, isVADSupported, startVAD, stopVAD, startSR, stopSR]
);
// 粘贴图片
@@ -260,11 +285,11 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
const handleVoiceToggle = useCallback(() => {
if (isListening) {
stopListening();
isVADSupported ? stopVAD() : stopSR();
} else {
startListening();
isVADSupported ? startVAD() : startSR();
}
}, [isListening, startListening, stopListening]);
}, [isListening, isVADSupported, startVAD, stopVAD, startSR, stopSR]);
return (
<div
@@ -305,8 +330,17 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
</div>
)}
{/* 实时识别文本提示 */}
{isListening && interimText && (
{/* VAD 语音状态提示 */}
{isVADSupported && isVADListening && (
<div className="text-sm text-pink-500 dark:text-pink-400 italic px-1" aria-live="polite">
{isVADSpeaking
? (vadInterimText || '检测到语音，正在识别...')
: '正在聆听...'}
</div>
)}
{/* 实时识别文本提示 (仅非 VAD 模式) */}
{!isVADSupported && isListening && interimText && (
<div
className="interim-text text-sm text-pink-500 dark:text-pink-400 italic px-1"
aria-live="polite"
@@ -317,12 +351,12 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
)}
{/* 错误提示 */}
{error && (
{voiceError && (
<div
className="text-xs text-red-500 dark:text-red-400 px-1"
role="alert"
>
⚠️ {error}
⚠️ {voiceError}
</div>
)}
@@ -434,18 +468,20 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
className="flex-1 resize-none rounded-xl border border-pink-200 dark:border-pink-800 bg-white dark:bg-gray-800 px-4 py-2 text-sm text-gray-700 dark:text-gray-200 placeholder-gray-400 focus:outline-none focus:ring-2 focus:ring-pink-400 focus:border-transparent disabled:opacity-50"
/>
{/* 语音输入按钮 (仅浏览器支持时显示) */}
{isSupported && (
{/* 语音输入按钮 (VAD 或浏览器 SpeechRecognition/MediaRecorder 支持时显示) */}
{(isVADSupported || isSRSpported) && (
<button
onClick={handleVoiceToggle}
disabled={disabled || uploading}
aria-label={isListening ? '停止语音输入' : '开始语音输入'}
aria-pressed={isListening}
title={isListening ? '停止聆听 (Ctrl+Shift+V)' : '语音输入 (Ctrl+Shift+V)'}
title={isListening ? '停止聆听 (Ctrl+Shift+V)' : isVADSupported ? '语音输入 (自动检测说话)' : '语音输入 (Ctrl+Shift+V)'}
className={`p-2 rounded-xl transition-all flex-shrink-0 border-2 ${
isListening
? 'voice-btn-active bg-red-500 border-red-500 text-white'
: 'bg-gray-100 dark:bg-gray-700 border-gray-200 dark:border-gray-600 text-gray-500 hover:text-red-500 hover:border-red-300'
: isVADSpeaking
? 'bg-yellow-400 border-yellow-400 text-white'
: 'bg-gray-100 dark:bg-gray-700 border-gray-200 dark:border-gray-600 text-gray-500 hover:text-red-500 hover:border-red-300'
} disabled:opacity-40 disabled:cursor-not-allowed`}
>
<svg
@@ -461,7 +497,7 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
)}
{/* 不支持时显示禁用按钮 */}
{!isSupported && (
{!isVADSupported && !isSRSpported && (
<button
disabled
title="您的浏览器不支持语音识别"
@@ -508,7 +544,10 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
{/* 语音输入状态提示 */}
{isListening && (
<p className="text-xs text-red-400 text-center animate-pulse">
{isFallbackMode ? '🎤 后端语音识别中...' : '🎤 正在聆听...'}
{isVADSupported
? (isVADSpeaking ? '🔊 检测到语音，正在识别...' : '🎤 正在聆听...')
: (isFallbackMode ? '🎤 后端语音识别中...' : '🎤 正在聆听...')
}
<span className="text-gray-400 ml-2">(Ctrl+Shift+V 停止)</span>
</p>
)}
frontend/web/src/hooks/useChat.ts
+8
View File
@@ -43,11 +43,19 @@ export function useChat() {
[addMessage, setTyping, sendMessage]
);
const sendVoiceStreamMessage = useCallback(
(msg: import('@/types/chat').WSClientMessage) => {
sendMessage(msg);
},
[sendMessage]
);
return {
messages,
isTyping,
isConnected,
send,
sendVoiceStreamMessage,
clearMessages,
};
}
frontend/web/src/hooks/useVoiceInput.ts
+221
View File
@@ -0,0 +1,221 @@
import { useState, useRef, useCallback, useEffect } from 'react';
import { MicVAD, utils } from '@ricky0123/vad-web';
import { transcribeAudio } from '@/api/voice';
import { useChatStore } from '@/store/chatStore';
import type { WSClientMessage } from '@/types/chat';
interface UseVoiceInputOptions {
onTranscription: (text: string) => void;
language?: string;
/** 提供后启用流式模式：通过 WebSocket 分片发送音频，实时返回中间结果 */
sendMessage?: (msg: WSClientMessage) => void;
}
interface UseVoiceInputReturn {
isListening: boolean;
isSpeaking: boolean;
isSupported: boolean;
/** 流式模式：voice_interim 中间识别文本 */
interimText: string;
/** 流式模式：voice_final 最终识别文本 */
finalText: string;
error: string | null;
startListening: () => Promise<void>;
stopListening: () => void;
}
export function useVoiceInput({
onTranscription,
language = 'zh',
sendMessage,
}: UseVoiceInputOptions): UseVoiceInputReturn {
const [isListening, setIsListening] = useState(false);
const [isSpeaking, setIsSpeaking] = useState(false);
const [error, setError] = useState<string | null>(null);
const vadRef = useRef<MicVAD | null>(null);
const inSpeechRef = useRef(false);
const seqRef = useRef(0);
const isSupported =
typeof window !== 'undefined' &&
typeof AudioWorkletNode !== 'undefined' &&
typeof WebAssembly !== 'undefined';
const isStreaming = !!sendMessage;
// Subscribe to streaming results from store
const voiceInterimText = useChatStore((s) => s.voiceInterimText);
const voiceFinalText = useChatStore((s) => s.voiceFinalText);
const stopListening = useCallback(() => {
if (vadRef.current) {
vadRef.current.pause();
}
inSpeechRef.current = false;
setIsListening(false);
setIsSpeaking(false);
}, []);
const startListening = useCallback(async () => {
if (!isSupported) return;
setError(null);
seqRef.current = 0;
try {
if (vadRef.current) {
await vadRef.current.destroy();
vadRef.current = null;
}
vadRef.current = await MicVAD.new({
onSpeechStart: () => {
setIsSpeaking(true);
inSpeechRef.current = true;
seqRef.current = 0;
if (isStreaming) {
sendMessage!({
type: 'voice_stream_start',
format: 'wav',
language,
timestamp: Date.now(),
});
}
},
onFrameProcessed: async (_probabilities, frame) => {
if (!isStreaming || !inSpeechRef.current) return;
// Accumulate audio frames during speech for the voice_stream_chunk
// Send every ~300ms of audio (10 frames at ~30ms each)
const frameSeq = seqRef.current++;
const CHUNK_INTERVAL = 10; // send every 10 frames
if (frameSeq % CHUNK_INTERVAL === 0) {
try {
const wavBuffer = utils.encodeWAV(frame);
const base64 = arrayBufferToBase64(wavBuffer);
sendMessage!({
type: 'voice_stream_chunk',
audio_data: base64,
sequence: Math.floor(frameSeq / CHUNK_INTERVAL),
timestamp: Date.now(),
});
} catch {
// Ignore encoding errors for individual chunks
}
}
},
onSpeechEnd: async (audio: Float32Array) => {
setIsSpeaking(false);
inSpeechRef.current = false;
if (isStreaming) {
// Send the last chunk with accumulated audio if any
try {
const wavBuffer = utils.encodeWAV(audio);
const base64 = arrayBufferToBase64(wavBuffer);
sendMessage!({
type: 'voice_stream_chunk',
audio_data: base64,
sequence: -1,
timestamp: Date.now(),
});
} catch {
// Ignore
}
// Signal end of voice stream — gateway returns voice_final
sendMessage!({
type: 'voice_stream_end',
timestamp: Date.now(),
});
// onTranscription will be called in useEffect when voiceFinalText updates
return;
}
// REST mode: send full audio for transcription
const wavBuffer = utils.encodeWAV(audio);
const wavBlob = new Blob([wavBuffer], { type: 'audio/wav' });
try {
const result = await transcribeAudio(wavBlob, language);
if (result.error) {
setError(result.error);
} else if (result.data?.text) {
onTranscription(result.data.text);
}
} catch (err) {
setError(err instanceof Error ? err.message : '语音识别失败');
}
},
onVADMisfire: () => {
setIsSpeaking(false);
inSpeechRef.current = false;
if (isStreaming) {
sendMessage!({
type: 'voice_stream_end',
timestamp: Date.now(),
});
}
},
startOnLoad: true,
});
setIsListening(true);
} catch (err) {
const message =
err instanceof DOMException && err.name === 'NotAllowedError'
? '麦克风权限被拒绝'
: err instanceof Error
? err.message
: 'VAD 初始化失败';
setError(message);
}
}, [isSupported, language, isStreaming, sendMessage, onTranscription]);
// In streaming mode, watch for voice_final from the server
useEffect(() => {
if (isStreaming && voiceFinalText) {
onTranscription(voiceFinalText);
useChatStore.getState().setVoiceFinalText('');
}
}, [isStreaming, voiceFinalText, onTranscription]);
useEffect(() => {
return () => {
if (vadRef.current) {
vadRef.current.destroy();
vadRef.current = null;
}
};
}, []);
return {
isListening,
isSpeaking,
isSupported,
interimText: isStreaming ? voiceInterimText : '',
finalText: isStreaming ? voiceFinalText : '',
error,
startListening,
stopListening,
};
}
function arrayBufferToBase64(buffer: ArrayBuffer): string {
const bytes = new Uint8Array(buffer);
let binary = '';
for (let i = 0; i < bytes.byteLength; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary);
}
frontend/web/src/hooks/useWebSocket.ts
+13 -1
View File
@@ -254,7 +254,7 @@ function handleServerMessage(msg: WSServerMessage) {
client_info: msg.client_info,
audioUrl: msg.full_audio_url,
segments: msg.segments,
metadata: msg.tool_calls ? { tool_calls: msg.tool_calls } : undefined,
metadata: msg.metadata || (msg.tool_calls ? { tool_calls: msg.tool_calls } : undefined),
});
}
setTyping(false);
@@ -462,6 +462,18 @@ function handleServerMessage(msg: WSServerMessage) {
}
break;
case 'voice_interim':
if (msg.text !== undefined) {
useChatStore.getState().setVoiceInterimText(msg.text);
}
break;
case 'voice_final':
if (msg.text !== undefined) {
useChatStore.getState().setVoiceFinalText(msg.text);
}
break;
case 'pong':
break;
frontend/web/src/store/chatStore.ts
+11
View File
@@ -21,6 +21,10 @@ interface ChatStore {
isLoadingHistory: boolean;
historyPage: number;
// 流式语音识别状态
voiceInterimText: string;
voiceFinalText: string;
// 多气泡消息队列：确保气泡依次出现 + 逐字动画
messageQueue: Message[];
@@ -37,6 +41,8 @@ interface ChatStore {
clearMessages: () => void;
setContinuousMode: (enabled: boolean) => void;
setVoiceInterimText: (text: string) => void;
setVoiceFinalText: (text: string) => void;
setBackgroundThinkingStatus: (status: BackgroundThinkingStatus) => void;
setIoTDevices: (devices: IoTDevice[]) => void;
@@ -54,6 +60,8 @@ export const useChatStore = create<ChatStore>((set) => ({
backgroundThinkingStatus: 'idle',
iotDevices: [],
iotDevicesLastUpdated: null,
voiceInterimText: '',
voiceFinalText: '',
hasMoreMessages: false,
isLoadingHistory: false,
historyPage: 1,
@@ -143,6 +151,9 @@ export const useChatStore = create<ChatStore>((set) => ({
setContinuousMode: (enabled) => set({ continuousMode: enabled }),
setVoiceInterimText: (text) => set({ voiceInterimText: text }),
setVoiceFinalText: (text) => set({ voiceFinalText: text, voiceInterimText: '' }),
setBackgroundThinkingStatus: (status) => set({ backgroundThinkingStatus: status }),
setIoTDevices: (devices) =>
frontend/web/src/store/pageStore.ts
+17
View File
@@ -0,0 +1,17 @@
// 页面路由 Store — 管理当前显示的页面
import { create } from 'zustand';
export type PageId = 'chat' | 'admin-models' | 'admin-dashboard' | 'profile';
interface PageState {
currentPage: PageId;
setPage: (page: PageId) => void;
goToChat: () => void;
}
export const usePageStore = create<PageState>((set) => ({
currentPage: 'chat',
setPage: (page) => set({ currentPage: page }),
goToChat: () => set({ currentPage: 'chat' }),
}));
frontend/web/src/types/chat.ts
+6 -2
View File
@@ -105,11 +105,14 @@ export interface StreamSegment {
/** WebSocket 客户端消息 */
export interface WSClientMessage {
type: 'message' | 'voice_input' | 'ping' | 'history';
type: 'message' | 'voice_input' | 'voice_stream_start' | 'voice_stream_chunk' | 'voice_stream_end' | 'ping' | 'history';
session_id?: string;
mode?: ChatMode;
content?: string;
audio_data?: string; // base64
format?: string; // 音频格式 (voice_stream_start): webm, wav, pcm, opus
language?: string; // 识别语言 (voice_stream_start): zh, en, ja, ko, auto
sequence?: number; // 音频分片序号 (voice_stream_chunk)
attachments?: MessageAttachment[];
timestamp: number;
client_id?: string;
@@ -139,7 +142,7 @@ export interface AppNotification extends NotificationData {
/** WebSocket 服务端消息 */
export interface WSServerMessage {
type: 'stream_start' | 'response' | 'segment' | 'audio' | 'error' | 'device_update' | 'pong' | 'history_response' | 'stream_chunk' | 'stream_end' | 'background_thinking' | 'notification' | 'multi_message' | 'stream_segments' | 'review' | 'thinking' | 'tool_progress' | 'system_info';
type: 'stream_start' | 'response' | 'segment' | 'audio' | 'error' | 'device_update' | 'pong' | 'history_response' | 'stream_chunk' | 'stream_end' | 'background_thinking' | 'notification' | 'multi_message' | 'stream_segments' | 'review' | 'thinking' | 'tool_progress' | 'system_info' | 'voice_interim' | 'voice_final';
message_id?: string;
text?: string;
content?: string;
@@ -161,6 +164,7 @@ export interface WSServerMessage {
notification?: NotificationData;
tool_progress?: ToolProgressInfo;
system_info?: SystemInfoPayload;
metadata?: Record<string, unknown>;
protocol_version?: number;
timestamp: number;
client_info?: ClientInfo;