feat: 语音流式输入管线 + VAD前端集成 + 插件-工具合并清理
- 前端: VAD语音检测(@ricky0123/vad-web) + useVoiceInput双模式(流式WS/REST) - Gateway: VoiceStreamManager代理WS流式STT到voice-service - Voice-service: DashScope REST → Realtime WS → Whisper三级引擎 + ffmpeg转码 - 共享模块: pkg/audio(音频转换) + pkg/dashscope(ASR REST客户端) - 清理: 移除旧plugin-manager和pkg/plugins,完成插件→工具合并 - 文档: 完善gateway-api.md和voice-service.md语音API文档 - 工具: scripts/voice/ 语音转换脚本集 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -15,11 +15,14 @@ type MessageAttachment struct {
|
||||
|
||||
// 客户端 → 服务端消息
|
||||
type ClientMessage struct {
|
||||
Type string `json:"type"` // message | voice_input | ping | history
|
||||
Type string `json:"type"` // message | voice_input | voice_stream_start | voice_stream_chunk | voice_stream_end | ping | history
|
||||
SessionID string `json:"session_id"`
|
||||
Mode string `json:"mode"` // text | voice_msg | voice_assistant
|
||||
Content string `json:"content"`
|
||||
AudioData string `json:"audio_data,omitempty"` // base64
|
||||
Format string `json:"format,omitempty"` // 音频格式: webm, wav, pcm, opus
|
||||
Language string `json:"language,omitempty"` // 识别语言: zh, en, ja, ko, auto
|
||||
Sequence int `json:"sequence,omitempty"` // 音频块序列号 (voice_stream_chunk)
|
||||
Attachments []MessageAttachment `json:"attachments,omitempty"` // 图片等附件
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
ClientID string `json:"client_id,omitempty"` // 客户端唯一标识 (多端区分)
|
||||
@@ -28,11 +31,12 @@ type ClientMessage struct {
|
||||
ClientMsgID string `json:"client_msg_id,omitempty"` // 客户端消息ID (跨端去重)
|
||||
}
|
||||
|
||||
// ReviewMessage 审查后的结构化消息(动作/聊天分离)
|
||||
// ReviewMessage 审查后的结构化消息(动作/聊天/Markdown/代码块/搜索结果)
|
||||
type ReviewMessage struct {
|
||||
Type string `json:"type"` // "action" | "chat"
|
||||
Content string `json:"content"`
|
||||
DelayMs int `json:"delay_ms,omitempty"` // ms to wait before sending (0 = immediate)
|
||||
Type string `json:"type"` // action | chat | markdown | code | search_result
|
||||
Content string `json:"content"`
|
||||
DelayMs int `json:"delay_ms,omitempty"` // ms to wait before sending (0 = immediate)
|
||||
Metadata map[string]any `json:"metadata,omitempty"` // 类型特定元数据 (code 语言、搜索结果 URL 等)
|
||||
}
|
||||
|
||||
// ClientInfo carries the originating client's device metadata.
|
||||
@@ -44,7 +48,7 @@ type ClientInfo struct {
|
||||
|
||||
// 服务端 → 客户端消息
|
||||
type ServerMessage struct {
|
||||
Type string `json:"type"` // response | segment | audio | error | device_update | pong | history_response | stream_chunk | stream_end | background_thinking | notification | multi_message | stream_segments | review | thinking | tool_progress | system_info
|
||||
Type string `json:"type"` // response | segment | audio | error | device_update | pong | history_response | stream_chunk | stream_end | background_thinking | notification | multi_message | stream_segments | review | thinking | tool_progress | system_info | voice_interim | voice_final
|
||||
MessageID string `json:"message_id"`
|
||||
Text string `json:"text,omitempty"`
|
||||
Content string `json:"content,omitempty"` // stream_chunk 的增量文本
|
||||
@@ -63,7 +67,8 @@ type ServerMessage struct {
|
||||
Notification *NotificationInfo `json:"notification,omitempty"` // 通知推送
|
||||
MultiMessage *MultiMessagePayload `json:"multi_message,omitempty"` // 多条消息批量发送
|
||||
ReviewMessages []ReviewMessage `json:"review_messages,omitempty"` // 审查后的结构化消息列表
|
||||
MsgType string `json:"msg_type,omitempty"` // 消息展示类型: action | chat | thinking | tool_progress | system_info
|
||||
MsgType string `json:"msg_type,omitempty"` // 消息展示类型: action | chat | thinking | tool_progress | system_info | markdown | code | search_result
|
||||
Metadata map[string]any `json:"metadata,omitempty"` // 消息元数据 (code 语言等)
|
||||
ToolProgress *ToolProgressInfo `json:"tool_progress,omitempty"` // 工具执行进度
|
||||
SystemInfo *SystemInfoPayload `json:"system_info,omitempty"` // 系统通知信息
|
||||
ProtocolVersion int `json:"protocol_version,omitempty"` // 协议版本
|
||||
|
||||
Reference in New Issue
Block a user