6ef9e082a6
- 前端: VAD语音检测(@ricky0123/vad-web) + useVoiceInput双模式(流式WS/REST) - Gateway: VoiceStreamManager代理WS流式STT到voice-service - Voice-service: DashScope REST → Realtime WS → Whisper三级引擎 + ffmpeg转码 - 共享模块: pkg/audio(音频转换) + pkg/dashscope(ASR REST客户端) - 清理: 移除旧plugin-manager和pkg/plugins,完成插件→工具合并 - 文档: 完善gateway-api.md和voice-service.md语音API文档 - 工具: scripts/voice/ 语音转换脚本集 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1208 lines
33 KiB
Go
1208 lines
33 KiB
Go
package handler
|
||
|
||
import (
|
||
"bufio"
|
||
"bytes"
|
||
"crypto/rand"
|
||
"encoding/base64"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"mime/multipart"
|
||
"net/http"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/gin-gonic/gin"
|
||
"github.com/gorilla/websocket"
|
||
|
||
"git.yeij.top/AskaEth/Cyrene/gateway/internal/config"
|
||
"git.yeij.top/AskaEth/Cyrene/gateway/internal/store"
|
||
"git.yeij.top/AskaEth/Cyrene/gateway/internal/ws"
|
||
"git.yeij.top/AskaEth/Cyrene/pkg/logger"
|
||
)
|
||
|
||
// queuedMsg 队列中的待处理消息
|
||
type queuedMsg struct {
|
||
client *ws.Client
|
||
mode string
|
||
reqBody []byte
|
||
content string
|
||
}
|
||
|
||
// ChatHandler 聊天处理器
|
||
type ChatHandler struct {
|
||
cfg *config.Config
|
||
hub *ws.Hub
|
||
sessionStore *store.SessionStore
|
||
fileStore *store.FileStore
|
||
voiceStream *VoiceStreamManager
|
||
upgrader websocket.Upgrader
|
||
pending map[string][]queuedMsg // per-session message queue
|
||
pendingMu sync.Mutex
|
||
}
|
||
|
||
// NewChatHandler 创建聊天处理器
|
||
func NewChatHandler(cfg *config.Config, hub *ws.Hub, sessionStore *store.SessionStore, fileStore *store.FileStore) *ChatHandler {
|
||
return &ChatHandler{
|
||
cfg: cfg,
|
||
hub: hub,
|
||
sessionStore: sessionStore,
|
||
fileStore: fileStore,
|
||
voiceStream: NewVoiceStreamManager(cfg.VoiceServiceURL),
|
||
pending: make(map[string][]queuedMsg),
|
||
upgrader: websocket.Upgrader{
|
||
ReadBufferSize: 1024,
|
||
WriteBufferSize: 1024,
|
||
CheckOrigin: func(r *http.Request) bool {
|
||
return true // 开发阶段允许所有来源
|
||
},
|
||
},
|
||
}
|
||
}
|
||
|
||
// HandleWebSocket 处理WebSocket升级和消息路由
|
||
func (h *ChatHandler) HandleWebSocket(c *gin.Context) {
|
||
// 从query参数获取token和session_id
|
||
token := c.Query("token")
|
||
sessionID := c.Query("session_id")
|
||
clientID := c.Query("client_id")
|
||
deviceName := c.Query("device_name")
|
||
userAgent := c.Request.UserAgent()
|
||
|
||
if token == "" {
|
||
// 也尝试从Authorization头读取
|
||
authHeader := c.GetHeader("Authorization")
|
||
if len(authHeader) > 7 && authHeader[:7] == "Bearer " {
|
||
token = authHeader[7:]
|
||
}
|
||
}
|
||
|
||
if token == "" {
|
||
c.JSON(http.StatusUnauthorized, gin.H{"error": "需要认证令牌"})
|
||
return
|
||
}
|
||
|
||
// 验证token
|
||
userID, err := h.cfg.ValidateToken(token)
|
||
if err != nil {
|
||
c.JSON(http.StatusUnauthorized, gin.H{"error": "认证令牌无效"})
|
||
return
|
||
}
|
||
|
||
// 主对话仅限管理员访问
|
||
if userID != "admin" {
|
||
c.JSON(http.StatusForbidden, gin.H{
|
||
"error": "主对话仅限管理员使用",
|
||
"errorType": "admin_only",
|
||
"hint": "请使用管理员账号登录以访问主对话功能",
|
||
})
|
||
return
|
||
}
|
||
|
||
if sessionID == "" {
|
||
sessionID = "session_" + generateID()
|
||
}
|
||
|
||
// 升级WebSocket连接
|
||
conn, err := h.upgrader.Upgrade(c.Writer, c.Request, nil)
|
||
if err != nil {
|
||
logger.Printf("[WS] 升级连接失败: %v", err)
|
||
return
|
||
}
|
||
|
||
// 创建客户端
|
||
client := ws.NewClient(h.hub, conn, userID, sessionID, clientID, deviceName, userAgent)
|
||
|
||
// 注册到Hub
|
||
h.hub.Register(client)
|
||
|
||
// 启动读写协程
|
||
go client.WritePump()
|
||
go client.ReadPump(func(client *ws.Client, msg ws.ClientMessage) {
|
||
h.handleMessage(client, msg)
|
||
})
|
||
}
|
||
|
||
// handleMessage 处理WebSocket消息
|
||
func (h *ChatHandler) handleMessage(client *ws.Client, msg ws.ClientMessage) {
|
||
switch msg.Type {
|
||
case "message":
|
||
h.handleChatMessage(client, msg)
|
||
case "voice_input":
|
||
h.handleVoiceInput(client, msg)
|
||
case "voice_stream_start":
|
||
h.handleVoiceStreamStart(client, msg)
|
||
case "voice_stream_chunk":
|
||
h.handleVoiceStreamChunk(client, msg)
|
||
case "voice_stream_end":
|
||
h.handleVoiceStreamEnd(client, msg)
|
||
case "history":
|
||
h.handleHistoryRequest(client, msg)
|
||
default:
|
||
logger.Printf("[WS] 未知消息类型: %s from user=%s", msg.Type, client.UserID)
|
||
}
|
||
}
|
||
|
||
// handleChatMessage 处理文字聊天消息 - 转发到 AI-Core(流式发送)
|
||
func (h *ChatHandler) handleChatMessage(client *ws.Client, msg ws.ClientMessage) {
|
||
mode := msg.Mode
|
||
if mode == "" {
|
||
mode = "text"
|
||
}
|
||
|
||
// 持久化用户消息到数据库(在 WebSocket 发送之前)
|
||
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
|
||
if err := h.sessionStore.AddMessage(client.SessionID, "user", "chat", msg.Content, client.ClientID); err != nil {
|
||
logger.Printf("[chat] 持久化用户消息失败: %v", err)
|
||
}
|
||
}
|
||
|
||
// 记录用户消息
|
||
h.hub.RecordMessage(client.SessionID, "user", msg.Content)
|
||
|
||
// 设置会话状态为 thinking
|
||
h.hub.UpdateSessionState(client.SessionID, "thinking")
|
||
|
||
// 构建 AI-Core 请求
|
||
aiReq := map[string]interface{}{
|
||
"user_id": client.UserID,
|
||
"session_id": client.SessionID,
|
||
"message": msg.Content,
|
||
"mode": mode,
|
||
}
|
||
if len(msg.Attachments) > 0 {
|
||
images := make([]string, 0, len(msg.Attachments))
|
||
for _, att := range msg.Attachments {
|
||
if att.Type == "image" {
|
||
imgURL := ""
|
||
// 优先使用 file_id 引用(轻量,支持跨设备同步)
|
||
if att.FileID != "" && h.fileStore != nil {
|
||
if f, err := h.fileStore.GetFile(att.FileID); err == nil {
|
||
imgURL = "http://127.0.0.1:" + h.cfg.Port + "/api/v1/files/" + f.ID + "/download"
|
||
} else {
|
||
logger.Printf("[chat] file_id 解析失败: %s, err=%v", att.FileID, err)
|
||
}
|
||
}
|
||
// 回退: 使用 URL(兼容旧客户端 base64 data URI)
|
||
if imgURL == "" && att.URL != "" {
|
||
imgURL = att.URL
|
||
if strings.HasPrefix(imgURL, "/") {
|
||
imgURL = "http://127.0.0.1:" + h.cfg.Port + imgURL
|
||
}
|
||
}
|
||
if imgURL != "" {
|
||
images = append(images, imgURL)
|
||
}
|
||
}
|
||
}
|
||
if len(images) > 0 {
|
||
aiReq["images"] = images
|
||
}
|
||
}
|
||
reqBody, err := json.Marshal(aiReq)
|
||
if err != nil {
|
||
logger.Printf("[chat] 序列化请求失败: %v", err)
|
||
h.hub.UpdateSessionState(client.SessionID, "error")
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "error",
|
||
MessageID: "msg_" + generateID(),
|
||
Error: "内部错误,请稍后重试",
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
|
||
// 缓存用户消息(在 goroutine 前完成,避免竞态)
|
||
userMsgID := msg.ClientMsgID
|
||
if userMsgID == "" {
|
||
userMsgID = "msg_" + generateID()
|
||
}
|
||
userMsg := ws.Message{
|
||
ID: userMsgID,
|
||
Role: "user",
|
||
Content: msg.Content,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
ClientInfo: &ws.ClientInfo{
|
||
ClientID: client.ClientID,
|
||
DeviceName: client.DeviceName,
|
||
},
|
||
}
|
||
if len(msg.Attachments) > 0 {
|
||
userMsg.Attachments = msg.Attachments
|
||
}
|
||
h.hub.CacheMessage(client.UserID, client.SessionID, userMsg)
|
||
|
||
// 广播用户消息给同用户其他设备(跨端同步,排除发送者自身)
|
||
h.broadcastToUserExcept(client.UserID, client.ClientID, ws.ServerMessage{
|
||
Type: "response",
|
||
MessageID: userMsgID,
|
||
Content: msg.Content,
|
||
Role: "user",
|
||
MsgType: "chat",
|
||
SessionID: client.SessionID,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
ClientInfo: &ws.ClientInfo{
|
||
ClientID: client.ClientID,
|
||
DeviceName: client.DeviceName,
|
||
},
|
||
})
|
||
|
||
// 排队处理:同一会话的消息串行化,避免并发请求导致上下文竞争
|
||
h.enqueueOrProcess(client, mode, reqBody, msg.Content)
|
||
}
|
||
|
||
// enqueueOrProcess 将消息加入 per-session 队列,若会话空闲则立即处理
|
||
func (h *ChatHandler) enqueueOrProcess(client *ws.Client, mode string, reqBody []byte, content string) {
|
||
h.pendingMu.Lock()
|
||
|
||
if queue, busy := h.pending[client.SessionID]; busy {
|
||
// 会话正在处理中,加入队列
|
||
h.pending[client.SessionID] = append(queue, queuedMsg{
|
||
client: client, mode: mode, reqBody: reqBody, content: content,
|
||
})
|
||
queueLen := len(h.pending[client.SessionID])
|
||
h.pendingMu.Unlock()
|
||
|
||
logger.Printf("[chat] 会话 %s 正在处理中,消息已加入队列 (位置 %d)", client.SessionID, queueLen)
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "queued",
|
||
SessionID: client.SessionID,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
|
||
// 标记为处理中
|
||
h.pending[client.SessionID] = nil
|
||
h.pendingMu.Unlock()
|
||
|
||
go h.processQueue(client, mode, reqBody, content)
|
||
}
|
||
|
||
// processQueue 处理当前消息,完成后自动消费队列中的下一条
|
||
func (h *ChatHandler) processQueue(client *ws.Client, mode string, reqBody []byte, content string) {
|
||
h.streamResponse(client, mode, reqBody, content)
|
||
|
||
// 处理队列中的后续消息
|
||
for {
|
||
h.pendingMu.Lock()
|
||
queue := h.pending[client.SessionID]
|
||
if len(queue) == 0 {
|
||
delete(h.pending, client.SessionID)
|
||
h.pendingMu.Unlock()
|
||
return
|
||
}
|
||
next := queue[0]
|
||
h.pending[client.SessionID] = queue[1:]
|
||
h.pendingMu.Unlock()
|
||
|
||
logger.Printf("[chat] 会话 %s 从队列取出消息继续处理 (剩余 %d)", client.SessionID, len(queue)-1)
|
||
h.streamResponse(next.client, next.mode, next.reqBody, next.content)
|
||
}
|
||
}
|
||
|
||
// streamResponse 调用 AI-Core SSE 流式接口并逐 delta 转发给客户端
|
||
func (h *ChatHandler) streamResponse(client *ws.Client, mode string, reqBody []byte, userMsg string) {
|
||
normalExit := false
|
||
defer func() {
|
||
if !normalExit {
|
||
h.broadcastToUser(client.UserID, ws.ServerMessage{
|
||
Type: "stream_end",
|
||
MessageID: "msg_" + generateID(),
|
||
SessionID: client.SessionID,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
if h.hub != nil {
|
||
h.hub.UpdateSessionState(client.SessionID, "idle")
|
||
}
|
||
}
|
||
}()
|
||
|
||
aiCoreURL := h.cfg.AICoreURL + "/api/v1/chat"
|
||
httpReq, err := http.NewRequest("POST", aiCoreURL, bytes.NewReader(reqBody))
|
||
if err != nil {
|
||
logger.Printf("[chat] 创建 AI-Core 请求失败: %v", err)
|
||
h.hub.UpdateSessionState(client.SessionID, "error")
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "error",
|
||
MessageID: "msg_" + generateID(),
|
||
Error: "服务暂不可用",
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
httpReq.Header.Set("Content-Type", "application/json")
|
||
httpReq.Header.Set("Accept", "text/event-stream")
|
||
|
||
httpClient := &http.Client{Timeout: 120 * time.Second}
|
||
resp, err := httpClient.Do(httpReq)
|
||
if err != nil {
|
||
logger.Printf("[chat] AI-Core 调用失败: %v", err)
|
||
h.hub.UpdateSessionState(client.SessionID, "error")
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "error",
|
||
MessageID: "msg_" + generateID(),
|
||
Error: fmt.Sprintf("AI-Core 调用失败: %v", err),
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
if resp.StatusCode != http.StatusOK {
|
||
body, _ := io.ReadAll(resp.Body)
|
||
logger.Printf("[chat] AI-Core 返回错误 [%d]: %s", resp.StatusCode, string(body))
|
||
h.hub.UpdateSessionState(client.SessionID, "error")
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "error",
|
||
MessageID: "msg_" + generateID(),
|
||
Error: fmt.Sprintf("AI-Core 错误 (%d)", resp.StatusCode),
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
|
||
// 使用 bufio.Scanner 逐行读取 SSE 响应
|
||
scanner := bufio.NewScanner(resp.Body)
|
||
// 增大 scanner buffer 以处理大块 SSE 数据
|
||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||
|
||
// 通知所有客户端 AI 开始生成回复
|
||
h.broadcastToUser(client.UserID, ws.ServerMessage{
|
||
Type: "stream_start",
|
||
MessageID: "msg_" + generateID(),
|
||
SessionID: client.SessionID,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
|
||
var fullText string
|
||
var msgID string
|
||
var hasReview bool // 是否有审查消息(避免重复持久化)
|
||
var segments []ws.VoiceSegment // 收集断句信息
|
||
|
||
for scanner.Scan() {
|
||
line := scanner.Text()
|
||
|
||
// 跳过非 data 行
|
||
if !strings.HasPrefix(line, "data: ") {
|
||
continue
|
||
}
|
||
|
||
data := strings.TrimPrefix(line, "data: ")
|
||
|
||
// SSE 流结束标记
|
||
if data == "[DONE]" {
|
||
break
|
||
}
|
||
|
||
// 解析 delta 数据
|
||
var chunk struct {
|
||
Delta string `json:"delta"`
|
||
Error string `json:"error,omitempty"`
|
||
MessageID string `json:"message_id,omitempty"`
|
||
Mode string `json:"mode,omitempty"`
|
||
Done bool `json:"done,omitempty"`
|
||
// 断句相关 (来自 AI-Core 新格式)
|
||
Segments []struct {
|
||
Index int `json:"index"`
|
||
Text string `json:"text"`
|
||
} `json:"segments,omitempty"`
|
||
// 审查后的结构化消息
|
||
ReviewMessages []ws.ReviewMessage `json:"review_messages,omitempty"`
|
||
}
|
||
if err := json.Unmarshal([]byte(data), &chunk); err != nil {
|
||
logger.Printf("[chat] 解析 SSE delta 失败: %v, raw=%s", err, data)
|
||
continue
|
||
}
|
||
|
||
// 错误处理
|
||
if chunk.Error != "" {
|
||
logger.Printf("[chat] AI-Core 流式错误: %s", chunk.Error)
|
||
h.hub.UpdateSessionState(client.SessionID, "error")
|
||
h.broadcastToUser(client.UserID, ws.ServerMessage{
|
||
Type: "error",
|
||
MessageID: "msg_" + generateID(),
|
||
Error: chunk.Error,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
|
||
// 记录 message_id
|
||
if chunk.MessageID != "" {
|
||
msgID = chunk.MessageID
|
||
}
|
||
|
||
// 如果是结束标记(含 done: true),跳出
|
||
if chunk.Done {
|
||
break
|
||
}
|
||
|
||
// 处理审查后的结构化消息 (review)
|
||
if len(chunk.ReviewMessages) > 0 {
|
||
for i, rm := range chunk.ReviewMessages {
|
||
msgType := rm.Type
|
||
if msgType == "" {
|
||
msgType = "chat"
|
||
}
|
||
role := "assistant"
|
||
if msgType == "action" {
|
||
role = "action"
|
||
}
|
||
reviewMsgID := fmt.Sprintf("%s_r%d", msgID, i)
|
||
// 持久化每条审查消息 (action 角色映射为 assistant,LLM 模型不支持自定义角色)
|
||
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
|
||
dbRole := role
|
||
if dbRole == "action" {
|
||
dbRole = "assistant"
|
||
}
|
||
if err := h.sessionStore.AddMessage(client.SessionID, dbRole, msgType, rm.Content, client.ClientID); err != nil {
|
||
logger.Printf("[chat] 持久化审查消息失败: %v", err)
|
||
}
|
||
}
|
||
clientInfo := &ws.ClientInfo{
|
||
ClientID: client.ClientID,
|
||
DeviceName: client.DeviceName,
|
||
}
|
||
h.hub.CacheMessage(client.UserID, client.SessionID, ws.Message{
|
||
ID: reviewMsgID,
|
||
Role: role,
|
||
Content: rm.Content,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
ClientInfo: clientInfo,
|
||
})
|
||
h.broadcastToUser(client.UserID, ws.ServerMessage{
|
||
Type: "response",
|
||
MessageID: reviewMsgID,
|
||
Content: rm.Content,
|
||
Role: role,
|
||
MsgType: msgType,
|
||
SessionID: client.SessionID,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
ClientInfo: clientInfo,
|
||
Metadata: rm.Metadata,
|
||
})
|
||
// 使用 MessageScheduler 计算的 per-message 延迟
|
||
if rm.DelayMs > 0 {
|
||
time.Sleep(time.Duration(rm.DelayMs) * time.Millisecond)
|
||
}
|
||
}
|
||
hasReview = true
|
||
continue
|
||
}
|
||
|
||
// 处理断句事件 (stream_segments)
|
||
if len(chunk.Segments) > 0 {
|
||
for _, seg := range chunk.Segments {
|
||
segments = append(segments, ws.VoiceSegment{
|
||
Index: seg.Index,
|
||
Text: seg.Text,
|
||
})
|
||
}
|
||
// 发送断句事件给所有客户端
|
||
h.broadcastToUser(client.UserID, ws.ServerMessage{
|
||
Type: "stream_segments",
|
||
MessageID: msgID,
|
||
Segments: segments,
|
||
SessionID: client.SessionID,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
continue
|
||
}
|
||
|
||
// 逐 delta 积累(不再逐块转发,由审查消息代替)
|
||
if chunk.Delta != "" {
|
||
fullText += chunk.Delta
|
||
}
|
||
}
|
||
|
||
if err := scanner.Err(); err != nil {
|
||
logger.Printf("[chat] SSE 读取错误: %v", err)
|
||
h.hub.UpdateSessionState(client.SessionID, "error")
|
||
h.broadcastToUser(client.UserID, ws.ServerMessage{
|
||
Type: "error",
|
||
MessageID: "msg_" + generateID(),
|
||
Error: fmt.Sprintf("流读取错误: %v", err),
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
|
||
if msgID == "" {
|
||
msgID = "msg_" + generateID()
|
||
}
|
||
|
||
// 检测是否为多消息格式(包含空行分隔的多条消息)
|
||
// 如果已有审查消息则跳过,避免与 review_messages 重复
|
||
multiParts := parseMultiMessage(fullText)
|
||
if !hasReview && len(multiParts) > 1 {
|
||
// 发送 multi_message 事件(每条消息带 msg_type)
|
||
var items []ws.MultiMessageItem
|
||
for i, seg := range multiParts {
|
||
items = append(items, ws.MultiMessageItem{
|
||
Index: i,
|
||
Content: seg.content,
|
||
MsgType: seg.msgType,
|
||
})
|
||
}
|
||
h.broadcastToUser(client.UserID, ws.ServerMessage{
|
||
Type: "multi_message",
|
||
MessageID: msgID,
|
||
SessionID: client.SessionID,
|
||
MultiMessage: &ws.MultiMessagePayload{
|
||
Messages: items,
|
||
},
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
}
|
||
|
||
// 发送 stream_end 到所有客户端
|
||
h.broadcastToUser(client.UserID, ws.ServerMessage{
|
||
Type: "stream_end",
|
||
MessageID: msgID,
|
||
SessionID: client.SessionID,
|
||
Content: fullText,
|
||
Text: fullText,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
|
||
// 持久化 AI 回复到数据库
|
||
// 如果有审查消息,每条已单独持久化,跳过 fullText 以避免重复
|
||
if !hasReview && fullText != "" {
|
||
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
|
||
if err := h.sessionStore.AddMessage(client.SessionID, "assistant", "chat", fullText, client.ClientID); err != nil {
|
||
logger.Printf("[chat] 持久化 AI 回复失败: %v", err)
|
||
}
|
||
}
|
||
|
||
h.hub.CacheMessage(client.UserID, client.SessionID, ws.Message{
|
||
ID: msgID,
|
||
Role: "assistant",
|
||
Content: fullText,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
ClientInfo: &ws.ClientInfo{
|
||
ClientID: client.ClientID,
|
||
DeviceName: client.DeviceName,
|
||
},
|
||
})
|
||
}
|
||
// RecordMessage 使用不带 [review] 标记的文本
|
||
recordText := fullText
|
||
if hasReview {
|
||
recordText = strings.ReplaceAll(fullText, "[review]", "")
|
||
}
|
||
h.hub.RecordMessage(client.SessionID, "assistant", recordText)
|
||
|
||
// 设置会话状态为 idle
|
||
normalExit = true
|
||
h.hub.UpdateSessionState(client.SessionID, "idle")
|
||
}
|
||
|
||
// handleVoiceInput 处理语音输入
|
||
func (h *ChatHandler) handleVoiceInput(client *ws.Client, msg ws.ClientMessage) {
|
||
audioB64 := msg.AudioData
|
||
if audioB64 == "" {
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "error",
|
||
MessageID: "msg_" + generateID(),
|
||
Error: "语音数据为空",
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
|
||
format := msg.Mode
|
||
if format == "" {
|
||
format = "webm"
|
||
}
|
||
|
||
// 在 goroutine 中处理转录,避免阻塞 ReadPump
|
||
go func() {
|
||
text, err := h.transcribeAudio(audioB64, format)
|
||
if err != nil {
|
||
logger.Printf("[voice] 转录失败: %v", err)
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "voice_transcript",
|
||
MessageID: "msg_" + generateID(),
|
||
Error: fmt.Sprintf("语音识别失败: %v", err),
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
|
||
if text == "" {
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "voice_transcript",
|
||
MessageID: "msg_" + generateID(),
|
||
Text: "",
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
|
||
// 发送转录结果给前端
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "voice_transcript",
|
||
MessageID: "msg_" + generateID(),
|
||
Text: text,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
|
||
// 将转录文本作为聊天消息处理
|
||
chatMsg := ws.ClientMessage{
|
||
Type: "message",
|
||
Content: text,
|
||
Mode: msg.Mode,
|
||
}
|
||
h.handleChatMessage(client, chatMsg)
|
||
}()
|
||
}
|
||
|
||
// handleVoiceStreamStart begins a streaming voice session via voice-service.
|
||
func (h *ChatHandler) handleVoiceStreamStart(client *ws.Client, msg ws.ClientMessage) {
|
||
format := msg.Format
|
||
if format == "" {
|
||
format = "webm"
|
||
}
|
||
language := msg.Language
|
||
if language == "" {
|
||
language = "zh"
|
||
}
|
||
|
||
if err := h.voiceStream.StartStream(client, format, language); err != nil {
|
||
logger.Printf("[voice-stream] 启动流式 STT 失败: %v", err)
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "error",
|
||
MessageID: "msg_" + generateID(),
|
||
Error: "启动语音流失败: " + err.Error(),
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "voice_interim",
|
||
MessageID: "voice_" + generateID(),
|
||
Text: "",
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
}
|
||
|
||
// handleVoiceStreamChunk forwards an audio chunk to the active voice stream.
|
||
func (h *ChatHandler) handleVoiceStreamChunk(client *ws.Client, msg ws.ClientMessage) {
|
||
if msg.AudioData == "" {
|
||
return
|
||
}
|
||
|
||
audioData, err := decodeBase64(msg.AudioData)
|
||
if err != nil {
|
||
logger.Printf("[voice-stream] 解码音频块失败: %v", err)
|
||
return
|
||
}
|
||
|
||
if err := h.voiceStream.SendChunk(client.ClientID, client.SessionID, audioData, msg.Sequence); err != nil {
|
||
logger.Printf("[voice-stream] 发送音频块失败: %v", err)
|
||
}
|
||
}
|
||
|
||
// handleVoiceStreamEnd stops the voice stream and processes the final transcription.
|
||
func (h *ChatHandler) handleVoiceStreamEnd(client *ws.Client, msg ws.ClientMessage) {
|
||
go func() {
|
||
text, err := h.voiceStream.EndStream(client.ClientID, client.SessionID)
|
||
if err != nil {
|
||
logger.Printf("[voice-stream] 结束流式 STT 失败: %v", err)
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "error",
|
||
MessageID: "msg_" + generateID(),
|
||
Error: "语音流处理失败: " + err.Error(),
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
|
||
if text == "" {
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "voice_final",
|
||
MessageID: "voice_" + generateID(),
|
||
Text: "",
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
return
|
||
}
|
||
|
||
// Send final transcription to frontend
|
||
client.SendMessage(ws.ServerMessage{
|
||
Type: "voice_final",
|
||
MessageID: "voice_" + generateID(),
|
||
Text: text,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
})
|
||
|
||
// Route the transcribed text as a regular chat message to ai-core
|
||
chatMsg := ws.ClientMessage{
|
||
Type: "message",
|
||
Content: text,
|
||
Mode: msg.Mode,
|
||
}
|
||
h.handleChatMessage(client, chatMsg)
|
||
}()
|
||
}
|
||
|
||
// transcribeAudio 将 base64 编码的音频发送到 voice-service 进行转录。
|
||
func (h *ChatHandler) transcribeAudio(audioB64 string, format string) (string, error) {
|
||
audioData, err := decodeBase64(audioB64)
|
||
if err != nil {
|
||
return "", fmt.Errorf("解码音频数据失败: %w", err)
|
||
}
|
||
|
||
// 构建 multipart form
|
||
var buf bytes.Buffer
|
||
mw := multipart.NewWriter(&buf)
|
||
|
||
ext := ".webm"
|
||
switch format {
|
||
case "wav", "wave":
|
||
ext = ".wav"
|
||
case "mp3", "mpeg":
|
||
ext = ".mp3"
|
||
case "ogg", "opus":
|
||
ext = ".ogg"
|
||
case "pcm":
|
||
ext = ".pcm"
|
||
}
|
||
|
||
fw, err := mw.CreateFormFile("audio", "recording"+ext)
|
||
if err != nil {
|
||
return "", fmt.Errorf("创建表单字段失败: %w", err)
|
||
}
|
||
if _, err := fw.Write(audioData); err != nil {
|
||
return "", fmt.Errorf("写入音频数据失败: %w", err)
|
||
}
|
||
mw.Close()
|
||
|
||
voiceURL := h.cfg.VoiceServiceURL
|
||
if voiceURL == "" {
|
||
voiceURL = "http://localhost:8093"
|
||
}
|
||
|
||
httpReq, err := http.NewRequest("POST", voiceURL+"/api/v1/transcribe", &buf)
|
||
if err != nil {
|
||
return "", fmt.Errorf("创建请求失败: %w", err)
|
||
}
|
||
httpReq.Header.Set("Content-Type", mw.FormDataContentType())
|
||
|
||
httpClient := &http.Client{Timeout: 60 * time.Second}
|
||
resp, err := httpClient.Do(httpReq)
|
||
if err != nil {
|
||
return "", fmt.Errorf("voice-service 调用失败: %w", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
respBody, err := io.ReadAll(resp.Body)
|
||
if err != nil {
|
||
return "", fmt.Errorf("读取响应失败: %w", err)
|
||
}
|
||
|
||
var result struct {
|
||
Success bool `json:"success"`
|
||
Text string `json:"text"`
|
||
Error string `json:"error"`
|
||
}
|
||
if err := json.Unmarshal(respBody, &result); err != nil {
|
||
return "", fmt.Errorf("解析响应失败: %w", err)
|
||
}
|
||
|
||
if !result.Success {
|
||
if result.Error != "" {
|
||
return "", fmt.Errorf("%s", result.Error)
|
||
}
|
||
return "", fmt.Errorf("转录返回空结果")
|
||
}
|
||
|
||
return result.Text, nil
|
||
}
|
||
|
||
// decodeBase64 解码 base64 字符串(支持 Data URL 前缀)。
|
||
func decodeBase64(s string) ([]byte, error) {
|
||
// 移除 data:xxx;base64, 前缀
|
||
if idx := strings.Index(s, ","); idx != -1 {
|
||
s = s[idx+1:]
|
||
}
|
||
return base64.StdEncoding.DecodeString(s)
|
||
}
|
||
|
||
// handleHistoryRequest 处理历史消息请求
|
||
func (h *ChatHandler) handleHistoryRequest(client *ws.Client, msg ws.ClientMessage) {
|
||
// 优先使用请求中的 session_id,否则使用客户端的 session_id
|
||
sessionID := msg.SessionID
|
||
if sessionID == "" {
|
||
sessionID = client.SessionID
|
||
}
|
||
|
||
messages := h.hub.GetConversation(client.UserID, sessionID)
|
||
|
||
// 如果内存缓存为空,尝试从数据库恢复(网关重启后缓存丢失的情况)
|
||
if len(messages) == 0 && h.sessionStore != nil && h.sessionStore.IsAvailable() {
|
||
dbMessages, err := h.sessionStore.GetMessages(sessionID, 50, 0)
|
||
if err == nil && len(dbMessages) > 0 {
|
||
logger.Printf("[history] 从数据库恢复会话历史: session=%s, %d 条消息", sessionID, len(dbMessages))
|
||
// 恢复到内存缓存
|
||
for _, dbMsg := range dbMessages {
|
||
var ci *ws.ClientInfo
|
||
if dbMsg.ClientID != "" {
|
||
ci = h.hub.ClientInfo(dbMsg.ClientID)
|
||
if ci == nil {
|
||
ci = &ws.ClientInfo{ClientID: dbMsg.ClientID}
|
||
}
|
||
}
|
||
messages = append(messages, ws.Message{
|
||
ID: fmt.Sprintf("db_%d", dbMsg.ID),
|
||
Role: dbMsg.Role,
|
||
MsgType: dbMsg.MsgType,
|
||
Content: dbMsg.Content,
|
||
Timestamp: dbMsg.CreatedAt.UnixMilli(),
|
||
ClientInfo: ci,
|
||
})
|
||
h.hub.CacheMessage(client.UserID, sessionID, ws.Message{
|
||
ID: fmt.Sprintf("db_%d", dbMsg.ID),
|
||
Role: dbMsg.Role,
|
||
MsgType: dbMsg.MsgType,
|
||
Content: dbMsg.Content,
|
||
Timestamp: dbMsg.CreatedAt.UnixMilli(),
|
||
ClientInfo: ci,
|
||
})
|
||
}
|
||
}
|
||
}
|
||
|
||
if messages == nil {
|
||
messages = []ws.Message{}
|
||
}
|
||
|
||
response := ws.ServerMessage{
|
||
Type: "history_response",
|
||
MessageID: "hist_" + generateID(),
|
||
Messages: messages,
|
||
Timestamp: time.Now().UnixMilli(),
|
||
}
|
||
|
||
if err := client.SendMessage(response); err != nil {
|
||
logger.Printf("[WS] 发送历史消息失败: %v", err)
|
||
}
|
||
}
|
||
|
||
// SendSystemMessage 向用户发送系统消息(用于主动通知)
|
||
func (h *ChatHandler) SendSystemMessage(userID, sessionID, text string) error {
|
||
msg := ws.ServerMessage{
|
||
Type: "response",
|
||
MessageID: "sys_" + generateID(),
|
||
Text: text,
|
||
Role: "system",
|
||
MsgType: "system_info",
|
||
Timestamp: time.Now().UnixMilli(),
|
||
}
|
||
|
||
data, err := json.Marshal(msg)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
h.hub.SendToSession(userID, sessionID, data)
|
||
return nil
|
||
}
|
||
|
||
// HandleProactiveMessage 处理来自 AI-Core 后台思考的主动消息
|
||
// POST /api/v1/internal/proactive-message
|
||
func (h *ChatHandler) HandleProactiveMessage(c *gin.Context) {
|
||
var req struct {
|
||
UserID string `json:"user_id" binding:"required"`
|
||
Content string `json:"content" binding:"required"`
|
||
SessionID string `json:"session_id"`
|
||
}
|
||
if err := c.ShouldBindJSON(&req); err != nil {
|
||
c.JSON(http.StatusBadRequest, gin.H{"error": "请求参数无效: " + err.Error()})
|
||
return
|
||
}
|
||
|
||
// Parse content to split (action) from chat text.
|
||
segments := parseProactiveContent(req.Content)
|
||
|
||
// Check online status.
|
||
onlineCount := h.hub.UserClientCount(req.UserID)
|
||
|
||
sessionID := req.SessionID
|
||
if sessionID == "" {
|
||
sessionID = "session_admin_main"
|
||
}
|
||
|
||
timestamp := time.Now().UnixMilli()
|
||
|
||
for i, seg := range segments {
|
||
msgID := fmt.Sprintf("proactive_%s_%d", generateID(), i)
|
||
msgType := "chat"
|
||
role := "assistant"
|
||
if seg.msgType == "action" {
|
||
msgType = "action"
|
||
role = "action"
|
||
}
|
||
|
||
msg := ws.ServerMessage{
|
||
Type: "response",
|
||
MessageID: msgID,
|
||
Content: seg.content,
|
||
Role: role,
|
||
MsgType: msgType,
|
||
SessionID: sessionID,
|
||
Timestamp: timestamp + int64(i),
|
||
}
|
||
|
||
data, err := json.Marshal(msg)
|
||
if err != nil {
|
||
logger.Printf("[proactive] 序列化消息失败: %v", err)
|
||
continue
|
||
}
|
||
|
||
if onlineCount == 0 {
|
||
h.hub.QueueProactiveMessage(req.UserID, data)
|
||
} else {
|
||
h.hub.SendToUser(req.UserID, data)
|
||
if i < len(segments)-1 {
|
||
delay := 200 + int(time.Now().UnixNano()%200)
|
||
time.Sleep(time.Duration(delay) * time.Millisecond)
|
||
}
|
||
}
|
||
|
||
// Persist to database so proactive messages survive restarts.
|
||
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
|
||
dbRole := role
|
||
if dbRole == "action" {
|
||
dbRole = "assistant"
|
||
}
|
||
if err := h.sessionStore.AddMessage(sessionID, dbRole, msgType, seg.content, ""); err != nil {
|
||
logger.Printf("[proactive] 持久化消息失败: %v", err)
|
||
}
|
||
}
|
||
|
||
// Cache each segment to conversation history.
|
||
h.hub.CacheMessage(req.UserID, sessionID, ws.Message{
|
||
ID: msgID,
|
||
Role: role,
|
||
MsgType: msgType,
|
||
Content: seg.content,
|
||
Timestamp: timestamp,
|
||
})
|
||
h.hub.RecordMessage(sessionID, role, seg.content)
|
||
}
|
||
|
||
logger.Printf("[proactive] 主动消息已推送: user=%s, online=%d, segments=%d", req.UserID, onlineCount, len(segments))
|
||
|
||
reason := "delivered"
|
||
if onlineCount == 0 {
|
||
reason = "queued"
|
||
}
|
||
c.JSON(http.StatusOK, gin.H{
|
||
"success": true,
|
||
"message": "消息已推送",
|
||
"segments": len(segments),
|
||
"delivered": onlineCount,
|
||
"reason": reason,
|
||
})
|
||
}
|
||
|
||
// proactiveSegment holds a parsed piece of a proactive message.
|
||
type proactiveSegment struct {
|
||
msgType string // "chat" or "action"
|
||
content string
|
||
}
|
||
|
||
// parseProactiveContent splits text by (parenthesized actions).
|
||
// "(笑) 你好呀 (调暗灯光) 今天过得如何" →
|
||
// [action: "笑", chat: "你好呀", action: "调暗灯光", chat: "今天过得如何"]
|
||
func parseProactiveContent(text string) []proactiveSegment {
|
||
if text == "" {
|
||
return nil
|
||
}
|
||
|
||
var segments []proactiveSegment
|
||
remaining := []rune(text)
|
||
|
||
for len(remaining) > 0 {
|
||
actionStart := -1 // index in remaining
|
||
actionEnd := -1 // index after closing paren
|
||
var actionContent string
|
||
|
||
for i, r := range remaining {
|
||
if r == '(' || r == '(' {
|
||
actionStart = i
|
||
closeRune := ')'
|
||
if r == '(' {
|
||
closeRune = ')'
|
||
}
|
||
for j := i + 1; j < len(remaining); j++ {
|
||
if remaining[j] == closeRune {
|
||
actionEnd = j + 1
|
||
actionContent = string(remaining[i+1 : j])
|
||
break
|
||
}
|
||
}
|
||
break
|
||
}
|
||
}
|
||
|
||
if actionStart >= 0 {
|
||
if actionStart > 0 {
|
||
prefix := strings.TrimSpace(string(remaining[:actionStart]))
|
||
if prefix != "" {
|
||
segments = append(segments, proactiveSegment{msgType: "chat", content: prefix})
|
||
}
|
||
}
|
||
content := strings.TrimSpace(actionContent)
|
||
if content != "" {
|
||
segments = append(segments, proactiveSegment{msgType: "action", content: content})
|
||
}
|
||
remaining = remaining[actionEnd:]
|
||
} else {
|
||
text := strings.TrimSpace(string(remaining))
|
||
if text != "" {
|
||
segments = append(segments, proactiveSegment{msgType: "chat", content: text})
|
||
}
|
||
break
|
||
}
|
||
}
|
||
|
||
if len(segments) == 0 && text != "" {
|
||
segments = append(segments, proactiveSegment{msgType: "chat", content: strings.TrimSpace(text)})
|
||
}
|
||
|
||
return segments
|
||
}
|
||
|
||
// ========== 多端客户端管理 API ==========
|
||
|
||
// HandleListClients returns all known clients for the authenticated user.
|
||
// GET /api/v1/admin/clients
|
||
func (h *ChatHandler) HandleListClients(c *gin.Context) {
|
||
userID := c.Query("user_id")
|
||
if userID == "" {
|
||
userID = "admin"
|
||
}
|
||
clients := h.hub.GetKnownClients(userID)
|
||
|
||
// Merge with persisted notes from DB
|
||
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
|
||
dbClients, err := h.sessionStore.GetClients(userID)
|
||
if err == nil {
|
||
noteByID := make(map[string]string)
|
||
for _, dc := range dbClients {
|
||
noteByID[dc.ClientID] = dc.Note
|
||
}
|
||
for i := range clients {
|
||
if note, ok := noteByID[clients[i].ClientID]; ok && note != "" {
|
||
clients[i].Note = note
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
c.JSON(http.StatusOK, gin.H{
|
||
"clients": clients,
|
||
"total": len(clients),
|
||
})
|
||
}
|
||
|
||
// HandleUpdateClientNote sets a label/note on a client.
|
||
// PUT /api/v1/admin/clients/:id/note
|
||
func (h *ChatHandler) HandleUpdateClientNote(c *gin.Context) {
|
||
clientID := c.Param("id")
|
||
var req struct {
|
||
Note string `json:"note"`
|
||
}
|
||
if err := c.ShouldBindJSON(&req); err != nil {
|
||
c.JSON(http.StatusBadRequest, gin.H{"error": "请求参数无效"})
|
||
return
|
||
}
|
||
|
||
// Update in-memory
|
||
if !h.hub.UpdateClientNote(clientID, req.Note) {
|
||
c.JSON(http.StatusNotFound, gin.H{"error": "客户端未找到"})
|
||
return
|
||
}
|
||
|
||
// Persist to DB
|
||
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
|
||
if err := h.sessionStore.UpdateClientNote(clientID, req.Note); err != nil {
|
||
logger.Printf("[clients] 持久化备注失败: %v", err)
|
||
}
|
||
}
|
||
|
||
c.JSON(http.StatusOK, gin.H{"status": "ok", "client_id": clientID, "note": req.Note})
|
||
}
|
||
|
||
func generateID() string {
|
||
return time.Now().Format("20060102150405") + randomStr(6)
|
||
}
|
||
|
||
func randomStr(n int) string {
|
||
b := make([]byte, n)
|
||
if _, err := rand.Read(b); err != nil {
|
||
// fallback: deterministic but hard to predict
|
||
for i := range b {
|
||
b[i] = byte(time.Now().UnixNano() % 256)
|
||
}
|
||
}
|
||
return hex.EncodeToString(b)[:n]
|
||
}
|
||
|
||
// broadcastToUser sends a server message to ALL connected clients for a user.
|
||
func (h *ChatHandler) broadcastToUser(userID string, msg ws.ServerMessage) {
|
||
data, err := json.Marshal(msg)
|
||
if err != nil {
|
||
logger.Printf("[chat] 序列化广播消息失败: %v", err)
|
||
return
|
||
}
|
||
h.hub.SendToUser(userID, data)
|
||
}
|
||
|
||
// broadcastToUserExcept sends a server message to ALL connected clients for a user,
|
||
// excluding the specified clientID (the sender).
|
||
func (h *ChatHandler) broadcastToUserExcept(userID, excludeClientID string, msg ws.ServerMessage) {
|
||
data, err := json.Marshal(msg)
|
||
if err != nil {
|
||
logger.Printf("[chat] 序列化广播消息失败: %v", err)
|
||
return
|
||
}
|
||
h.hub.SendToUserExcept(userID, excludeClientID, data)
|
||
}
|
||
|
||
// parseMultiMessage 检测并解析多消息格式
|
||
// 如果文本包含空行分隔的多条消息,拆分为多条;否则返回单条
|
||
func parseMultiMessage(text string) []proactiveSegment {
|
||
if text == "" {
|
||
return nil
|
||
}
|
||
// 按双换行(空行)分割
|
||
parts := strings.Split(text, "\n\n")
|
||
// 过滤空字符串并去除首尾空白,检测消息类型
|
||
var result []proactiveSegment
|
||
for _, p := range parts {
|
||
p = strings.TrimSpace(p)
|
||
if p != "" {
|
||
msgType := "chat"
|
||
// 检测括号包裹的动作标记
|
||
if (strings.HasPrefix(p, "(") && strings.HasSuffix(p, ")")) ||
|
||
(strings.HasPrefix(p, "(") && strings.HasSuffix(p, ")")) {
|
||
msgType = "action"
|
||
}
|
||
result = append(result, proactiveSegment{msgType: msgType, content: p})
|
||
}
|
||
}
|
||
// 如果只有一条,返回 nil 表示不是多消息格式
|
||
if len(result) <= 1 {
|
||
return nil
|
||
}
|
||
return result
|
||
}
|