Files
Cyrene/backend/gateway/internal/handler/chat_handler.go
T
AskaEth 6ef9e082a6 feat: 语音流式输入管线 + VAD前端集成 + 插件-工具合并清理
- 前端: VAD语音检测(@ricky0123/vad-web) + useVoiceInput双模式(流式WS/REST)
- Gateway: VoiceStreamManager代理WS流式STT到voice-service
- Voice-service: DashScope REST → Realtime WS → Whisper三级引擎 + ffmpeg转码
- 共享模块: pkg/audio(音频转换) + pkg/dashscope(ASR REST客户端)
- 清理: 移除旧plugin-manager和pkg/plugins,完成插件→工具合并
- 文档: 完善gateway-api.md和voice-service.md语音API文档
- 工具: scripts/voice/ 语音转换脚本集

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-06-06 11:50:40 +08:00

1208 lines
33 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package handler
import (
"bufio"
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"strings"
"sync"
"time"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"git.yeij.top/AskaEth/Cyrene/gateway/internal/config"
"git.yeij.top/AskaEth/Cyrene/gateway/internal/store"
"git.yeij.top/AskaEth/Cyrene/gateway/internal/ws"
"git.yeij.top/AskaEth/Cyrene/pkg/logger"
)
// queuedMsg 队列中的待处理消息
type queuedMsg struct {
client *ws.Client
mode string
reqBody []byte
content string
}
// ChatHandler 聊天处理器
type ChatHandler struct {
cfg *config.Config
hub *ws.Hub
sessionStore *store.SessionStore
fileStore *store.FileStore
voiceStream *VoiceStreamManager
upgrader websocket.Upgrader
pending map[string][]queuedMsg // per-session message queue
pendingMu sync.Mutex
}
// NewChatHandler 创建聊天处理器
func NewChatHandler(cfg *config.Config, hub *ws.Hub, sessionStore *store.SessionStore, fileStore *store.FileStore) *ChatHandler {
return &ChatHandler{
cfg: cfg,
hub: hub,
sessionStore: sessionStore,
fileStore: fileStore,
voiceStream: NewVoiceStreamManager(cfg.VoiceServiceURL),
pending: make(map[string][]queuedMsg),
upgrader: websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
CheckOrigin: func(r *http.Request) bool {
return true // 开发阶段允许所有来源
},
},
}
}
// HandleWebSocket 处理WebSocket升级和消息路由
func (h *ChatHandler) HandleWebSocket(c *gin.Context) {
// 从query参数获取token和session_id
token := c.Query("token")
sessionID := c.Query("session_id")
clientID := c.Query("client_id")
deviceName := c.Query("device_name")
userAgent := c.Request.UserAgent()
if token == "" {
// 也尝试从Authorization头读取
authHeader := c.GetHeader("Authorization")
if len(authHeader) > 7 && authHeader[:7] == "Bearer " {
token = authHeader[7:]
}
}
if token == "" {
c.JSON(http.StatusUnauthorized, gin.H{"error": "需要认证令牌"})
return
}
// 验证token
userID, err := h.cfg.ValidateToken(token)
if err != nil {
c.JSON(http.StatusUnauthorized, gin.H{"error": "认证令牌无效"})
return
}
// 主对话仅限管理员访问
if userID != "admin" {
c.JSON(http.StatusForbidden, gin.H{
"error": "主对话仅限管理员使用",
"errorType": "admin_only",
"hint": "请使用管理员账号登录以访问主对话功能",
})
return
}
if sessionID == "" {
sessionID = "session_" + generateID()
}
// 升级WebSocket连接
conn, err := h.upgrader.Upgrade(c.Writer, c.Request, nil)
if err != nil {
logger.Printf("[WS] 升级连接失败: %v", err)
return
}
// 创建客户端
client := ws.NewClient(h.hub, conn, userID, sessionID, clientID, deviceName, userAgent)
// 注册到Hub
h.hub.Register(client)
// 启动读写协程
go client.WritePump()
go client.ReadPump(func(client *ws.Client, msg ws.ClientMessage) {
h.handleMessage(client, msg)
})
}
// handleMessage 处理WebSocket消息
func (h *ChatHandler) handleMessage(client *ws.Client, msg ws.ClientMessage) {
switch msg.Type {
case "message":
h.handleChatMessage(client, msg)
case "voice_input":
h.handleVoiceInput(client, msg)
case "voice_stream_start":
h.handleVoiceStreamStart(client, msg)
case "voice_stream_chunk":
h.handleVoiceStreamChunk(client, msg)
case "voice_stream_end":
h.handleVoiceStreamEnd(client, msg)
case "history":
h.handleHistoryRequest(client, msg)
default:
logger.Printf("[WS] 未知消息类型: %s from user=%s", msg.Type, client.UserID)
}
}
// handleChatMessage 处理文字聊天消息 - 转发到 AI-Core(流式发送)
func (h *ChatHandler) handleChatMessage(client *ws.Client, msg ws.ClientMessage) {
mode := msg.Mode
if mode == "" {
mode = "text"
}
// 持久化用户消息到数据库(在 WebSocket 发送之前)
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
if err := h.sessionStore.AddMessage(client.SessionID, "user", "chat", msg.Content, client.ClientID); err != nil {
logger.Printf("[chat] 持久化用户消息失败: %v", err)
}
}
// 记录用户消息
h.hub.RecordMessage(client.SessionID, "user", msg.Content)
// 设置会话状态为 thinking
h.hub.UpdateSessionState(client.SessionID, "thinking")
// 构建 AI-Core 请求
aiReq := map[string]interface{}{
"user_id": client.UserID,
"session_id": client.SessionID,
"message": msg.Content,
"mode": mode,
}
if len(msg.Attachments) > 0 {
images := make([]string, 0, len(msg.Attachments))
for _, att := range msg.Attachments {
if att.Type == "image" {
imgURL := ""
// 优先使用 file_id 引用(轻量,支持跨设备同步)
if att.FileID != "" && h.fileStore != nil {
if f, err := h.fileStore.GetFile(att.FileID); err == nil {
imgURL = "http://127.0.0.1:" + h.cfg.Port + "/api/v1/files/" + f.ID + "/download"
} else {
logger.Printf("[chat] file_id 解析失败: %s, err=%v", att.FileID, err)
}
}
// 回退: 使用 URL(兼容旧客户端 base64 data URI
if imgURL == "" && att.URL != "" {
imgURL = att.URL
if strings.HasPrefix(imgURL, "/") {
imgURL = "http://127.0.0.1:" + h.cfg.Port + imgURL
}
}
if imgURL != "" {
images = append(images, imgURL)
}
}
}
if len(images) > 0 {
aiReq["images"] = images
}
}
reqBody, err := json.Marshal(aiReq)
if err != nil {
logger.Printf("[chat] 序列化请求失败: %v", err)
h.hub.UpdateSessionState(client.SessionID, "error")
client.SendMessage(ws.ServerMessage{
Type: "error",
MessageID: "msg_" + generateID(),
Error: "内部错误,请稍后重试",
Timestamp: time.Now().UnixMilli(),
})
return
}
// 缓存用户消息(在 goroutine 前完成,避免竞态)
userMsgID := msg.ClientMsgID
if userMsgID == "" {
userMsgID = "msg_" + generateID()
}
userMsg := ws.Message{
ID: userMsgID,
Role: "user",
Content: msg.Content,
Timestamp: time.Now().UnixMilli(),
ClientInfo: &ws.ClientInfo{
ClientID: client.ClientID,
DeviceName: client.DeviceName,
},
}
if len(msg.Attachments) > 0 {
userMsg.Attachments = msg.Attachments
}
h.hub.CacheMessage(client.UserID, client.SessionID, userMsg)
// 广播用户消息给同用户其他设备(跨端同步,排除发送者自身)
h.broadcastToUserExcept(client.UserID, client.ClientID, ws.ServerMessage{
Type: "response",
MessageID: userMsgID,
Content: msg.Content,
Role: "user",
MsgType: "chat",
SessionID: client.SessionID,
Timestamp: time.Now().UnixMilli(),
ClientInfo: &ws.ClientInfo{
ClientID: client.ClientID,
DeviceName: client.DeviceName,
},
})
// 排队处理:同一会话的消息串行化,避免并发请求导致上下文竞争
h.enqueueOrProcess(client, mode, reqBody, msg.Content)
}
// enqueueOrProcess 将消息加入 per-session 队列,若会话空闲则立即处理
func (h *ChatHandler) enqueueOrProcess(client *ws.Client, mode string, reqBody []byte, content string) {
h.pendingMu.Lock()
if queue, busy := h.pending[client.SessionID]; busy {
// 会话正在处理中,加入队列
h.pending[client.SessionID] = append(queue, queuedMsg{
client: client, mode: mode, reqBody: reqBody, content: content,
})
queueLen := len(h.pending[client.SessionID])
h.pendingMu.Unlock()
logger.Printf("[chat] 会话 %s 正在处理中,消息已加入队列 (位置 %d)", client.SessionID, queueLen)
client.SendMessage(ws.ServerMessage{
Type: "queued",
SessionID: client.SessionID,
Timestamp: time.Now().UnixMilli(),
})
return
}
// 标记为处理中
h.pending[client.SessionID] = nil
h.pendingMu.Unlock()
go h.processQueue(client, mode, reqBody, content)
}
// processQueue 处理当前消息,完成后自动消费队列中的下一条
func (h *ChatHandler) processQueue(client *ws.Client, mode string, reqBody []byte, content string) {
h.streamResponse(client, mode, reqBody, content)
// 处理队列中的后续消息
for {
h.pendingMu.Lock()
queue := h.pending[client.SessionID]
if len(queue) == 0 {
delete(h.pending, client.SessionID)
h.pendingMu.Unlock()
return
}
next := queue[0]
h.pending[client.SessionID] = queue[1:]
h.pendingMu.Unlock()
logger.Printf("[chat] 会话 %s 从队列取出消息继续处理 (剩余 %d)", client.SessionID, len(queue)-1)
h.streamResponse(next.client, next.mode, next.reqBody, next.content)
}
}
// streamResponse 调用 AI-Core SSE 流式接口并逐 delta 转发给客户端
func (h *ChatHandler) streamResponse(client *ws.Client, mode string, reqBody []byte, userMsg string) {
normalExit := false
defer func() {
if !normalExit {
h.broadcastToUser(client.UserID, ws.ServerMessage{
Type: "stream_end",
MessageID: "msg_" + generateID(),
SessionID: client.SessionID,
Timestamp: time.Now().UnixMilli(),
})
if h.hub != nil {
h.hub.UpdateSessionState(client.SessionID, "idle")
}
}
}()
aiCoreURL := h.cfg.AICoreURL + "/api/v1/chat"
httpReq, err := http.NewRequest("POST", aiCoreURL, bytes.NewReader(reqBody))
if err != nil {
logger.Printf("[chat] 创建 AI-Core 请求失败: %v", err)
h.hub.UpdateSessionState(client.SessionID, "error")
client.SendMessage(ws.ServerMessage{
Type: "error",
MessageID: "msg_" + generateID(),
Error: "服务暂不可用",
Timestamp: time.Now().UnixMilli(),
})
return
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Accept", "text/event-stream")
httpClient := &http.Client{Timeout: 120 * time.Second}
resp, err := httpClient.Do(httpReq)
if err != nil {
logger.Printf("[chat] AI-Core 调用失败: %v", err)
h.hub.UpdateSessionState(client.SessionID, "error")
client.SendMessage(ws.ServerMessage{
Type: "error",
MessageID: "msg_" + generateID(),
Error: fmt.Sprintf("AI-Core 调用失败: %v", err),
Timestamp: time.Now().UnixMilli(),
})
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
logger.Printf("[chat] AI-Core 返回错误 [%d]: %s", resp.StatusCode, string(body))
h.hub.UpdateSessionState(client.SessionID, "error")
client.SendMessage(ws.ServerMessage{
Type: "error",
MessageID: "msg_" + generateID(),
Error: fmt.Sprintf("AI-Core 错误 (%d)", resp.StatusCode),
Timestamp: time.Now().UnixMilli(),
})
return
}
// 使用 bufio.Scanner 逐行读取 SSE 响应
scanner := bufio.NewScanner(resp.Body)
// 增大 scanner buffer 以处理大块 SSE 数据
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
// 通知所有客户端 AI 开始生成回复
h.broadcastToUser(client.UserID, ws.ServerMessage{
Type: "stream_start",
MessageID: "msg_" + generateID(),
SessionID: client.SessionID,
Timestamp: time.Now().UnixMilli(),
})
var fullText string
var msgID string
var hasReview bool // 是否有审查消息(避免重复持久化)
var segments []ws.VoiceSegment // 收集断句信息
for scanner.Scan() {
line := scanner.Text()
// 跳过非 data 行
if !strings.HasPrefix(line, "data: ") {
continue
}
data := strings.TrimPrefix(line, "data: ")
// SSE 流结束标记
if data == "[DONE]" {
break
}
// 解析 delta 数据
var chunk struct {
Delta string `json:"delta"`
Error string `json:"error,omitempty"`
MessageID string `json:"message_id,omitempty"`
Mode string `json:"mode,omitempty"`
Done bool `json:"done,omitempty"`
// 断句相关 (来自 AI-Core 新格式)
Segments []struct {
Index int `json:"index"`
Text string `json:"text"`
} `json:"segments,omitempty"`
// 审查后的结构化消息
ReviewMessages []ws.ReviewMessage `json:"review_messages,omitempty"`
}
if err := json.Unmarshal([]byte(data), &chunk); err != nil {
logger.Printf("[chat] 解析 SSE delta 失败: %v, raw=%s", err, data)
continue
}
// 错误处理
if chunk.Error != "" {
logger.Printf("[chat] AI-Core 流式错误: %s", chunk.Error)
h.hub.UpdateSessionState(client.SessionID, "error")
h.broadcastToUser(client.UserID, ws.ServerMessage{
Type: "error",
MessageID: "msg_" + generateID(),
Error: chunk.Error,
Timestamp: time.Now().UnixMilli(),
})
return
}
// 记录 message_id
if chunk.MessageID != "" {
msgID = chunk.MessageID
}
// 如果是结束标记(含 done: true),跳出
if chunk.Done {
break
}
// 处理审查后的结构化消息 (review)
if len(chunk.ReviewMessages) > 0 {
for i, rm := range chunk.ReviewMessages {
msgType := rm.Type
if msgType == "" {
msgType = "chat"
}
role := "assistant"
if msgType == "action" {
role = "action"
}
reviewMsgID := fmt.Sprintf("%s_r%d", msgID, i)
// 持久化每条审查消息 (action 角色映射为 assistantLLM 模型不支持自定义角色)
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
dbRole := role
if dbRole == "action" {
dbRole = "assistant"
}
if err := h.sessionStore.AddMessage(client.SessionID, dbRole, msgType, rm.Content, client.ClientID); err != nil {
logger.Printf("[chat] 持久化审查消息失败: %v", err)
}
}
clientInfo := &ws.ClientInfo{
ClientID: client.ClientID,
DeviceName: client.DeviceName,
}
h.hub.CacheMessage(client.UserID, client.SessionID, ws.Message{
ID: reviewMsgID,
Role: role,
Content: rm.Content,
Timestamp: time.Now().UnixMilli(),
ClientInfo: clientInfo,
})
h.broadcastToUser(client.UserID, ws.ServerMessage{
Type: "response",
MessageID: reviewMsgID,
Content: rm.Content,
Role: role,
MsgType: msgType,
SessionID: client.SessionID,
Timestamp: time.Now().UnixMilli(),
ClientInfo: clientInfo,
Metadata: rm.Metadata,
})
// 使用 MessageScheduler 计算的 per-message 延迟
if rm.DelayMs > 0 {
time.Sleep(time.Duration(rm.DelayMs) * time.Millisecond)
}
}
hasReview = true
continue
}
// 处理断句事件 (stream_segments)
if len(chunk.Segments) > 0 {
for _, seg := range chunk.Segments {
segments = append(segments, ws.VoiceSegment{
Index: seg.Index,
Text: seg.Text,
})
}
// 发送断句事件给所有客户端
h.broadcastToUser(client.UserID, ws.ServerMessage{
Type: "stream_segments",
MessageID: msgID,
Segments: segments,
SessionID: client.SessionID,
Timestamp: time.Now().UnixMilli(),
})
continue
}
// 逐 delta 积累(不再逐块转发,由审查消息代替)
if chunk.Delta != "" {
fullText += chunk.Delta
}
}
if err := scanner.Err(); err != nil {
logger.Printf("[chat] SSE 读取错误: %v", err)
h.hub.UpdateSessionState(client.SessionID, "error")
h.broadcastToUser(client.UserID, ws.ServerMessage{
Type: "error",
MessageID: "msg_" + generateID(),
Error: fmt.Sprintf("流读取错误: %v", err),
Timestamp: time.Now().UnixMilli(),
})
return
}
if msgID == "" {
msgID = "msg_" + generateID()
}
// 检测是否为多消息格式(包含空行分隔的多条消息)
// 如果已有审查消息则跳过,避免与 review_messages 重复
multiParts := parseMultiMessage(fullText)
if !hasReview && len(multiParts) > 1 {
// 发送 multi_message 事件(每条消息带 msg_type
var items []ws.MultiMessageItem
for i, seg := range multiParts {
items = append(items, ws.MultiMessageItem{
Index: i,
Content: seg.content,
MsgType: seg.msgType,
})
}
h.broadcastToUser(client.UserID, ws.ServerMessage{
Type: "multi_message",
MessageID: msgID,
SessionID: client.SessionID,
MultiMessage: &ws.MultiMessagePayload{
Messages: items,
},
Timestamp: time.Now().UnixMilli(),
})
}
// 发送 stream_end 到所有客户端
h.broadcastToUser(client.UserID, ws.ServerMessage{
Type: "stream_end",
MessageID: msgID,
SessionID: client.SessionID,
Content: fullText,
Text: fullText,
Timestamp: time.Now().UnixMilli(),
})
// 持久化 AI 回复到数据库
// 如果有审查消息,每条已单独持久化,跳过 fullText 以避免重复
if !hasReview && fullText != "" {
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
if err := h.sessionStore.AddMessage(client.SessionID, "assistant", "chat", fullText, client.ClientID); err != nil {
logger.Printf("[chat] 持久化 AI 回复失败: %v", err)
}
}
h.hub.CacheMessage(client.UserID, client.SessionID, ws.Message{
ID: msgID,
Role: "assistant",
Content: fullText,
Timestamp: time.Now().UnixMilli(),
ClientInfo: &ws.ClientInfo{
ClientID: client.ClientID,
DeviceName: client.DeviceName,
},
})
}
// RecordMessage 使用不带 [review] 标记的文本
recordText := fullText
if hasReview {
recordText = strings.ReplaceAll(fullText, "[review]", "")
}
h.hub.RecordMessage(client.SessionID, "assistant", recordText)
// 设置会话状态为 idle
normalExit = true
h.hub.UpdateSessionState(client.SessionID, "idle")
}
// handleVoiceInput 处理语音输入
func (h *ChatHandler) handleVoiceInput(client *ws.Client, msg ws.ClientMessage) {
audioB64 := msg.AudioData
if audioB64 == "" {
client.SendMessage(ws.ServerMessage{
Type: "error",
MessageID: "msg_" + generateID(),
Error: "语音数据为空",
Timestamp: time.Now().UnixMilli(),
})
return
}
format := msg.Mode
if format == "" {
format = "webm"
}
// 在 goroutine 中处理转录,避免阻塞 ReadPump
go func() {
text, err := h.transcribeAudio(audioB64, format)
if err != nil {
logger.Printf("[voice] 转录失败: %v", err)
client.SendMessage(ws.ServerMessage{
Type: "voice_transcript",
MessageID: "msg_" + generateID(),
Error: fmt.Sprintf("语音识别失败: %v", err),
Timestamp: time.Now().UnixMilli(),
})
return
}
if text == "" {
client.SendMessage(ws.ServerMessage{
Type: "voice_transcript",
MessageID: "msg_" + generateID(),
Text: "",
Timestamp: time.Now().UnixMilli(),
})
return
}
// 发送转录结果给前端
client.SendMessage(ws.ServerMessage{
Type: "voice_transcript",
MessageID: "msg_" + generateID(),
Text: text,
Timestamp: time.Now().UnixMilli(),
})
// 将转录文本作为聊天消息处理
chatMsg := ws.ClientMessage{
Type: "message",
Content: text,
Mode: msg.Mode,
}
h.handleChatMessage(client, chatMsg)
}()
}
// handleVoiceStreamStart begins a streaming voice session via voice-service.
func (h *ChatHandler) handleVoiceStreamStart(client *ws.Client, msg ws.ClientMessage) {
format := msg.Format
if format == "" {
format = "webm"
}
language := msg.Language
if language == "" {
language = "zh"
}
if err := h.voiceStream.StartStream(client, format, language); err != nil {
logger.Printf("[voice-stream] 启动流式 STT 失败: %v", err)
client.SendMessage(ws.ServerMessage{
Type: "error",
MessageID: "msg_" + generateID(),
Error: "启动语音流失败: " + err.Error(),
Timestamp: time.Now().UnixMilli(),
})
return
}
client.SendMessage(ws.ServerMessage{
Type: "voice_interim",
MessageID: "voice_" + generateID(),
Text: "",
Timestamp: time.Now().UnixMilli(),
})
}
// handleVoiceStreamChunk forwards an audio chunk to the active voice stream.
func (h *ChatHandler) handleVoiceStreamChunk(client *ws.Client, msg ws.ClientMessage) {
if msg.AudioData == "" {
return
}
audioData, err := decodeBase64(msg.AudioData)
if err != nil {
logger.Printf("[voice-stream] 解码音频块失败: %v", err)
return
}
if err := h.voiceStream.SendChunk(client.ClientID, client.SessionID, audioData, msg.Sequence); err != nil {
logger.Printf("[voice-stream] 发送音频块失败: %v", err)
}
}
// handleVoiceStreamEnd stops the voice stream and processes the final transcription.
func (h *ChatHandler) handleVoiceStreamEnd(client *ws.Client, msg ws.ClientMessage) {
go func() {
text, err := h.voiceStream.EndStream(client.ClientID, client.SessionID)
if err != nil {
logger.Printf("[voice-stream] 结束流式 STT 失败: %v", err)
client.SendMessage(ws.ServerMessage{
Type: "error",
MessageID: "msg_" + generateID(),
Error: "语音流处理失败: " + err.Error(),
Timestamp: time.Now().UnixMilli(),
})
return
}
if text == "" {
client.SendMessage(ws.ServerMessage{
Type: "voice_final",
MessageID: "voice_" + generateID(),
Text: "",
Timestamp: time.Now().UnixMilli(),
})
return
}
// Send final transcription to frontend
client.SendMessage(ws.ServerMessage{
Type: "voice_final",
MessageID: "voice_" + generateID(),
Text: text,
Timestamp: time.Now().UnixMilli(),
})
// Route the transcribed text as a regular chat message to ai-core
chatMsg := ws.ClientMessage{
Type: "message",
Content: text,
Mode: msg.Mode,
}
h.handleChatMessage(client, chatMsg)
}()
}
// transcribeAudio 将 base64 编码的音频发送到 voice-service 进行转录。
func (h *ChatHandler) transcribeAudio(audioB64 string, format string) (string, error) {
audioData, err := decodeBase64(audioB64)
if err != nil {
return "", fmt.Errorf("解码音频数据失败: %w", err)
}
// 构建 multipart form
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
ext := ".webm"
switch format {
case "wav", "wave":
ext = ".wav"
case "mp3", "mpeg":
ext = ".mp3"
case "ogg", "opus":
ext = ".ogg"
case "pcm":
ext = ".pcm"
}
fw, err := mw.CreateFormFile("audio", "recording"+ext)
if err != nil {
return "", fmt.Errorf("创建表单字段失败: %w", err)
}
if _, err := fw.Write(audioData); err != nil {
return "", fmt.Errorf("写入音频数据失败: %w", err)
}
mw.Close()
voiceURL := h.cfg.VoiceServiceURL
if voiceURL == "" {
voiceURL = "http://localhost:8093"
}
httpReq, err := http.NewRequest("POST", voiceURL+"/api/v1/transcribe", &buf)
if err != nil {
return "", fmt.Errorf("创建请求失败: %w", err)
}
httpReq.Header.Set("Content-Type", mw.FormDataContentType())
httpClient := &http.Client{Timeout: 60 * time.Second}
resp, err := httpClient.Do(httpReq)
if err != nil {
return "", fmt.Errorf("voice-service 调用失败: %w", err)
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("读取响应失败: %w", err)
}
var result struct {
Success bool `json:"success"`
Text string `json:"text"`
Error string `json:"error"`
}
if err := json.Unmarshal(respBody, &result); err != nil {
return "", fmt.Errorf("解析响应失败: %w", err)
}
if !result.Success {
if result.Error != "" {
return "", fmt.Errorf("%s", result.Error)
}
return "", fmt.Errorf("转录返回空结果")
}
return result.Text, nil
}
// decodeBase64 解码 base64 字符串(支持 Data URL 前缀)。
func decodeBase64(s string) ([]byte, error) {
// 移除 data:xxx;base64, 前缀
if idx := strings.Index(s, ","); idx != -1 {
s = s[idx+1:]
}
return base64.StdEncoding.DecodeString(s)
}
// handleHistoryRequest 处理历史消息请求
func (h *ChatHandler) handleHistoryRequest(client *ws.Client, msg ws.ClientMessage) {
// 优先使用请求中的 session_id,否则使用客户端的 session_id
sessionID := msg.SessionID
if sessionID == "" {
sessionID = client.SessionID
}
messages := h.hub.GetConversation(client.UserID, sessionID)
// 如果内存缓存为空,尝试从数据库恢复(网关重启后缓存丢失的情况)
if len(messages) == 0 && h.sessionStore != nil && h.sessionStore.IsAvailable() {
dbMessages, err := h.sessionStore.GetMessages(sessionID, 50, 0)
if err == nil && len(dbMessages) > 0 {
logger.Printf("[history] 从数据库恢复会话历史: session=%s, %d 条消息", sessionID, len(dbMessages))
// 恢复到内存缓存
for _, dbMsg := range dbMessages {
var ci *ws.ClientInfo
if dbMsg.ClientID != "" {
ci = h.hub.ClientInfo(dbMsg.ClientID)
if ci == nil {
ci = &ws.ClientInfo{ClientID: dbMsg.ClientID}
}
}
messages = append(messages, ws.Message{
ID: fmt.Sprintf("db_%d", dbMsg.ID),
Role: dbMsg.Role,
MsgType: dbMsg.MsgType,
Content: dbMsg.Content,
Timestamp: dbMsg.CreatedAt.UnixMilli(),
ClientInfo: ci,
})
h.hub.CacheMessage(client.UserID, sessionID, ws.Message{
ID: fmt.Sprintf("db_%d", dbMsg.ID),
Role: dbMsg.Role,
MsgType: dbMsg.MsgType,
Content: dbMsg.Content,
Timestamp: dbMsg.CreatedAt.UnixMilli(),
ClientInfo: ci,
})
}
}
}
if messages == nil {
messages = []ws.Message{}
}
response := ws.ServerMessage{
Type: "history_response",
MessageID: "hist_" + generateID(),
Messages: messages,
Timestamp: time.Now().UnixMilli(),
}
if err := client.SendMessage(response); err != nil {
logger.Printf("[WS] 发送历史消息失败: %v", err)
}
}
// SendSystemMessage 向用户发送系统消息(用于主动通知)
func (h *ChatHandler) SendSystemMessage(userID, sessionID, text string) error {
msg := ws.ServerMessage{
Type: "response",
MessageID: "sys_" + generateID(),
Text: text,
Role: "system",
MsgType: "system_info",
Timestamp: time.Now().UnixMilli(),
}
data, err := json.Marshal(msg)
if err != nil {
return err
}
h.hub.SendToSession(userID, sessionID, data)
return nil
}
// HandleProactiveMessage 处理来自 AI-Core 后台思考的主动消息
// POST /api/v1/internal/proactive-message
func (h *ChatHandler) HandleProactiveMessage(c *gin.Context) {
var req struct {
UserID string `json:"user_id" binding:"required"`
Content string `json:"content" binding:"required"`
SessionID string `json:"session_id"`
}
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "请求参数无效: " + err.Error()})
return
}
// Parse content to split (action) from chat text.
segments := parseProactiveContent(req.Content)
// Check online status.
onlineCount := h.hub.UserClientCount(req.UserID)
sessionID := req.SessionID
if sessionID == "" {
sessionID = "session_admin_main"
}
timestamp := time.Now().UnixMilli()
for i, seg := range segments {
msgID := fmt.Sprintf("proactive_%s_%d", generateID(), i)
msgType := "chat"
role := "assistant"
if seg.msgType == "action" {
msgType = "action"
role = "action"
}
msg := ws.ServerMessage{
Type: "response",
MessageID: msgID,
Content: seg.content,
Role: role,
MsgType: msgType,
SessionID: sessionID,
Timestamp: timestamp + int64(i),
}
data, err := json.Marshal(msg)
if err != nil {
logger.Printf("[proactive] 序列化消息失败: %v", err)
continue
}
if onlineCount == 0 {
h.hub.QueueProactiveMessage(req.UserID, data)
} else {
h.hub.SendToUser(req.UserID, data)
if i < len(segments)-1 {
delay := 200 + int(time.Now().UnixNano()%200)
time.Sleep(time.Duration(delay) * time.Millisecond)
}
}
// Persist to database so proactive messages survive restarts.
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
dbRole := role
if dbRole == "action" {
dbRole = "assistant"
}
if err := h.sessionStore.AddMessage(sessionID, dbRole, msgType, seg.content, ""); err != nil {
logger.Printf("[proactive] 持久化消息失败: %v", err)
}
}
// Cache each segment to conversation history.
h.hub.CacheMessage(req.UserID, sessionID, ws.Message{
ID: msgID,
Role: role,
MsgType: msgType,
Content: seg.content,
Timestamp: timestamp,
})
h.hub.RecordMessage(sessionID, role, seg.content)
}
logger.Printf("[proactive] 主动消息已推送: user=%s, online=%d, segments=%d", req.UserID, onlineCount, len(segments))
reason := "delivered"
if onlineCount == 0 {
reason = "queued"
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "消息已推送",
"segments": len(segments),
"delivered": onlineCount,
"reason": reason,
})
}
// proactiveSegment holds a parsed piece of a proactive message.
type proactiveSegment struct {
msgType string // "chat" or "action"
content string
}
// parseProactiveContent splits text by (parenthesized actions).
// "(笑) 你好呀 (调暗灯光) 今天过得如何" →
// [action: "笑", chat: "你好呀", action: "调暗灯光", chat: "今天过得如何"]
func parseProactiveContent(text string) []proactiveSegment {
if text == "" {
return nil
}
var segments []proactiveSegment
remaining := []rune(text)
for len(remaining) > 0 {
actionStart := -1 // index in remaining
actionEnd := -1 // index after closing paren
var actionContent string
for i, r := range remaining {
if r == '(' || r == '' {
actionStart = i
closeRune := ')'
if r == '' {
closeRune = ''
}
for j := i + 1; j < len(remaining); j++ {
if remaining[j] == closeRune {
actionEnd = j + 1
actionContent = string(remaining[i+1 : j])
break
}
}
break
}
}
if actionStart >= 0 {
if actionStart > 0 {
prefix := strings.TrimSpace(string(remaining[:actionStart]))
if prefix != "" {
segments = append(segments, proactiveSegment{msgType: "chat", content: prefix})
}
}
content := strings.TrimSpace(actionContent)
if content != "" {
segments = append(segments, proactiveSegment{msgType: "action", content: content})
}
remaining = remaining[actionEnd:]
} else {
text := strings.TrimSpace(string(remaining))
if text != "" {
segments = append(segments, proactiveSegment{msgType: "chat", content: text})
}
break
}
}
if len(segments) == 0 && text != "" {
segments = append(segments, proactiveSegment{msgType: "chat", content: strings.TrimSpace(text)})
}
return segments
}
// ========== 多端客户端管理 API ==========
// HandleListClients returns all known clients for the authenticated user.
// GET /api/v1/admin/clients
func (h *ChatHandler) HandleListClients(c *gin.Context) {
userID := c.Query("user_id")
if userID == "" {
userID = "admin"
}
clients := h.hub.GetKnownClients(userID)
// Merge with persisted notes from DB
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
dbClients, err := h.sessionStore.GetClients(userID)
if err == nil {
noteByID := make(map[string]string)
for _, dc := range dbClients {
noteByID[dc.ClientID] = dc.Note
}
for i := range clients {
if note, ok := noteByID[clients[i].ClientID]; ok && note != "" {
clients[i].Note = note
}
}
}
}
c.JSON(http.StatusOK, gin.H{
"clients": clients,
"total": len(clients),
})
}
// HandleUpdateClientNote sets a label/note on a client.
// PUT /api/v1/admin/clients/:id/note
func (h *ChatHandler) HandleUpdateClientNote(c *gin.Context) {
clientID := c.Param("id")
var req struct {
Note string `json:"note"`
}
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "请求参数无效"})
return
}
// Update in-memory
if !h.hub.UpdateClientNote(clientID, req.Note) {
c.JSON(http.StatusNotFound, gin.H{"error": "客户端未找到"})
return
}
// Persist to DB
if h.sessionStore != nil && h.sessionStore.IsAvailable() {
if err := h.sessionStore.UpdateClientNote(clientID, req.Note); err != nil {
logger.Printf("[clients] 持久化备注失败: %v", err)
}
}
c.JSON(http.StatusOK, gin.H{"status": "ok", "client_id": clientID, "note": req.Note})
}
func generateID() string {
return time.Now().Format("20060102150405") + randomStr(6)
}
func randomStr(n int) string {
b := make([]byte, n)
if _, err := rand.Read(b); err != nil {
// fallback: deterministic but hard to predict
for i := range b {
b[i] = byte(time.Now().UnixNano() % 256)
}
}
return hex.EncodeToString(b)[:n]
}
// broadcastToUser sends a server message to ALL connected clients for a user.
func (h *ChatHandler) broadcastToUser(userID string, msg ws.ServerMessage) {
data, err := json.Marshal(msg)
if err != nil {
logger.Printf("[chat] 序列化广播消息失败: %v", err)
return
}
h.hub.SendToUser(userID, data)
}
// broadcastToUserExcept sends a server message to ALL connected clients for a user,
// excluding the specified clientID (the sender).
func (h *ChatHandler) broadcastToUserExcept(userID, excludeClientID string, msg ws.ServerMessage) {
data, err := json.Marshal(msg)
if err != nil {
logger.Printf("[chat] 序列化广播消息失败: %v", err)
return
}
h.hub.SendToUserExcept(userID, excludeClientID, data)
}
// parseMultiMessage 检测并解析多消息格式
// 如果文本包含空行分隔的多条消息,拆分为多条;否则返回单条
func parseMultiMessage(text string) []proactiveSegment {
if text == "" {
return nil
}
// 按双换行(空行)分割
parts := strings.Split(text, "\n\n")
// 过滤空字符串并去除首尾空白,检测消息类型
var result []proactiveSegment
for _, p := range parts {
p = strings.TrimSpace(p)
if p != "" {
msgType := "chat"
// 检测括号包裹的动作标记
if (strings.HasPrefix(p, "(") && strings.HasSuffix(p, ")")) ||
(strings.HasPrefix(p, "") && strings.HasSuffix(p, "")) {
msgType = "action"
}
result = append(result, proactiveSegment{msgType: msgType, content: p})
}
}
// 如果只有一条,返回 nil 表示不是多消息格式
if len(result) <= 1 {
return nil
}
return result
}