feat: admin 端点补齐 + 视觉真推理 + 语音链路 + wire.go 收口(A+B+第3步)
批次A admin 端点(原本 404 变 200 真实数据):
- GET /admin/llm:读 s.cfg.LLM 组 openai+anthropic 两 provider(enabled=当前 Provider、multimodal=配 VisionModel),APIKey 不进 wire 零泄露。
- GET /admin/automations:*automations.Service.List()(reminders+event rules+cron schedules,cron 加只读登记)。
- GET /admin/users:cfg.AdminUser + *auth.APIKey.List()(密钥掩码);无用户存储,恒先返回当前 admin 账号。
- GET /admin/backthink:*backthink.Backthink.List(ctx,50)。均只读、owner/admin RBAC、{data} 信封。
批次B 视觉真推理 + 语音:
- rest.go chatRequest 加 Image?;流式 publishChatStream 带 image;userChatMessages 构造 ImageMessage(model.go:71)。
- loop/vision.go:ImageResolver 可注入(MVP 先看图再对话);run.go Run 开头 resolveImages,无 resolver 透传直交主模型。
- vision/resolver.go:MessageResolver 调 vision.Router.Route——KindMain 保留图片交主模型多模态、KindVisionTool/OCRTool 经 dispatcher.Invoke 折叠成 [图片内容] 文本、KindUnsupported 记日志保留原图不崩。
- gateway/voice.go:VoiceService 接口 + SetVoice 注入点 + handleVoiceInput(未注入回 error 不崩)+ runVoiceTurn 异步 STT→loop chat→TTS→定向 voice_output 帧。
- ws_hub/ws_route:voice_input 接 handleVoiceInput;前端 ws.ts decodeBase64/playVoiceAudio/case voice_output。
第3步 wire.go 收口(唯一碰 wire.go 的一步):
- Deps 补传 Automations:a.automation / Backthink:a.backthink / Users:auth.NewAPIKey(a.cfg.Auth)(*auth.APIKey 原生满足 UserStore,避免 admin 重复的复合类型)。
- gw.SetVoice(a.voice);loop.Options.Vision = vision.NewMessageResolver(router, info, dispatcher.Invoke)(router=NewRouter(WithVisionTool/WithOCRTool)、info=cfg.LLM.VisionModel)。
实机验证:四 admin 端点 200 真实数据(llm 双 provider 脱敏、users=admin,yeij0942,owner);WS 握手正常。go build/vet/test 全绿 + 前端 tsc。wire.go 557 行。
This commit is contained in:
1 parent
648292d035
commit
518bd0a230
21 files changed
+976
-38
No files matched your search
@@ -0,0 +1,163 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/multimodal/voice"
|
||||
)
|
||||
|
||||
// VoiceService runs speech-to-text and text-to-speech for the voice_input WS
|
||||
// route. *voice.Chain (the degrading provider chain) and *voice.Service (the
|
||||
// background worker pool) both satisfy it. It is nil-tolerant: the voice route
|
||||
// replies with a clear error when no service is injected.
|
||||
type VoiceService interface {
|
||||
// STT transcribes audio to text.
|
||||
STT(ctx context.Context, audio voice.Audio) (string, error)
|
||||
// TTS synthesizes audio from text.
|
||||
TTS(ctx context.Context, text string) (voice.Audio, error)
|
||||
}
|
||||
|
||||
// voiceInputRequest is the decoded payload of a voice_input frame. Audio is a
|
||||
// base64 (or data: URI) audio blob; Format is its encoding ("wav"/"mp3"/"ogg");
|
||||
// Text overrides STT when present (a client that already transcribed can skip
|
||||
// recognition); SessionID defaults to "web".
|
||||
type voiceInputRequest struct {
|
||||
Audio string `json:"audio"`
|
||||
Format string `json:"format"`
|
||||
Text string `json:"text"`
|
||||
SessionID string `json:"sessionId"`
|
||||
}
|
||||
|
||||
// SetVoice injects the voice STT/TTS service used by the voice_input WS route.
|
||||
// It is the gateway's exposed injection point so the composition root can wire a
|
||||
// *voice.Chain or *voice.Service without touching Deps. A nil value disables the
|
||||
// route (it replies with a clear error).
|
||||
func (s *Server) SetVoice(v VoiceService) {
|
||||
s.voice = v
|
||||
s.hub.voice = v
|
||||
}
|
||||
|
||||
// handleVoiceInput routes one voice_input frame. Without an injected voice
|
||||
// service it replies with a clear error (never a crash); otherwise it decodes the
|
||||
// request and runs the STT→chat→TTS turn asynchronously so the read pump is never
|
||||
// blocked by provider latency.
|
||||
func (c *wsConn) handleVoiceInput(in wsInbound) {
|
||||
if c.h.voice == nil {
|
||||
c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": "voice unavailable"}})
|
||||
return
|
||||
}
|
||||
var req voiceInputRequest
|
||||
_ = json.Unmarshal(in.Data, &req)
|
||||
if strings.TrimSpace(req.Audio) == "" && strings.TrimSpace(req.Text) == "" {
|
||||
c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": "voice audio or text required"}})
|
||||
return
|
||||
}
|
||||
go c.h.runVoiceTurn(c, req)
|
||||
}
|
||||
|
||||
// runVoiceTurn drives one voice turn: STT (or a supplied transcript) → agent chat
|
||||
// → TTS, then replies to the requesting connection with a voice_output frame
|
||||
// carrying the base64 audio plus the recognized transcript and assistant reply.
|
||||
func (h *hub) runVoiceTurn(c *wsConn, req voiceInputRequest) {
|
||||
ctx := context.Background()
|
||||
if req.SessionID == "" {
|
||||
req.SessionID = "web"
|
||||
}
|
||||
transcript := strings.TrimSpace(req.Text)
|
||||
if transcript == "" {
|
||||
if strings.TrimSpace(req.Audio) == "" {
|
||||
h.voiceError(c, "voice audio or text required")
|
||||
return
|
||||
}
|
||||
audio, err := decodeVoiceAudio(req.Audio, req.Format)
|
||||
if err != nil {
|
||||
h.log.Warn("voice audio decode", "err", err)
|
||||
h.voiceError(c, "voice audio could not be decoded")
|
||||
return
|
||||
}
|
||||
stt, err := h.voice.STT(ctx, audio)
|
||||
if err != nil {
|
||||
h.log.Warn("voice stt", "err", err)
|
||||
h.voiceError(c, "voice transcription failed")
|
||||
return
|
||||
}
|
||||
transcript = strings.TrimSpace(stt)
|
||||
}
|
||||
if transcript == "" {
|
||||
h.voiceError(c, "no speech recognized")
|
||||
return
|
||||
}
|
||||
if h.runner == nil {
|
||||
h.voiceError(c, "chat service is not configured")
|
||||
return
|
||||
}
|
||||
res, err := h.runner.Run(ctx, req.SessionID, []model.Message{model.TextMessage(model.RoleUser, transcript)})
|
||||
if err != nil {
|
||||
h.log.Warn("voice chat run", "err", err, "session", req.SessionID)
|
||||
h.voiceError(c, "voice chat failed")
|
||||
return
|
||||
}
|
||||
reply := res.Text
|
||||
if strings.TrimSpace(reply) == "" {
|
||||
reply = res.StopReason
|
||||
}
|
||||
audi, err := h.voice.TTS(ctx, reply)
|
||||
if err != nil {
|
||||
h.log.Warn("voice tts", "err", err)
|
||||
h.voiceError(c, "voice synthesis failed")
|
||||
return
|
||||
}
|
||||
c.enqueueJSON(wsEvent{Type: wsVoiceOutput, Data: map[string]any{
|
||||
"audio": base64.StdEncoding.EncodeToString(audi.Data),
|
||||
"format": audi.Format,
|
||||
"text": transcript,
|
||||
"reply": reply,
|
||||
}})
|
||||
}
|
||||
|
||||
// voiceError replies with a typed error frame for a failed voice turn.
|
||||
func (h *hub) voiceError(c *wsConn, message string) {
|
||||
c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": message}})
|
||||
}
|
||||
|
||||
// decodeVoiceAudio turns a base64 (or data:) audio payload into voice.Audio. The
|
||||
// format is taken from req when set, guessed from a data: URI mime otherwise, and
|
||||
// defaulted to "wav".
|
||||
func decodeVoiceAudio(src, format string) (voice.Audio, error) {
|
||||
payload := src
|
||||
if strings.HasPrefix(src, "data:") {
|
||||
if comma := strings.IndexByte(src, ','); comma >= 0 {
|
||||
if format == "" {
|
||||
format = formatFromMime(src[:comma])
|
||||
}
|
||||
payload = src[comma+1:]
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(payload) == "" {
|
||||
return voice.Audio{}, fmt.Errorf("voice: empty audio")
|
||||
}
|
||||
data, err := base64.StdEncoding.DecodeString(payload)
|
||||
if err != nil {
|
||||
return voice.Audio{}, fmt.Errorf("voice: decode audio: %w", err)
|
||||
}
|
||||
if format == "" {
|
||||
format = "wav"
|
||||
}
|
||||
return voice.Audio{Format: format, Data: data}, nil
|
||||
}
|
||||
|
||||
// formatFromMime guesses an audio format from a data: URI mime segment.
|
||||
func formatFromMime(meta string) string {
|
||||
lower := strings.ToLower(meta)
|
||||
for _, f := range []string{"wav", "mp3", "ogg", "webm", "mpeg"} {
|
||||
if strings.Contains(lower, f) {
|
||||
return f
|
||||
}
|
||||
}
|
||||
return "wav"
|
||||
}
|
||||
Reference in new issue
Block a user