package gateway import ( "context" "encoding/base64" "encoding/json" "fmt" "strings" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/loop" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/multimodal/voice" ) // VoiceService runs speech-to-text and text-to-speech for the voice_input WS // route. *voice.Chain (the degrading provider chain) and *voice.Service (the // background worker pool) both satisfy it. It is nil-tolerant: the voice route // replies with a clear error when no service is injected. type VoiceService interface { // STT transcribes audio to text. STT(ctx context.Context, audio voice.Audio) (string, error) // TTS synthesizes audio from text. TTS(ctx context.Context, text string) (voice.Audio, error) } // voiceInputRequest is the decoded payload of a voice_input frame. Audio is a // base64 (or data: URI) audio blob; Format is its encoding ("wav"/"mp3"/"ogg"); // Text overrides STT when present (a client that already transcribed can skip // recognition); SessionID defaults to "web". type voiceInputRequest struct { Audio string `json:"audio"` Format string `json:"format"` Text string `json:"text"` SessionID string `json:"sessionId"` } // SetVoice injects the voice STT/TTS service used by the voice_input WS route. // It is the gateway's exposed injection point so the composition root can wire a // *voice.Chain or *voice.Service without touching Deps. A nil value disables the // route (it replies with a clear error). func (s *Server) SetVoice(v VoiceService) { s.voice = v s.hub.voice = v } // handleVoiceInput routes one voice_input frame. Without an injected voice // service it replies with a clear error (never a crash); otherwise it decodes the // request and runs the STT→chat→TTS turn asynchronously so the read pump is never // blocked by provider latency. func (c *wsConn) handleVoiceInput(in wsInbound) { if c.h.voice == nil { c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": "voice unavailable"}}) return } var req voiceInputRequest _ = json.Unmarshal(in.Data, &req) if strings.TrimSpace(req.Audio) == "" && strings.TrimSpace(req.Text) == "" { c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": "voice audio or text required"}}) return } go c.h.runVoiceTurn(c, req) } // runVoiceTurn drives one voice turn: STT (or a supplied transcript) → agent chat // → TTS, then replies to the requesting connection with a voice_output frame // carrying the base64 audio plus the recognized transcript and assistant reply. func (h *hub) runVoiceTurn(c *wsConn, req voiceInputRequest) { ctx := context.Background() if req.SessionID == "" { req.SessionID = "web" } transcript := strings.TrimSpace(req.Text) if transcript == "" { if strings.TrimSpace(req.Audio) == "" { h.voiceError(c, "voice audio or text required") return } audio, err := decodeVoiceAudio(req.Audio, req.Format) if err != nil { h.log.Warn("voice audio decode", "err", err) h.voiceError(c, "voice audio could not be decoded") return } stt, err := h.voice.STT(ctx, audio) if err != nil { h.log.Warn("voice stt", "err", err) h.voiceError(c, "voice transcription failed") return } transcript = strings.TrimSpace(stt) } if transcript == "" { h.voiceError(c, "no speech recognized") return } if h.runner == nil { h.voiceError(c, "chat service is not configured") return } // The Web main session is the owner's private conversation, so it binds the // full-access env: memory read+write, unchanged behavior. res, err := h.runner.Run(loop.WithSessionEnv(ctx, loop.OwnerEnv()), req.SessionID, []model.Message{model.TextMessage(model.RoleUser, transcript)}) if err != nil { h.log.Warn("voice chat run", "err", err, "session", req.SessionID) h.voiceError(c, "voice chat failed") return } reply := res.Text if strings.TrimSpace(reply) == "" { reply = res.StopReason } audi, err := h.voice.TTS(ctx, reply) if err != nil { h.log.Warn("voice tts", "err", err) h.voiceError(c, "voice synthesis failed") return } c.enqueueJSON(wsEvent{Type: wsVoiceOutput, Data: map[string]any{ "audio": base64.StdEncoding.EncodeToString(audi.Data), "format": audi.Format, "text": transcript, "reply": reply, }}) } // voiceError replies with a typed error frame for a failed voice turn. func (h *hub) voiceError(c *wsConn, message string) { c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": message}}) } // decodeVoiceAudio turns a base64 (or data:) audio payload into voice.Audio. The // format is taken from req when set, guessed from a data: URI mime otherwise, and // defaulted to "wav". func decodeVoiceAudio(src, format string) (voice.Audio, error) { payload := src if strings.HasPrefix(src, "data:") { if comma := strings.IndexByte(src, ','); comma >= 0 { if format == "" { format = formatFromMime(src[:comma]) } payload = src[comma+1:] } } if strings.TrimSpace(payload) == "" { return voice.Audio{}, fmt.Errorf("voice: empty audio") } data, err := base64.StdEncoding.DecodeString(payload) if err != nil { return voice.Audio{}, fmt.Errorf("voice: decode audio: %w", err) } if format == "" { format = "wav" } return voice.Audio{Format: format, Data: data}, nil } // formatFromMime guesses an audio format from a data: URI mime segment. func formatFromMime(meta string) string { lower := strings.ToLower(meta) for _, f := range []string{"wav", "mp3", "ogg", "webm", "mpeg"} { if strings.Contains(lower, f) { return f } } return "wav" }