package gateway import ( "context" "encoding/base64" "encoding/json" "fmt" "strings" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/loop" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/multimodal/voice" ) // VoiceService runs speech-to-text and text-to-speech for the voice_input WS // route. *voice.Chain (the degrading provider chain) and *voice.Service (the // background worker pool) both satisfy it. It is nil-tolerant: the voice route // replies with a clear error when no service is injected. type VoiceService interface { // STT transcribes audio to text. STT(ctx context.Context, audio voice.Audio) (string, error) // TTS synthesizes audio from text. A per-request emotion/tone instruction may // be threaded through ctx (voice.WithTTSInstruction) to override the configured // VOICE_TTS_INSTRUCTION for one call. TTS(ctx context.Context, text string) (voice.Audio, error) // CloneVoice creates a custom voice (声音复刻 create_voice) from a public // reference audio URL and returns its voice_id. CloneVoice(ctx context.Context, referenceURL, prefix string) (string, error) } // voiceProviderHeader is the response header carrying the provider rung on the // binary /api/v1/audio/speech response, where a JSON signal field cannot be added. const voiceProviderHeader = "X-Voice-Provider" // voiceProviderReporter is the optional extension of VoiceService: a service that // can report which provider rung (cloud / local / edge-tts / espeak / silent) // produced a result. *voice.Chain and *voice.Service implement it; a custom // VoiceService that does not is still supported and falls back to the plain // STT/TTS methods with no reported rung. type voiceProviderReporter interface { STTWithProvider(ctx context.Context, audio voice.Audio) (string, string, error) TTSWithProvider(ctx context.Context, text string) (voice.Audio, string, error) } // sttWithProvider runs STT and returns the transcript plus the provider rung that // served it ("" when the injected service cannot report one). func sttWithProvider(ctx context.Context, v VoiceService, audio voice.Audio) (string, string, error) { if r, ok := v.(voiceProviderReporter); ok { return r.STTWithProvider(ctx, audio) } text, err := v.STT(ctx, audio) return text, "", err } // ttsWithProvider runs TTS and returns the audio plus the provider rung that // served it. func ttsWithProvider(ctx context.Context, v VoiceService, text string) (voice.Audio, string, error) { if r, ok := v.(voiceProviderReporter); ok { return r.TTSWithProvider(ctx, text) } audio, err := v.TTS(ctx, text) return audio, "", err } // providerSignal returns the "provider" degraded signal for a voice result: the // actual provider rung when the result is empty, "silent" when the rung is // unknown, and "" (omit the field) when the result carries real text/audio. It // only flags "no real speech/audio"; it is never a business error. func providerSignal(rung string, empty bool) string { if !empty { return "" } if rung == "" { return "silent" } return rung } // voiceInputRequest is the decoded payload of a voice_input frame. Audio is a // base64 (or data: URI) audio blob; Format is its encoding ("wav"/"mp3"/"ogg"); // Text overrides STT when present (a client that already transcribed can skip // recognition); SessionID defaults to "web"; DeviceID is the client-produced // end identifier (design/16 §4.5) threaded into the persisted voice turn. type voiceInputRequest struct { Audio string `json:"audio"` Format string `json:"format"` Text string `json:"text"` SessionID string `json:"sessionId"` DeviceID string `json:"deviceId,omitempty"` } // SetVoice injects the voice STT/TTS service used by the voice_input WS route. // It is the gateway's exposed injection point so the composition root can wire a // *voice.Chain or *voice.Service without touching Deps. A nil value disables the // route (it replies with a clear error). func (s *Server) SetVoice(v VoiceService) { s.voice = v s.hub.voice = v } // handleVoiceInput routes one voice_input frame. Without an injected voice // service it replies with a clear error (never a crash); otherwise it decodes the // request and runs the STT→chat→TTS turn asynchronously so the read pump is never // blocked by provider latency. func (c *wsConn) handleVoiceInput(in wsInbound) { if c.h.voice == nil { c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": "voice unavailable"}}) return } var req voiceInputRequest _ = json.Unmarshal(in.Data, &req) if strings.TrimSpace(req.Audio) == "" && strings.TrimSpace(req.Text) == "" { c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": "voice audio or text required"}}) return } go c.h.runVoiceTurn(c, req) } // runVoiceTurn drives one voice turn: STT (or a supplied transcript) → agent chat // → TTS, then replies to the requesting connection with a voice_output frame // carrying the base64 audio plus the recognized transcript and assistant reply. func (h *hub) runVoiceTurn(c *wsConn, req voiceInputRequest) { ctx := context.Background() // Thread the client's deviceID so the persisted voice turn is tagged with the // sending end (design/16 §4.5), mirroring ws_hub.go:240. ctx = withDeviceID(ctx, req.DeviceID) if req.SessionID == "" { req.SessionID = "web" } transcript := strings.TrimSpace(req.Text) if transcript == "" { if strings.TrimSpace(req.Audio) == "" { h.voiceError(c, "voice audio or text required") return } audio, err := decodeVoiceAudio(req.Audio, req.Format) if err != nil { h.log.Warn("voice audio decode", "err", err) h.voiceError(c, "voice audio could not be decoded") return } stt, err := h.voice.STT(ctx, audio) if err != nil { h.log.Warn("voice stt", "err", err) h.voiceError(c, "voice transcription failed") return } transcript = strings.TrimSpace(stt) } if transcript == "" { h.voiceError(c, "no speech recognized") return } if h.runner == nil { h.voiceError(c, "chat service is not configured") return } // The Web main session is the owner's private conversation, so it binds the // full-access env: memory read+write, unchanged behavior. res, err := h.runner.Run(loop.WithSessionEnv(ctx, loop.OwnerEnv()), req.SessionID, []model.Message{model.TextMessage(model.RoleUser, transcript)}) if err != nil { h.log.Warn("voice chat run", "err", err, "session", req.SessionID) h.voiceError(c, "voice chat failed") return } reply := res.Text if strings.TrimSpace(reply) == "" { reply = res.StopReason } audi, rung, err := ttsWithProvider(ctx, h.voice, reply) if err != nil { h.log.Warn("voice tts", "err", err) h.voiceError(c, "voice synthesis failed") return } data := map[string]any{ "audio": base64.StdEncoding.EncodeToString(audi.Data), "format": audi.Format, "text": transcript, "reply": reply, } if sig := providerSignal(rung, len(audi.Data) == 0); sig != "" { data["provider"] = sig } c.enqueueJSON(wsEvent{Type: wsVoiceOutput, Data: data}) } // voiceError replies with a typed error frame for a failed voice turn. func (h *hub) voiceError(c *wsConn, message string) { c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": message}}) } // decodeVoiceAudio turns a base64 (or data:) audio payload into voice.Audio. The // format is taken from req when set, guessed from a data: URI mime otherwise, and // defaulted to "wav". func decodeVoiceAudio(src, format string) (voice.Audio, error) { payload := src if strings.HasPrefix(src, "data:") { if comma := strings.IndexByte(src, ','); comma >= 0 { if format == "" { format = formatFromMime(src[:comma]) } payload = src[comma+1:] } } if strings.TrimSpace(payload) == "" { return voice.Audio{}, fmt.Errorf("voice: empty audio") } data, err := base64.StdEncoding.DecodeString(payload) if err != nil { return voice.Audio{}, fmt.Errorf("voice: decode audio: %w", err) } if format == "" { format = "wav" } return voice.Audio{Format: format, Data: data}, nil } // formatFromMime guesses an audio format from a data: URI mime segment. func formatFromMime(meta string) string { lower := strings.ToLower(meta) for _, f := range []string{"wav", "mp3", "ogg", "webm", "mpeg"} { if strings.Contains(lower, f) { return f } } return "wav" }