feat(voice): 阿里百炼 STT/TTS 主链路接通(base64 STT + cosyvoice TTS按voice_id + REST /audio/* + voice_input.deviceId)
- voice/dashscope.go 新建:DashscopeSTT(qwen-audio-3.0-asr-flash, base64 data URI→text) + DashscopeTTS(cosyvoice 按 voice_id, 取非流式 24h URL 下载字节)
- cloud.go newDashboardSTT/TTS 由空 endpoint stub 改为真实百炼 adapter
- chain.go gating 从 DashscopeAPIKey!="" 改为 voiceConfigured(cfg); silent 兜底保留
- config.VoiceConfig 扩展 DASHSCOPE_BASE_URL/VOICE_STT_*/VOICE_TTS_* 字段(必填 DASHSCOPE_API_KEY)
- gateway: POST /api/v1/audio/{transcribe,speech} (owner+审计+MaxBytesReader); voiceInputRequest 增 deviceId 并透传 runner.Run
- 实测: cosyvoice-v3.5-plus+longxiaochun 回 418(引擎拒绝), 改用 cosyvoice-v3-flash+longxiaochun_v3; TTS 你好我是昔涟->106124B wav -> STT 回 你好我是西联, 链返回真实字节/文本
- 单测: dashscope_test/config voice defaults/gateway audio_test; go build/vet/test 全绿
- 声音复刻(响昔涟音色)待公网参考音频; 不做 create_voice
This commit is contained in:
1 parent
d8044763ac
commit
73dabb9dde
13 files changed
+1070
-15
No files matched your search
@@ -0,0 +1,122 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
apperr "git.yeij.top/AskaEth/Cyrene_v2/internal/errors"
|
||||
)
|
||||
|
||||
// maxAudioRequestBytes caps the JSON body a /api/v1/audio/* request may carry so
|
||||
// a huge base64 audio payload cannot exhaust memory.
|
||||
const maxAudioRequestBytes = 32 << 20
|
||||
|
||||
// transcribeRequest is the POST /api/v1/audio/transcribe body. Audio is a base64
|
||||
// (or data: URI) blob; Format is its encoding ("wav"/"mp3"/"ogg").
|
||||
type transcribeRequest struct {
|
||||
Audio string `json:"audio"`
|
||||
Format string `json:"format"`
|
||||
DeviceID string `json:"deviceId,omitempty"`
|
||||
}
|
||||
|
||||
// speechRequest is the POST /api/v1/audio/speech body. Voice/format are accepted
|
||||
// for forward compatibility; synthesis currently follows the server's
|
||||
// VOICE_TTS_VOICE_ID / VOICE_TTS_FORMAT config.
|
||||
type speechRequest struct {
|
||||
Text string `json:"text"`
|
||||
Voice string `json:"voice,omitempty"`
|
||||
Format string `json:"format,omitempty"`
|
||||
}
|
||||
|
||||
// handleTranscribe runs STT on the uploaded audio and returns the recognized
|
||||
// text. It is the REST source for native clients, bypassing the 4KB WebSocket
|
||||
// frame limit that would truncate a base64 audio blob.
|
||||
func (s *Server) handleTranscribe(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxAudioRequestBytes)
|
||||
var req transcribeRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
s.auditClaims(r, "audio.transcribe", "error")
|
||||
writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "invalid request body"))
|
||||
return
|
||||
}
|
||||
if s.voice == nil {
|
||||
s.auditClaims(r, "audio.transcribe", "error")
|
||||
writeError(w, apperr.Wrap(apperr.ErrInternal, "voice_unavailable", "voice service is not configured"))
|
||||
return
|
||||
}
|
||||
audio, err := decodeVoiceAudio(req.Audio, req.Format)
|
||||
if err != nil {
|
||||
s.auditClaims(r, "audio.transcribe", "error")
|
||||
writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "audio could not be decoded"))
|
||||
return
|
||||
}
|
||||
if len(audio.Data) == 0 {
|
||||
s.auditClaims(r, "audio.transcribe", "error")
|
||||
writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "audio is required"))
|
||||
return
|
||||
}
|
||||
text, err := s.voice.STT(r.Context(), audio)
|
||||
if err != nil {
|
||||
s.auditClaims(r, "audio.transcribe", "error")
|
||||
s.log.Warn("voice transcribe", "err", err)
|
||||
writeError(w, apperr.Wrap(apperr.ErrInternal, "stt_failed", "speech transcription failed"))
|
||||
return
|
||||
}
|
||||
s.auditClaims(r, "audio.transcribe", "ok")
|
||||
writeJSON(w, http.StatusOK, map[string]any{"text": text})
|
||||
}
|
||||
|
||||
// handleSpeech synthesizes speech for the given text and writes the raw audio
|
||||
// bytes (audio/<format>) back — a binary response, not a JSON envelope.
|
||||
func (s *Server) handleSpeech(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxAudioRequestBytes)
|
||||
var req speechRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
s.auditClaims(r, "audio.speech", "error")
|
||||
writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "invalid request body"))
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(req.Text) == "" {
|
||||
s.auditClaims(r, "audio.speech", "error")
|
||||
writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "text is required"))
|
||||
return
|
||||
}
|
||||
if s.voice == nil {
|
||||
s.auditClaims(r, "audio.speech", "error")
|
||||
writeError(w, apperr.Wrap(apperr.ErrInternal, "voice_unavailable", "voice service is not configured"))
|
||||
return
|
||||
}
|
||||
audi, err := s.voice.TTS(r.Context(), req.Text)
|
||||
if err != nil {
|
||||
s.auditClaims(r, "audio.speech", "error")
|
||||
s.log.Warn("voice speech", "err", err)
|
||||
writeError(w, apperr.Wrap(apperr.ErrInternal, "tts_failed", "speech synthesis failed"))
|
||||
return
|
||||
}
|
||||
if len(audi.Data) == 0 {
|
||||
s.auditClaims(r, "audio.speech", "error")
|
||||
writeError(w, apperr.Wrap(apperr.ErrInternal, "tts_empty", "no audio synthesized"))
|
||||
return
|
||||
}
|
||||
s.auditClaims(r, "audio.speech", "ok")
|
||||
w.Header().Set("Content-Type", audioMime(audi.Format))
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(audi.Data)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(audi.Data)
|
||||
}
|
||||
|
||||
// audioMime maps an audio format to its Content-Type for the speech response.
|
||||
func audioMime(format string) string {
|
||||
switch strings.ToLower(format) {
|
||||
case "mp3", "mpeg":
|
||||
return "audio/mpeg"
|
||||
case "ogg", "oga":
|
||||
return "audio/ogg"
|
||||
case "webm":
|
||||
return "audio/webm"
|
||||
default:
|
||||
return "audio/wav"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/auth"
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/multimodal/voice"
|
||||
)
|
||||
|
||||
// audioStubVoice is a VoiceService returning fixed STT/TTS results for the
|
||||
// REST /api/v1/audio/* handler tests.
|
||||
type audioStubVoice struct {
|
||||
sttText string
|
||||
sttErr error
|
||||
ttsAudio voice.Audio
|
||||
ttsErr error
|
||||
gotAudio voice.Audio
|
||||
gotText string
|
||||
sttCalled bool
|
||||
ttsCalled bool
|
||||
}
|
||||
|
||||
func (s *audioStubVoice) STT(ctx context.Context, audio voice.Audio) (string, error) {
|
||||
s.sttCalled = true
|
||||
s.gotAudio = audio
|
||||
return s.sttText, s.sttErr
|
||||
}
|
||||
|
||||
func (s *audioStubVoice) TTS(ctx context.Context, text string) (voice.Audio, error) {
|
||||
s.ttsCalled = true
|
||||
s.gotText = text
|
||||
return s.ttsAudio, s.ttsErr
|
||||
}
|
||||
|
||||
func TestHandleTranscribe(t *testing.T) {
|
||||
env := newEnv(t, nil, false)
|
||||
sv := &audioStubVoice{sttText: "你好世界"}
|
||||
env.srv.SetVoice(sv)
|
||||
|
||||
token := env.token(t, auth.RoleOwner)
|
||||
resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/transcribe", map[string]any{
|
||||
"audio": base64.StdEncoding.EncodeToString([]byte("raw")),
|
||||
"format": "wav",
|
||||
}, token)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", resp.StatusCode, body)
|
||||
}
|
||||
if !sv.sttCalled {
|
||||
t.Fatal("STT not called")
|
||||
}
|
||||
if string(sv.gotAudio.Data) != "raw" || sv.gotAudio.Format != "wav" {
|
||||
t.Errorf("audio = %+v, want format wav / data raw", sv.gotAudio)
|
||||
}
|
||||
env2 := decodeBody(t, body)
|
||||
if got := string(env2["data"]); got == "" {
|
||||
t.Errorf("data envelope empty: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTranscribe_AuthRequired(t *testing.T) {
|
||||
env := newEnv(t, nil, false)
|
||||
sv := &audioStubVoice{sttText: "x"}
|
||||
env.srv.SetVoice(sv)
|
||||
|
||||
resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/transcribe", map[string]any{
|
||||
"audio": base64.StdEncoding.EncodeToString([]byte("raw")),
|
||||
}, "")
|
||||
if resp.StatusCode != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want 401 (body=%s)", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTranscribe_EmptyAudio(t *testing.T) {
|
||||
env := newEnv(t, nil, false)
|
||||
sv := &audioStubVoice{sttText: "x"}
|
||||
env.srv.SetVoice(sv)
|
||||
|
||||
token := env.token(t, auth.RoleOwner)
|
||||
resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/transcribe", map[string]any{
|
||||
"audio": base64.StdEncoding.EncodeToString([]byte("")),
|
||||
}, token)
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400 (body=%s)", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleSpeech(t *testing.T) {
|
||||
env := newEnv(t, nil, false)
|
||||
sv := &audioStubVoice{ttsAudio: voice.Audio{Format: "wav", Data: []byte("audio-bytes")}}
|
||||
env.srv.SetVoice(sv)
|
||||
|
||||
token := env.token(t, auth.RoleOwner)
|
||||
resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/speech", map[string]any{
|
||||
"text": "你好",
|
||||
}, token)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", resp.StatusCode, body)
|
||||
}
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "audio/wav" {
|
||||
t.Errorf("Content-Type = %q, want audio/wav", ct)
|
||||
}
|
||||
if string(body) != "audio-bytes" {
|
||||
t.Errorf("body = %q, want audio-bytes", body)
|
||||
}
|
||||
if !sv.ttsCalled || sv.gotText != "你好" {
|
||||
t.Errorf("TTS got text = %q (called=%v), want 你好", sv.gotText, sv.ttsCalled)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleSpeech_EmptyText(t *testing.T) {
|
||||
env := newEnv(t, nil, false)
|
||||
env.srv.SetVoice(&audioStubVoice{})
|
||||
|
||||
token := env.token(t, auth.RoleOwner)
|
||||
resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/speech", map[string]any{
|
||||
"text": " ",
|
||||
}, token)
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400 (body=%s)", resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
@@ -385,6 +385,12 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.Handle("GET /api/v1/sessions", readAuth(http.HandlerFunc(s.handleListSessions)))
|
||||
mux.Handle("GET /api/v1/personas", readAuth(http.HandlerFunc(s.handleListPersonas)))
|
||||
|
||||
// Audio (voice) STT/TTS. Owner-only (the owner's private voice assistant),
|
||||
// audited. These are the REST surface for native clients to upload audio and
|
||||
// fetch synthesized speech, bypassing the 4KB WebSocket frame limit.
|
||||
mux.Handle("POST /api/v1/audio/transcribe", s.auth.Authorize(auth.RoleOwner)(http.HandlerFunc(s.handleTranscribe)))
|
||||
mux.Handle("POST /api/v1/audio/speech", s.auth.Authorize(auth.RoleOwner)(http.HandlerFunc(s.handleSpeech)))
|
||||
|
||||
// Session/model reads and writes are the owner's private history; owner/admin only
|
||||
// (chat uses the same roles). Session metadata in GET /sessions stays readAuth for
|
||||
// any authenticated role, but the message content (and thinking) is owner/admin-only
|
||||
|
||||
@@ -26,12 +26,14 @@ type VoiceService interface {
|
||||
// voiceInputRequest is the decoded payload of a voice_input frame. Audio is a
|
||||
// base64 (or data: URI) audio blob; Format is its encoding ("wav"/"mp3"/"ogg");
|
||||
// Text overrides STT when present (a client that already transcribed can skip
|
||||
// recognition); SessionID defaults to "web".
|
||||
// recognition); SessionID defaults to "web"; DeviceID is the client-produced
|
||||
// end identifier (design/16 §4.5) threaded into the persisted voice turn.
|
||||
type voiceInputRequest struct {
|
||||
Audio string `json:"audio"`
|
||||
Format string `json:"format"`
|
||||
Text string `json:"text"`
|
||||
SessionID string `json:"sessionId"`
|
||||
DeviceID string `json:"deviceId,omitempty"`
|
||||
}
|
||||
|
||||
// SetVoice injects the voice STT/TTS service used by the voice_input WS route.
|
||||
@@ -66,6 +68,9 @@ func (c *wsConn) handleVoiceInput(in wsInbound) {
|
||||
// carrying the base64 audio plus the recognized transcript and assistant reply.
|
||||
func (h *hub) runVoiceTurn(c *wsConn, req voiceInputRequest) {
|
||||
ctx := context.Background()
|
||||
// Thread the client's deviceID so the persisted voice turn is tagged with the
|
||||
// sending end (design/16 §4.5), mirroring ws_hub.go:240.
|
||||
ctx = withDeviceID(ctx, req.DeviceID)
|
||||
if req.SessionID == "" {
|
||||
req.SessionID = "web"
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user