From 03d7050e1e81f29262e6cb146c6b9d1219fcd067 Mon Sep 17 00:00:00 2001 From: Aska Date: Tue, 8 Sep 2026 06:48:15 +0800 Subject: [PATCH] =?UTF-8?q?feat(voice):=20=E7=99=BE=E7=82=BC=E5=A3=B0?= =?UTF-8?q?=E9=9F=B3=E5=A4=8D=E5=88=BBcreate=5Fvoice=20+=20TTS=20instructi?= =?UTF-8?q?on=E6=83=85=E7=BB=AA=E9=A9=B1=E5=8A=A8(=E4=B8=BBTTS-=E6=98=94?= =?UTF-8?q?=E6=B6=9F=E5=A4=8D=E5=88=BB=E9=9F=B3=E8=89=B2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 13 +- Docs/api-client.md | 7 +- internal/config/config.go | 9 + internal/config/config_test.go | 23 ++ internal/runtime/multimodal/voice/chain.go | 19 +- internal/runtime/multimodal/voice/cloud.go | 6 + .../runtime/multimodal/voice/dashscope.go | 183 ++++++++++++-- internal/runtime/multimodal/voice/service.go | 6 + internal/runtime/multimodal/voice/voice.go | 28 ++ .../multimodal/voice/voice_clone_test.go | 239 ++++++++++++++++++ internal/services/gateway/audio.go | 60 ++++- internal/services/gateway/audio_test.go | 110 ++++++++ internal/services/gateway/extra_test.go | 5 +- internal/services/gateway/server.go | 1 + internal/services/gateway/voice.go | 7 +- 15 files changed, 686 insertions(+), 30 deletions(-) create mode 100644 internal/runtime/multimodal/voice/voice_clone_test.go diff --git a/.env.example b/.env.example index 7021f72..9eff37e 100644 --- a/.env.example +++ b/.env.example @@ -78,8 +78,17 @@ STT_PROVIDER=dashscope # VOICE_TTS_VOICE_ID=longxiaochun_v3 # VOICE_TTS_FORMAT=wav # VOICE_TTS_SAMPLE_RATE=24000 -# 声音复刻（响昔涟音色）：先用参考音频（公网 URL）在百炼「声音复刻」建自定义音色， -# 再把返回的 voice_id 填到 VOICE_TTS_VOICE_ID。TODO：待公网参考音频；本项目不做 create_voice。 +# 情绪/语气指令（TTS instruction）：只对「复刻/设计音色」生效（可任意指令，如 +# "语气开朗俏皮，像女朋友撒娇"），系统音色 longxiaochun_v3 必须用固定格式故不传。 +# 长度≤100 字符（汉字记 2）。也可在 POST /api/v1/audio/speech 用 instruction 按请求覆盖。 +# VOICE_TTS_INSTRUCTION=语气开朗俏皮，像女朋友撒娇 +# 声音复刻前缀（create_voice 的 prefix，≤10 字母/数字，空则用 cyrene）。target_model +# 固定为 VOICE_TTS_MODEL（须与合成模型一致）。 +# VOICE_CLONE_PREFIX=cyrene +# 声音复刻（响昔涟音色）：先用参考音频（公网 URL）调 POST /api/v1/audio/voice-clone +# 建自定义音色，再把返回的 voice_id 填到 VOICE_TTS_VOICE_ID。 +# ⚠️ 建音色必须公网可访问的参考音频 URL（create_voice 只收 input.url，不收 base64）； +# 待主人提供参考音频公网 URL 方能真建音色。 # ---- 后台思考 ---- ENABLE_BACKGROUND_THINKING=true diff --git a/Docs/api-client.md b/Docs/api-client.md index 3b32c1e..cd030d2 100644 --- a/Docs/api-client.md +++ b/Docs/api-client.md @@ -30,11 +30,13 @@ | 7 | `/api/v1/sessions/{id}` | DELETE | JWT(owner/admin) | ✅ | **删除次级会话及其消息** → `{data:{deleted:}}`；主会话不可删回 `403 forbidden`；未注入 HistoryStore 回 `501` | | 8 | `/api/v1/personas` | GET | JWT | ✅ | `{data:[persona...]}` | | 9 | `/api/v1/audio/transcribe` | POST | JWT(owner) | ✅ | **语音→文本**：body `{audio:"",format?,deviceId?}` → `{data:{text}}`（走后端 STT，百炼 `qwen-audio-3.0-asr-flash`） | -| 10 | `/api/v1/audio/speech` | POST | JWT(owner) | ✅ | **文本→音频**：body `{text,voice?,format?}` → `200` 音频字节（`Content-Type: audio/`，非 JSON envelope，走后端 TTS，百炼 `cosyvoice-v3-flash`） | +| 10 | `/api/v1/audio/speech` | POST | JWT(owner) | ✅ | **文本→音频**：body `{text,voice?,format?,instruction?}` → `200` 音频字节（`Content-Type: audio/`，非 JSON envelope，走后端 TTS，百炼 `cosyvoice-v3-flash`，复刻音色可带 `instruction` 情绪指令） | +| 11 | `/api/v1/audio/voice-clone` | POST | JWT(owner) | ✅ | **声音复刻建音色**：body `{url:"<公网参考音频URL>",prefix?}` → `{data:{voiceId}}`（走百炼 `voice-enrollment` create_voice） | > **语音 REST 源（`/api/v1/audio/*`，均 owner-only + 审计）**：原生/移动端用它上传/获取音频，**绕开 WS 4KB 帧限**。 > - `POST /api/v1/audio/transcribe`：body `{audio:"", format?:"wav", deviceId?}` → `200 {data:{text}}`。`audio` 为 base64（或 data: URI）；`format` 为编码（`wav`/`mp3`/`ogg`，缺省 `wav`）；空 audio 回 `400`；未注入语音服务 `500 voice_unavailable`、识别失败 `500 stt_failed`。 -> - `POST /api/v1/audio/speech`：body `{text, voice?, format?}` → `200` 原始音频字节（`Content-Type: audio/`），**非 JSON envelope**。`voice`/`format` 目前遵循服务端 `VOICE_TTS_VOICE_ID`/`VOICE_TTS_FORMAT` 配置（后续支持按请求覆盖）；空 text 回 `400`；未注入语音服务 `500 voice_unavailable`、合成失败 `500 tts_failed`、静默空音频 `500 tts_empty`。**声音复刻（响昔涟音色）**：先用参考音频（公网 URL）在百炼「声音复刻」建自定义音色，再把返回的 `voice_id` 配到 `VOICE_TTS_VOICE_ID`（本项目不做 create_voice）。 +> - `POST /api/v1/audio/speech`：body `{text, voice?, format?, instruction?}` → `200` 原始音频字节（`Content-Type: audio/`），**非 JSON envelope**。`voice`/`format` 目前遵循服务端 `VOICE_TTS_VOICE_ID`/`VOICE_TTS_FORMAT` 配置（后续支持按请求覆盖）；`instruction` 为**可选** per-request 情绪/语气指令（如 `"语气开朗俏皮，像女朋友撒娇"`），只对**复刻/设计音色**生效——**系统音色 `longxiaochun_v3` 被服务端忽略，永不发送自由指令**（CosyVoice 系统音色需固定格式）；长度 ≤100 字符（汉字记 2）；未传时回退到服务端 `VOICE_TTS_INSTRUCTION`。空 text 回 `400`；未注入语音服务 `500 voice_unavailable`、合成失败 `500 tts_failed`、静默空音频 `500 tts_empty`。 +> - `POST /api/v1/audio/voice-clone`：body `{url:"<公网参考音频URL>", prefix?:"cyrene"}` → `200 {data:{voiceId}}`。`url` 为**公网可访问的参考音频 URL**（create_voice **只收 `input.url`，不收 base64**），音频要求 10~20s 连续清晰朗读、≤10MB、≥16kHz；`prefix` 为音色名前缀（**≤10 字母/数字**，缺省 `cyrene`）。返回的 `voiceId` 填到 `VOICE_TTS_VOICE_ID` 即可用该复刻音色合成（`target_model` 固定为 `VOICE_TTS_MODEL`，须与合成模型一致）。**URL 不写审计/日志（红线，绝不打日志）**。空 url 回 `400`；未注入语音服务 `500 voice_unavailable`；建音色失败 `500 voice_clone_failed`。**⚠️ 真建音色需主人提供公网参考音频 URL（当前无，断言待提供）；且需北京地域 API Key（CosyVoice 声音复刻仅华北2北京地域）。** ## 2. REST 端点（管理，`/api/v1/admin/*`） @@ -202,6 +204,7 @@ - 语音：上行 `voice_input`（base64 音频或已转写 text），下行 `voice_output`（合成音频 base64 + text）。 ## 5. 变更记录 +- `v0.19`（**百炼声音复刻 create_voice + TTS instruction 情绪驱动，主 TTS 用昔涟复刻音色**）：`POST /api/v1/audio/voice-clone`（**新增**，owner-only + 审计，body `{url:"<公网参考音频URL>", prefix?}` → `{data:{voiceId}}`）——走后端 `voice.DashscopeCloner`（百炼 `voice-enrollment` create_voice，`POST {base}/services/audio/tts/customization`，`input{action:create_voice, target_model:VOICE_TTS_MODEL, prefix, url}`，**只收公网 URL 不收 base64**，返回 `output.voice_id`；**URL 不打日志不写审计**）。`POST /api/v1/audio/speech` 新增**可选 `instruction`**（per-request 情绪/语气指令，如 `"语气开朗俏皮，像女朋友撒娇"`），经 ctx 透传给 TTS adapter；**只对复刻/设计音色生效**（`cosyvoice-v3-flash` 复刻音色可任意指令；系统音色 `longxiaochun_v3` 需固定格式，服务端 `canInstruct` 判定后**永不发送**）。config 新增 `VOICE_TTS_INSTRUCTION`（默认 `""`）、`VOICE_CLONE_PREFIX`（默认 `cyrene`，≤10 字母/数字）；`voice.Chain`/`voice.Service` 新增 `CloneVoice`，`VoiceCloner` 接口（cloud rung 提供；未配 cloud 回 `voice_unavailable`）。**⚠️ 真建昔涟音色仍待主人提供公网参考音频 URL（当前无）**，且 CosyVoice 声音复刻仅华北2北京地域（确认 key 地域）；TTS instruction 用现有复刻音色或先仅单测请求体，避免改系统音色行为。 - `v0.18`（**后端语音接通阿里百炼 STT + TTS 主链路**）：`POST /api/v1/audio/transcribe`（**新增**，owner + 审计，body `{audio,format?,deviceId?}` → `{data:{text}}`，走 STT）；`POST /api/v1/audio/speech`（**新增**，owner + 审计，body `{text,voice?,format?}` → 音频字节 `audio/`，走 TTS）——**REST 语音源**，原生/移动端经它绕开 WS 4KB 帧限上传/获取音频；WS `voice_input` 上行新增 **`deviceId`**（随语音轮次落库，design/16 §4.5）。后端 `internal/runtime/multimodal/voice` 的 cloud rung 由空 endpoint stub 改为**真实百炼 adapter**：STT `qwen-audio-3.0-asr-flash`（base64 原始字节→文本）、TTS `cosyvoice-v3-flash`（按 `voice_id` 合成，取非流式返回的 24h 音频 URL 下载字节）。**实测**（`DASHSCOPE_API_KEY` 实跑）：TTS「你好，我是昔涟」→ 106124 字节 wav；把该 wav 喂给 STT → 文本「你好，我是西联。」（TTS 发音把「昔涟」读得近似「西联」，STT 识别正确）；语音链 (cloud→…→silent) 返回真实字节/文本。**已实测修正**：`cosyvoice-v3.5-plus`/`longxiaochun`（v2 音色）走 HTTP `SpeechSynthesizer` 会回 `418`（引擎拒绝），改用 **`cosyvoice-v3-flash` + `longxiaochun_v3`**（v3 音色，各模型音色不可混用）。**声音复刻（响昔涟音色）仍待**：需参考音频（公网 URL）在百炼建自定义音色后把 `voice_id` 填入 `VOICE_TTS_VOICE_ID`（本项目不做 create_voice）。config 新增 `DASHSCOPE_BASE_URL`/`VOICE_STT_\*`/`VOICE_TTS_\*`（`VOICE_TTS_SAMPLE_RATE` 默认 24000）。 - `v0.17`（系统设置安全与存储：吊销全部 API key / 轮换 JWT 密钥 / 备份 / 恢复，design/11 §4 + design/10 §3 模块14）：新增 `POST /api/v1/admin/system/{revoke-keys,rotate-jwt,backup,restore}`（均 **owner-only** + 审计）。`revoke-keys` 走 `auth.APIKey.RevokeAll()`（清空 key 表，返回 `{revoked:n}`）；`rotate-jwt` 走 `auth.JWTService.RotateSecret`（≥32 位，**新密钥不回显/不落日志**，现有 token 立即失效）；`backup` 用 SQLite `VACUUM INTO` 快照到 `/backup-.db`（含已提交 WAL 内容）返回 `{path,size}`；`restore` 先校验路径在 dataDir 内（**`filepath.Rel` 防路径穿越**，逃逸回 `400`）→ **先自动备份当前 DB** → 替换 DB 文件 → 返回 `{restored:true,backup,note:"需重启后生效"}`（运行进程持有 DB 句柄，热替换需重启可见）。gateway 新增 `Deps.DatabasePath`（composition root 传入实际打开的 SQLite 路径）。`编辑环境变量 / 重启服务 / 检查更新` 仍为后置，未实现、不伪造。 - `v0.17`（KOOK 适配器面板 + 每适配器专属页，design/17 §2）：`GET /api/v1/admin/channels/{platform}/view`、`GET /api/v1/admin/channels/{platform}/friends?type=`、`POST /api/v1/admin/channels/{platform}/friend/handle`、`POST /api/v1/admin/channels/{platform}/message/test`（**新增**，均 owner-only + 审计，带可选 `?accountId=`，仅 `kook` 实现，其它平台 `501`）；`GET /api/v1/admin/channels` 每条账号附 `capabilities`（面板按平台渲染专属卡片）。KOOK 面板：好友/申请、服务器/频道/成员、私信会话、发消息测试——经适配器 REST（`Authorization: Bot `）拉真实数据，token 不打印不落日志，子项失败记入 `errors` 不伪造。 diff --git a/internal/config/config.go b/internal/config/config.go index bdef200..13ab830 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -126,6 +126,15 @@ type VoiceConfig struct { TTSVoiceID string `env:"VOICE_TTS_VOICE_ID" default:"longxiaochun_v3"` TTSFormat string `env:"VOICE_TTS_FORMAT" default:"wav"` TTSSampleRate int `env:"VOICE_TTS_SAMPLE_RATE" default:"24000"` + // VoiceTTSInstruction is the emotion/tone instruction driving a cloned/designed + // TTS voice (e.g. "语气开朗俏皮，像女朋友撒娇"). cosyvoice v3 system voices (the + // long*_v3 family) require a fixed instruction format, so a free-form instruction + // is never sent for them — only for a cloned/designed voice (声音复刻). + VoiceTTSInstruction string `env:"VOICE_TTS_INSTRUCTION" default:""` + // VoiceClonePrefix is the name prefix used when creating a custom voice (声音复刻 + // create_voice). It must be ≤10 alphanumeric chars; the empty value falls back + // to "cyrene". target_model stays TTSModel (must match the synthesis model). + VoiceClonePrefix string `env:"VOICE_CLONE_PREFIX" default:"cyrene"` } // BackgroundConfig holds the autonomous thinking settings. QuietStartHour and diff --git a/internal/config/config_test.go b/internal/config/config_test.go index be81912..d6b3764 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -190,6 +190,12 @@ func TestLoad_VoiceDefaults(t *testing.T) { if cfg.Voice.TTSSampleRate != 24000 { t.Errorf("TTSSampleRate = %d, want 24000", cfg.Voice.TTSSampleRate) } + if cfg.Voice.VoiceTTSInstruction != "" { + t.Errorf("VoiceTTSInstruction = %q, want empty (default)", cfg.Voice.VoiceTTSInstruction) + } + if cfg.Voice.VoiceClonePrefix != "cyrene" { + t.Errorf("VoiceClonePrefix = %q, want cyrene (default)", cfg.Voice.VoiceClonePrefix) + } } func TestLoad_VoiceEnvOverrides(t *testing.T) { @@ -209,6 +215,23 @@ func TestLoad_VoiceEnvOverrides(t *testing.T) { } } +func TestLoad_VoiceCloneInstructionEnvOverrides(t *testing.T) { + t.Setenv("JWT_SECRET", strings.Repeat("s", 40)) + t.Setenv("VOICE_TTS_INSTRUCTION", "语气开朗俏皮，像女朋友撒娇") + t.Setenv("VOICE_CLONE_PREFIX", "xilian") + + cfg, err := Load("") + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if cfg.Voice.VoiceTTSInstruction != "语气开朗俏皮，像女朋友撒娇" { + t.Errorf("VoiceTTSInstruction = %q, want the env instruction", cfg.Voice.VoiceTTSInstruction) + } + if cfg.Voice.VoiceClonePrefix != "xilian" { + t.Errorf("VoiceClonePrefix = %q, want xilian", cfg.Voice.VoiceClonePrefix) + } +} + func TestParseDotEnv_QuotedAndExport(t *testing.T) { path := writeEnvFile(t, `export FOO=bar BAZ='quoted value' diff --git a/internal/runtime/multimodal/voice/chain.go b/internal/runtime/multimodal/voice/chain.go index b7b0273..cc3d2e8 100644 --- a/internal/runtime/multimodal/voice/chain.go +++ b/internal/runtime/multimodal/voice/chain.go @@ -12,7 +12,8 @@ import ( // provider is unavailable (nil direction) or fails. It implements STT and TTS so // it can be used anywhere a single provider is expected. type Chain struct { - steps []Provider + steps []Provider + cloner VoiceCloner } // NewChain builds a degrading chain from the given providers in the order they @@ -42,6 +43,16 @@ func (c *Chain) STT(ctx context.Context, audio Audio) (string, error) { return "", fmt.Errorf("voice: %w: %v", ErrProviderFailed, lastErr) } +// CloneVoice creates a custom voice via the cloud provider's voice-enrollment +// endpoint (声音复刻 create_voice). A chain without a cloud rung returns +// ErrNotConfigured. It is not a degradation step — the caller gets one result. +func (c *Chain) CloneVoice(ctx context.Context, referenceURL, prefix string) (string, error) { + if c.cloner == nil { + return "", ErrNotConfigured + } + return c.cloner.CloneVoice(ctx, referenceURL, prefix) +} + // TTS walks the chain for synthesized audio. func (c *Chain) TTS(ctx context.Context, text string) (Audio, error) { if len(c.steps) == 0 { @@ -98,8 +109,10 @@ func New(cfg config.VoiceConfig, opts ...Option) *Chain { } steps := make([]Provider, 0, 5) + var cloner VoiceCloner if voiceConfigured(cfg) { steps = append(steps, CloudProvider(newDashboardSTT(cfg), newDashboardTTS(cfg))) + cloner = newDashboardCloner(cfg) } switch cfg.TTSProvider { case "edge-tts", "edge": @@ -108,7 +121,9 @@ func New(cfg config.VoiceConfig, opts ...Option) *Chain { steps = append(steps, EspeakProvider(newEspeak())) } steps = append(steps, SilentProvider()) - return NewChain(steps...) + chain := NewChain(steps...) + chain.cloner = cloner + return chain } // voiceConfigured reports whether any cloud voice credential or model is set, so diff --git a/internal/runtime/multimodal/voice/cloud.go b/internal/runtime/multimodal/voice/cloud.go index 20e5eb3..760cbd5 100644 --- a/internal/runtime/multimodal/voice/cloud.go +++ b/internal/runtime/multimodal/voice/cloud.go @@ -212,6 +212,12 @@ func newDashboardTTS(cfg config.VoiceConfig) TTS { return NewDashscopeTTS(cfg) } +// newDashboardCloner returns the cloud voice-enrollment cloner for a DashScope +// config, used by the create_voice (声音复刻) endpoint. +func newDashboardCloner(cfg config.VoiceConfig) VoiceCloner { + return NewDashscopeCloner(cfg) +} + // newEdgeTTS returns the edge-tts adapter point (not yet implemented). func newEdgeTTS() TTS { return notImplementedTTS{} } diff --git a/internal/runtime/multimodal/voice/dashscope.go b/internal/runtime/multimodal/voice/dashscope.go index 14c38c5..a0831a3 100644 --- a/internal/runtime/multimodal/voice/dashscope.go +++ b/internal/runtime/multimodal/voice/dashscope.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "strings" + "unicode" "git.yeij.top/AskaEth/Cyrene_v2/internal/config" ) @@ -33,8 +34,14 @@ const ( dashscopeSTTPath = "/services/aigc/multimodal-generation/generation" // dashscopeTTSPath is the SpeechSynthesizer service path (TTS). dashscopeTTSPath = "/services/audio/tts/SpeechSynthesizer" + // dashscopeClonePath is the voice-customization (声音复刻 create_voice) path. + dashscopeClonePath = "/services/audio/tts/customization" // defaultDashscopeBaseURL is the Bailian API root when none is configured. defaultDashscopeBaseURL = "https://dashscope.aliyuncs.com/api/v1" + // defaultClonePrefix is the voice-name prefix when none is supplied. + defaultClonePrefix = "cyrene" + // voiceEnrollmentModel is the DashScope voice-enrollment model ("create_voice"). + voiceEnrollmentModel = "voice-enrollment" ) // DashscopeSTT transcribes audio via DashScope multimodal-generation. @@ -114,13 +121,14 @@ func (s *DashscopeSTT) STT(ctx context.Context, audio Audio) (string, error) { // DashscopeTTS synthesizes speech video via DashScope SpeechSynthesizer. type DashscopeTTS struct { - endpoint string - apiKey string - model string - voiceID string - format string - sampleRate int - client *http.Client + endpoint string + apiKey string + model string + voiceID string + format string + sampleRate int + instruction string + client *http.Client } // NewDashscopeTTS builds the DashScope TTS adapter from cfg. Missing model / @@ -134,13 +142,14 @@ func NewDashscopeTTS(cfg config.VoiceConfig) *DashscopeTTS { sampleRate = 24000 } return &DashscopeTTS{ - endpoint: dashscopeEndpoint(cfg.DashscopeBaseURL) + dashscopeTTSPath, - apiKey: cfg.DashscopeAPIKey, - model: model, - voiceID: voiceID, - format: format, - sampleRate: sampleRate, - client: defaultHTTPClient(), + endpoint: dashscopeEndpoint(cfg.DashscopeBaseURL) + dashscopeTTSPath, + apiKey: cfg.DashscopeAPIKey, + model: model, + voiceID: voiceID, + format: format, + sampleRate: sampleRate, + instruction: cfg.VoiceTTSInstruction, + client: defaultHTTPClient(), } } @@ -157,7 +166,7 @@ func (t *DashscopeTTS) TTS(ctx context.Context, text string) (Audio, error) { if err := ctx.Err(); err != nil { return Audio{}, err } - payload, err := json.Marshal(dashscopeTTSRequest{ + req := dashscopeTTSRequest{ Model: t.model, Input: dashscopeTTSInput{ Text: text, @@ -165,7 +174,11 @@ func (t *DashscopeTTS) TTS(ctx context.Context, text string) (Audio, error) { Format: t.format, SampleRate: t.sampleRate, }, - }) + } + if instruction := t.EffectiveInstruction(ctx); instruction != "" { + req.Input.Instruction = instruction + } + payload, err := json.Marshal(req) if err != nil { return Audio{}, fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err) } @@ -181,6 +194,29 @@ func (t *DashscopeTTS) TTS(ctx context.Context, text string) (Audio, error) { return Audio{Format: t.format, Data: data}, nil } +// EffectiveInstruction returns the instruction to send for this synthesis: a +// per-request override threaded through ctx wins, else the configured default. An +// empty result means the request carries no instruction. A system voice never gets +// a free-form instruction (see canInstruct), because cosyvoice v3 system voices +// require a fixed instruction format. +func (t *DashscopeTTS) EffectiveInstruction(ctx context.Context) string { + if !canInstruct(t.voiceID) { + return "" + } + if override := TTSInstructionFromContext(ctx); override != "" { + return override + } + return t.instruction +} + +// canInstruct reports whether voiceID accepts a free-form instruction. cosyvoice v3 +// system voices (the long*_v3 family, e.g. longxiaochun_v3) require a fixed +// instruction format; a cloned/designed voice (声音复刻, whose id never ends in +// "_v3") accepts an arbitrary instruction. +func canInstruct(voiceID string) bool { + return !strings.HasSuffix(voiceID, "_v3") +} + // audioBytes resolves the synthesized audio from the DashScope response: a 24h // URL is downloaded, or an inline base64 blob is decoded. func (t *DashscopeTTS) audioBytes(ctx context.Context, body []byte) ([]byte, error) { @@ -227,6 +263,95 @@ func (t *DashscopeTTS) download(ctx context.Context, url string) ([]byte, error) return data, nil } +// DashscopeCloner creates custom TTS voices via the DashScope voice-enrollment +// API (声音复刻 create_voice). The reference audio must be a public URL — the +// create_voice action only accepts input.url (no base64). The returned voice_id is +// bound to TTSModel, so target_model must match the model used at synthesis time. +type DashscopeCloner struct { + endpoint string + apiKey string + model string + client *http.Client +} + +// NewDashscopeCloner builds the DashScope voice cloner from cfg. Missing model / +// base URL fall back to the DashScope defaults; the model tracks TTSModel so the +// cloned voice is usable by the TTS rung (target_model must match). +func NewDashscopeCloner(cfg config.VoiceConfig) *DashscopeCloner { + return &DashscopeCloner{ + endpoint: dashscopeEndpoint(cfg.DashscopeBaseURL) + dashscopeClonePath, + apiKey: cfg.DashscopeAPIKey, + model: firstNonEmpty(cfg.TTSModel, "cosyvoice-v3-flash"), + client: defaultHTTPClient(), + } +} + +// CloneVoice submits a create_voice request and returns the created voice_id. It +// is the adapter behind the REST /api/v1/audio/voice-clone endpoint. A missing API +// key returns ErrNotConfigured; an empty URL or an invalid prefix is rejected. +func (d *DashscopeCloner) CloneVoice(ctx context.Context, referenceURL, prefix string) (string, error) { + if d.apiKey == "" { + return "", ErrNotConfigured + } + referenceURL = strings.TrimSpace(referenceURL) + if referenceURL == "" { + return "", fmt.Errorf("voice: %w: reference audio url is required", ErrProviderFailed) + } + prefix, err := normalizeClonePrefix(prefix) + if err != nil { + return "", err + } + if err := ctx.Err(); err != nil { + return "", err + } + payload, err := json.Marshal(dashscopeCloneRequest{ + Model: voiceEnrollmentModel, + Input: dashscopeCloneInput{ + Action: "create_voice", + TargetModel: d.model, + Prefix: prefix, + URL: referenceURL, + }, + }) + if err != nil { + return "", fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err) + } + body, err := dashscopePost(ctx, d.client, d.apiKey, d.endpoint, payload, "") + if err != nil { + return "", err + } + var resp dashscopeCloneResponse + if err := json.Unmarshal(body, &resp); err != nil { + return "", fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err) + } + if resp.Message != "" { + return "", fmt.Errorf("voice: %w: %s", ErrProviderFailed, resp.Message) + } + if resp.Output.VoiceID == "" { + return "", fmt.Errorf("voice: %w: no voice_id in response", ErrProviderFailed) + } + return resp.Output.VoiceID, nil +} + +// normalizeClonePrefix trims and validates the voice-name prefix. An empty value +// falls back to defaultClonePrefix; the result must be ≤10 letter/digit chars +// (the voice-enrollment prefix constraint). +func normalizeClonePrefix(prefix string) (string, error) { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + return defaultClonePrefix, nil + } + if len(prefix) > 10 { + return "", fmt.Errorf("voice: %w: prefix %q exceeds 10 chars", ErrProviderFailed, prefix) + } + for _, r := range prefix { + if !unicode.IsLetter(r) && !unicode.IsDigit(r) { + return "", fmt.Errorf("voice: %w: prefix must be alphanumeric", ErrProviderFailed) + } + } + return prefix, nil +} + // dashscopeEndpoint returns the DashScope API base with a trailing slash trimmed, // defaulting to the public Bailian root when none is configured. func dashscopeEndpoint(baseURL string) string { @@ -394,4 +519,30 @@ type dashscopeTTSInput struct { Voice string `json:"voice"` Format string `json:"format"` SampleRate int `json:"sample_rate"` + // Instruction is the CosyVoice emotion/tone instruction; omitempty keeps a + // system voice (which must use a fixed instruction) free of a free-form one. + Instruction string `json:"instruction,omitempty"` +} + +// dashscopeCloneRequest is the voice-enrollment create_voice request body. +type dashscopeCloneRequest struct { + Model string `json:"model"` + Input dashscopeCloneInput `json:"input"` +} + +// dashscopeCloneInput carries the create_voice action parameters. +type dashscopeCloneInput struct { + Action string `json:"action"` + TargetModel string `json:"target_model"` + Prefix string `json:"prefix"` + URL string `json:"url"` +} + +// dashscopeCloneResponse is the voice-enrollment response envelope. A top-level +// message marks a model-level error surfaced even on an HTTP 200. +type dashscopeCloneResponse struct { + Output struct { + VoiceID string `json:"voice_id"` + } `json:"output"` + Message string `json:"message"` } diff --git a/internal/runtime/multimodal/voice/service.go b/internal/runtime/multimodal/voice/service.go index fa6f347..3bfa295 100644 --- a/internal/runtime/multimodal/voice/service.go +++ b/internal/runtime/multimodal/voice/service.go @@ -126,6 +126,12 @@ func (s *Service) TTS(ctx context.Context, text string) (Audio, error) { return res.audio, res.err } +// CloneVoice creates a custom voice, delegating to the underlying chain. It is a +// one-shot admin operation (not a worker-pool job), so it runs directly. +func (s *Service) CloneVoice(ctx context.Context, referenceURL, prefix string) (string, error) { + return s.chain.CloneVoice(ctx, referenceURL, prefix) +} + // submit enqueues j and blocks until the result or a cancellation. func (s *Service) submit(ctx context.Context, j job) (jobResult, error) { select { diff --git a/internal/runtime/multimodal/voice/voice.go b/internal/runtime/multimodal/voice/voice.go index 3637dca..b6229a3 100644 --- a/internal/runtime/multimodal/voice/voice.go +++ b/internal/runtime/multimodal/voice/voice.go @@ -33,3 +33,31 @@ type TTS interface { // honored. TTS(ctx context.Context, text string) (Audio, error) } + +// VoiceCloner creates a custom voice (声音复刻) from a public reference audio URL. +// A nil-cloner provider (no cloud rung configured) returns ErrNotConfigured. +type VoiceCloner interface { + // CloneVoice creates a custom voice and returns its voice_id. referenceURL is + // a public audio URL; prefix is the voice-name prefix (≤10 alphanumeric chars). + CloneVoice(ctx context.Context, referenceURL, prefix string) (string, error) +} + +// ctxKeyTTSInstruction is the context key for a per-request TTS instruction. +// The TTS interface only carries (ctx, text), so an optional instruction is +// threaded through ctx to reach the adapter without widening the interface. +type ctxKeyTTSInstruction struct{} + +// WithTTSInstruction returns ctx carrying an optional per-request TTS instruction +// (emotion/tone) that overrides the configured VOICE_TTS_INSTRUCTION for one call. +func WithTTSInstruction(ctx context.Context, instruction string) context.Context { + return context.WithValue(ctx, ctxKeyTTSInstruction{}, instruction) +} + +// TTSInstructionFromContext returns the per-request TTS instruction set by +// WithTTSInstruction, or "" when none is carried. +func TTSInstructionFromContext(ctx context.Context) string { + if v, ok := ctx.Value(ctxKeyTTSInstruction{}).(string); ok { + return v + } + return "" +} diff --git a/internal/runtime/multimodal/voice/voice_clone_test.go b/internal/runtime/multimodal/voice/voice_clone_test.go new file mode 100644 index 0000000..b576617 --- /dev/null +++ b/internal/runtime/multimodal/voice/voice_clone_test.go @@ -0,0 +1,239 @@ +package voice + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/config" +) + +// TestDashscopeCloner_Ok asserts the create_voice request body shape +// (model/action/target_model/prefix/url) and the voice_id parsing. +func TestDashscopeCloner_Ok(t *testing.T) { + t.Parallel() + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/customization") { + t.Errorf("path = %q, want .../customization", r.URL.Path) + } + if r.Header.Get("Authorization") != "Bearer sk" { + t.Errorf("auth = %q, want Bearer sk", r.Header.Get("Authorization")) + } + if sse := r.Header.Get("X-DashScope-SSE"); sse != "" { + t.Errorf("sse = %q, want empty", sse) + } + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode: %v", err) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"output":{"voice_id":"cosyvoice-v3-flash-cyrene-abc123"}}`)) + })) + defer srv.Close() + + cloner := NewDashscopeCloner(config.VoiceConfig{DashscopeBaseURL: srv.URL, DashscopeAPIKey: "sk"}) + voiceID, err := cloner.CloneVoice(context.Background(), "https://example.com/ref.wav", "cyrene") + if err != nil { + t.Fatalf("CloneVoice error: %v", err) + } + if voiceID != "cosyvoice-v3-flash-cyrene-abc123" { + t.Errorf("voice_id = %q, want cosyvoice-v3-flash-cyrene-abc123", voiceID) + } + if gotBody["model"] != "voice-enrollment" { + t.Errorf("model = %v, want voice-enrollment", gotBody["model"]) + } + input := gotBody["input"].(map[string]any) + if input["action"] != "create_voice" { + t.Errorf("action = %v, want create_voice", input["action"]) + } + if input["target_model"] != "cosyvoice-v3-flash" { + t.Errorf("target_model = %v, want cosyvoice-v3-flash", input["target_model"]) + } + if input["prefix"] != "cyrene" { + t.Errorf("prefix = %v, want cyrene", input["prefix"]) + } + if input["url"] != "https://example.com/ref.wav" { + t.Errorf("url = %v, want the reference url", input["url"]) + } +} + +// TestDashscopeCloner_DefaultPrefix asserts the empty prefix falls back to the +// configured VOICE_CLONE_PREFIX default ("cyrene"). +func TestDashscopeCloner_DefaultPrefix(t *testing.T) { + t.Parallel() + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode: %v", err) + } + _, _ = w.Write([]byte(`{"output":{"voice_id":"v1"}}`)) + })) + defer srv.Close() + + cloner := NewDashscopeCloner(config.VoiceConfig{DashscopeBaseURL: srv.URL, DashscopeAPIKey: "sk", VoiceClonePrefix: "cyrene"}) + if _, err := cloner.CloneVoice(context.Background(), "https://example.com/ref.wav", ""); err != nil { + t.Fatalf("CloneVoice error: %v", err) + } + if p := gotBody["input"].(map[string]any)["prefix"]; p != "cyrene" { + t.Errorf("prefix = %v, want default cyrene", p) + } +} + +// TestDashscopeCloner_TargetModelOverride asserts target_model tracks the config +// TTSModel (must match the synthesis model). +func TestDashscopeCloner_TargetModelOverride(t *testing.T) { + t.Parallel() + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode: %v", err) + } + _, _ = w.Write([]byte(`{"output":{"voice_id":"v1"}}`)) + })) + defer srv.Close() + + cloner := NewDashscopeCloner(config.VoiceConfig{DashscopeBaseURL: srv.URL, DashscopeAPIKey: "sk", TTSModel: "cosyvoice-v3.5-plus"}) + if _, err := cloner.CloneVoice(context.Background(), "https://example.com/ref.wav", "x"); err != nil { + t.Fatalf("CloneVoice error: %v", err) + } + if tm := gotBody["input"].(map[string]any)["target_model"]; tm != "cosyvoice-v3.5-plus" { + t.Errorf("target_model = %v, want cosyvoice-v3.5-plus", tm) + } +} + +// TestDashscopeCloner_NotConfigured asserts a missing API key degrades. +func TestDashscopeCloner_NotConfigured(t *testing.T) { + t.Parallel() + cloner := NewDashscopeCloner(config.VoiceConfig{}) + if _, err := cloner.CloneVoice(context.Background(), "https://example.com/ref.wav", "x"); err != ErrNotConfigured { + t.Fatalf("err = %v, want ErrNotConfigured", err) + } +} + +// TestDashscopeCloner_EmptyURL asserts an empty reference URL is rejected. +func TestDashscopeCloner_EmptyURL(t *testing.T) { + t.Parallel() + cloner := NewDashscopeCloner(config.VoiceConfig{DashscopeBaseURL: "http://x", DashscopeAPIKey: "sk"}) + if _, err := cloner.CloneVoice(context.Background(), " ", "x"); !errors.Is(err, ErrProviderFailed) { + t.Fatalf("err = %v, want ErrProviderFailed", err) + } +} + +// TestDashscopeCloner_BadPrefix asserts a non-alphanumeric or >10-char prefix is +// rejected before any request is sent. +func TestDashscopeCloner_BadPrefix(t *testing.T) { + t.Parallel() + cloner := NewDashscopeCloner(config.VoiceConfig{DashscopeBaseURL: "http://x", DashscopeAPIKey: "sk"}) + for _, prefix := range []string{"invalid-prefix!", "this-prefix-is-way-too-long"} { + if _, err := cloner.CloneVoice(context.Background(), "https://example.com/ref.wav", prefix); !errors.Is(err, ErrProviderFailed) { + t.Errorf("prefix %q: err = %v, want ErrProviderFailed", prefix, err) + } + } +} + +// TestDashscopeCloner_ModelError asserts a top-level message in an HTTP 200 +// response is surfaced as an error (no voice_id). +func TestDashscopeCloner_ModelError(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"message":"invalid reference url"}`)) + })) + defer srv.Close() + + cloner := NewDashscopeCloner(config.VoiceConfig{DashscopeBaseURL: srv.URL, DashscopeAPIKey: "sk"}) + if _, err := cloner.CloneVoice(context.Background(), "https://example.com/ref.wav", "x"); !errors.Is(err, ErrProviderFailed) { + t.Fatalf("err = %v, want ErrProviderFailed", err) + } +} + +// TestDashscopeTTS_Instruction_ClonedVoice asserts a cloned voice carries the +// configured instruction in the request input. +func TestDashscopeTTS_Instruction_ClonedVoice(t *testing.T) { + t.Parallel() + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode: %v", err) + } + _, _ = w.Write([]byte(`{"output":{"audio":{"data":"eA=="}}}`)) + })) + defer srv.Close() + + tts := NewDashscopeTTS(config.VoiceConfig{ + DashscopeBaseURL: srv.URL, + DashscopeAPIKey: "sk", + TTSVoiceID: "cosyvoice-v3-flash-cyrene-abc123", + VoiceTTSInstruction: "语气开朗俏皮", + }) + if _, err := tts.TTS(context.Background(), "你好"); err != nil { + t.Fatalf("TTS error: %v", err) + } + if gotBody["model"] != "cosyvoice-v3-flash" { + t.Errorf("model = %v, want cosyvoice-v3-flash", gotBody["model"]) + } + input := gotBody["input"].(map[string]any) + if input["instruction"] != "语气开朗俏皮" { + t.Errorf("instruction = %v, want the configured instruction", input["instruction"]) + } +} + +// TestDashscopeTTS_NoInstruction_SystemVoice asserts a system voice (longxiaochun_v3) +// never receives a free-form instruction even when one is configured. +func TestDashscopeTTS_NoInstruction_SystemVoice(t *testing.T) { + t.Parallel() + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode: %v", err) + } + _, _ = w.Write([]byte(`{"output":{"audio":{"data":"eA=="}}}`)) + })) + defer srv.Close() + + tts := NewDashscopeTTS(config.VoiceConfig{ + DashscopeBaseURL: srv.URL, + DashscopeAPIKey: "sk", + TTSVoiceID: "longxiaochun_v3", + VoiceTTSInstruction: "语气开朗俏皮", + }) + if _, err := tts.TTS(context.Background(), "你好"); err != nil { + t.Fatalf("TTS error: %v", err) + } + input := gotBody["input"].(map[string]any) + if _, ok := input["instruction"]; ok { + t.Errorf("instruction present for system voice, want omitted: %v", input) + } +} + +// TestDashscopeTTS_Instruction_ContextOverride asserts a per-request ctx +// instruction wins over the configured one for a cloned voice. +func TestDashscopeTTS_Instruction_ContextOverride(t *testing.T) { + t.Parallel() + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode: %v", err) + } + _, _ = w.Write([]byte(`{"output":{"audio":{"data":"eA=="}}}`)) + })) + defer srv.Close() + + tts := NewDashscopeTTS(config.VoiceConfig{ + DashscopeBaseURL: srv.URL, + DashscopeAPIKey: "sk", + TTSVoiceID: "cosyvoice-v3-flash-cyrene-abc123", + VoiceTTSInstruction: "语气温柔", + }) + ctx := WithTTSInstruction(context.Background(), "语气开朗俏皮") + if _, err := tts.TTS(ctx, "你好"); err != nil { + t.Fatalf("TTS error: %v", err) + } + if instr := gotBody["input"].(map[string]any)["instruction"]; instr != "语气开朗俏皮" { + t.Errorf("instruction = %v, want the ctx override", instr) + } +} diff --git a/internal/services/gateway/audio.go b/internal/services/gateway/audio.go index acfe8b7..3dd492d 100644 --- a/internal/services/gateway/audio.go +++ b/internal/services/gateway/audio.go @@ -7,6 +7,7 @@ import ( "strings" apperr "git.yeij.top/AskaEth/Cyrene_v2/internal/errors" + "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/multimodal/voice" ) // maxAudioRequestBytes caps the JSON body a /api/v1/audio/* request may carry so @@ -23,11 +24,14 @@ type transcribeRequest struct { // speechRequest is the POST /api/v1/audio/speech body. Voice/format are accepted // for forward compatibility; synthesis currently follows the server's -// VOICE_TTS_VOICE_ID / VOICE_TTS_FORMAT config. +// VOICE_TTS_VOICE_ID / VOICE_TTS_FORMAT config. Instruction is an optional +// per-request emotion/tone override for a cloned voice (声音复刻); a system voice +// (longxiaochun_v3) ignores it and never receives a free-form instruction. type speechRequest struct { - Text string `json:"text"` - Voice string `json:"voice,omitempty"` - Format string `json:"format,omitempty"` + Text string `json:"text"` + Voice string `json:"voice,omitempty"` + Format string `json:"format,omitempty"` + Instruction string `json:"instruction,omitempty"` } // handleTranscribe runs STT on the uploaded audio and returns the recognized @@ -88,7 +92,11 @@ func (s *Server) handleSpeech(w http.ResponseWriter, r *http.Request) { writeError(w, apperr.Wrap(apperr.ErrInternal, "voice_unavailable", "voice service is not configured")) return } - audi, err := s.voice.TTS(r.Context(), req.Text) + ctx := r.Context() + if req.Instruction != "" { + ctx = voice.WithTTSInstruction(ctx, req.Instruction) + } + audi, err := s.voice.TTS(ctx, req.Text) if err != nil { s.auditClaims(r, "audio.speech", "error") s.log.Warn("voice speech", "err", err) @@ -107,6 +115,48 @@ func (s *Server) handleSpeech(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(audi.Data) } +// voiceCloneRequest is the POST /api/v1/audio/voice-clone body. URL is the public +// reference audio URL; prefix is the optional voice-name prefix (≤10 alnum chars). +// Neither is logged, and the URL is not persisted (owner + audit only, red line). +type voiceCloneRequest struct { + URL string `json:"url"` + Prefix string `json:"prefix,omitempty"` +} + +// handleVoiceClone creates a custom TTS voice from a public reference audio URL +// via the Bailian voice-enrollment API (声音复刻 create_voice) and returns the new +// voice_id. Owner-only + audited; the reference URL is never written to the audit +// trail or the log (red line). +func (s *Server) handleVoiceClone(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxAudioRequestBytes) + var req voiceCloneRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + s.auditClaims(r, "audio.voice-clone", "error") + writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "invalid request body")) + return + } + if strings.TrimSpace(req.URL) == "" { + s.auditClaims(r, "audio.voice-clone", "error") + writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "url is required")) + return + } + if s.voice == nil { + s.auditClaims(r, "audio.voice-clone", "error") + writeError(w, apperr.Wrap(apperr.ErrInternal, "voice_unavailable", "voice service is not configured")) + return + } + voiceID, err := s.voice.CloneVoice(r.Context(), req.URL, req.Prefix) + if err != nil { + s.auditClaims(r, "audio.voice-clone", "error") + // Do not log the cause: a provider error body may echo the reference URL. + s.log.Warn("voice clone failed") + writeError(w, apperr.Wrap(apperr.ErrInternal, "voice_clone_failed", "voice clone failed")) + return + } + s.auditClaims(r, "audio.voice-clone", "ok") + writeJSON(w, http.StatusOK, map[string]any{"voiceId": voiceID}) +} + // audioMime maps an audio format to its Content-Type for the speech response. func audioMime(format string) string { switch strings.ToLower(format) { diff --git a/internal/services/gateway/audio_test.go b/internal/services/gateway/audio_test.go index a7d38c5..50edabd 100644 --- a/internal/services/gateway/audio_test.go +++ b/internal/services/gateway/audio_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "net/http" + "strings" "testing" "git.yeij.top/AskaEth/Cyrene_v2/internal/auth" @@ -21,6 +22,13 @@ type audioStubVoice struct { gotText string sttCalled bool ttsCalled bool + + cloneVoiceID string + cloneErr error + cloneCalled bool + gotCloneURL string + gotClonePfx string + gotInstr string } func (s *audioStubVoice) STT(ctx context.Context, audio voice.Audio) (string, error) { @@ -32,9 +40,17 @@ func (s *audioStubVoice) STT(ctx context.Context, audio voice.Audio) (string, er func (s *audioStubVoice) TTS(ctx context.Context, text string) (voice.Audio, error) { s.ttsCalled = true s.gotText = text + s.gotInstr = voice.TTSInstructionFromContext(ctx) return s.ttsAudio, s.ttsErr } +func (s *audioStubVoice) CloneVoice(ctx context.Context, referenceURL, prefix string) (string, error) { + s.cloneCalled = true + s.gotCloneURL = referenceURL + s.gotClonePfx = prefix + return s.cloneVoiceID, s.cloneErr +} + func TestHandleTranscribe(t *testing.T) { env := newEnv(t, nil, false) sv := &audioStubVoice{sttText: "你好世界"} @@ -122,3 +138,97 @@ func TestHandleSpeech_EmptyText(t *testing.T) { t.Fatalf("status = %d, want 400 (body=%s)", resp.StatusCode, body) } } + +func TestHandleSpeech_InstructionPassthrough(t *testing.T) { + env := newEnv(t, nil, false) + sv := &audioStubVoice{ttsAudio: voice.Audio{Format: "wav", Data: []byte("audio-bytes")}} + env.srv.SetVoice(sv) + + token := env.token(t, auth.RoleOwner) + resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/speech", map[string]any{ + "text": "你好，我是昔涟", + "instruction": "语气开朗俏皮，像女朋友撒娇", + }, token) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, body = %s", resp.StatusCode, body) + } + if !sv.ttsCalled { + t.Fatal("TTS not called") + } + if sv.gotText != "你好，我是昔涟" { + t.Errorf("TTS text = %q, want 你好，我是昔涟", sv.gotText) + } + if sv.gotInstr != "语气开朗俏皮，像女朋友撒娇" { + t.Errorf("instruction threaded = %q, want the per-request instruction", sv.gotInstr) + } +} + +func TestHandleSpeech_NoInstruction(t *testing.T) { + env := newEnv(t, nil, false) + sv := &audioStubVoice{ttsAudio: voice.Audio{Format: "wav", Data: []byte("audio-bytes")}} + env.srv.SetVoice(sv) + + token := env.token(t, auth.RoleOwner) + resp, _ := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/speech", map[string]any{ + "text": "你好", + }, token) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + if sv.gotInstr != "" { + t.Errorf("instruction = %q, want empty when not supplied", sv.gotInstr) + } +} + +func TestHandleVoiceClone(t *testing.T) { + env := newEnv(t, nil, false) + sv := &audioStubVoice{cloneVoiceID: "cosyvoice-v3-flash-cyrene-abc123"} + env.srv.SetVoice(sv) + + token := env.token(t, auth.RoleOwner) + resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/voice-clone", map[string]any{ + "url": "https://example.com/ref.wav", + "prefix": "xilian", + }, token) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, body = %s", resp.StatusCode, body) + } + if !sv.cloneCalled { + t.Fatal("CloneVoice not called") + } + if sv.gotCloneURL != "https://example.com/ref.wav" || sv.gotClonePfx != "xilian" { + t.Errorf("clone url/prefix = %q / %q, want the request values", sv.gotCloneURL, sv.gotClonePfx) + } + env2 := decodeBody(t, body) + if got := string(env2["data"]); got == "" { + t.Errorf("data envelope empty: %s", body) + } + if !strings.Contains(string(env2["data"]), "cosyvoice-v3-flash-cyrene-abc123") { + t.Errorf("voiceId missing: %s", body) + } +} + +func TestHandleVoiceClone_AuthRequired(t *testing.T) { + env := newEnv(t, nil, false) + env.srv.SetVoice(&audioStubVoice{}) + + resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/voice-clone", map[string]any{ + "url": "https://example.com/ref.wav", + }, "") + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 (body=%s)", resp.StatusCode, body) + } +} + +func TestHandleVoiceClone_EmptyURL(t *testing.T) { + env := newEnv(t, nil, false) + env.srv.SetVoice(&audioStubVoice{}) + + token := env.token(t, auth.RoleOwner) + resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/voice-clone", map[string]any{ + "url": " ", + }, token) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (body=%s)", resp.StatusCode, body) + } +} diff --git a/internal/services/gateway/extra_test.go b/internal/services/gateway/extra_test.go index 181b173..ae0c863 100644 --- a/internal/services/gateway/extra_test.go +++ b/internal/services/gateway/extra_test.go @@ -1090,8 +1090,9 @@ type stubVoice struct { ttsErr error } -func (s *stubVoice) STT(_ context.Context, _ voice.Audio) (string, error) { return s.sttText, s.sttErr } -func (s *stubVoice) TTS(_ context.Context, _ string) (voice.Audio, error) { return s.tts, s.ttsErr } +func (s *stubVoice) STT(_ context.Context, _ voice.Audio) (string, error) { return s.sttText, s.sttErr } +func (s *stubVoice) TTS(_ context.Context, _ string) (voice.Audio, error) { return s.tts, s.ttsErr } +func (s *stubVoice) CloneVoice(_ context.Context, _, _ string) (string, error) { return "", nil } func TestHubVoiceInputTurn(t *testing.T) { t.Parallel() diff --git a/internal/services/gateway/server.go b/internal/services/gateway/server.go index a989a12..884cdd2 100644 --- a/internal/services/gateway/server.go +++ b/internal/services/gateway/server.go @@ -390,6 +390,7 @@ func (s *Server) Handler() http.Handler { // fetch synthesized speech, bypassing the 4KB WebSocket frame limit. mux.Handle("POST /api/v1/audio/transcribe", s.auth.Authorize(auth.RoleOwner)(http.HandlerFunc(s.handleTranscribe))) mux.Handle("POST /api/v1/audio/speech", s.auth.Authorize(auth.RoleOwner)(http.HandlerFunc(s.handleSpeech))) + mux.Handle("POST /api/v1/audio/voice-clone", s.auth.Authorize(auth.RoleOwner)(http.HandlerFunc(s.handleVoiceClone))) // Session/model reads and writes are the owner's private history; owner/admin only // (chat uses the same roles). Session metadata in GET /sessions stays readAuth for diff --git a/internal/services/gateway/voice.go b/internal/services/gateway/voice.go index 135da8a..ffc3528 100644 --- a/internal/services/gateway/voice.go +++ b/internal/services/gateway/voice.go @@ -19,8 +19,13 @@ import ( type VoiceService interface { // STT transcribes audio to text. STT(ctx context.Context, audio voice.Audio) (string, error) - // TTS synthesizes audio from text. + // TTS synthesizes audio from text. A per-request emotion/tone instruction may + // be threaded through ctx (voice.WithTTSInstruction) to override the configured + // VOICE_TTS_INSTRUCTION for one call. TTS(ctx context.Context, text string) (voice.Audio, error) + // CloneVoice creates a custom voice (声音复刻 create_voice) from a public + // reference audio URL and returns its voice_id. + CloneVoice(ctx context.Context, referenceURL, prefix string) (string, error) } // voiceInputRequest is the decoded payload of a voice_input frame. Audio is a