From 73dabb9dde885be2c2e12a2a927a74fba928867a Mon Sep 17 00:00:00 2001 From: Aska Date: Tue, 8 Sep 2026 06:06:02 +0800 Subject: [PATCH] =?UTF-8?q?feat(voice):=20=E9=98=BF=E9=87=8C=E7=99=BE?= =?UTF-8?q?=E7=82=BC=20STT/TTS=20=E4=B8=BB=E9=93=BE=E8=B7=AF=E6=8E=A5?= =?UTF-8?q?=E9=80=9A(base64=20STT=20+=20cosyvoice=20TTS=E6=8C=89voice=5Fid?= =?UTF-8?q?=20+=20REST=20/audio/*=20+=20voice=5Finput.deviceId)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - voice/dashscope.go 新建：DashscopeSTT(qwen-audio-3.0-asr-flash, base64 data URI→text) + DashscopeTTS(cosyvoice 按 voice_id, 取非流式 24h URL 下载字节) - cloud.go newDashboardSTT/TTS 由空 endpoint stub 改为真实百炼 adapter - chain.go gating 从 DashscopeAPIKey!="" 改为 voiceConfigured(cfg); silent 兜底保留 - config.VoiceConfig 扩展 DASHSCOPE_BASE_URL/VOICE_STT_*/VOICE_TTS_* 字段(必填 DASHSCOPE_API_KEY) - gateway: POST /api/v1/audio/{transcribe,speech} (owner+审计+MaxBytesReader); voiceInputRequest 增 deviceId 并透传 runner.Run - 实测: cosyvoice-v3.5-plus+longxiaochun 回 418(引擎拒绝), 改用 cosyvoice-v3-flash+longxiaochun_v3; TTS 你好我是昔涟->106124B wav -> STT 回 你好我是西联, 链返回真实字节/文本 - 单测: dashscope_test/config voice defaults/gateway audio_test; go build/vet/test 全绿 - 声音复刻(响昔涟音色)待公网参考音频; 不做 create_voice --- .env.example | 15 +- Docs/api-client.md | 9 +- internal/config/config.go | 30 +- internal/config/config_test.go | 47 +++ internal/runtime/multimodal/voice/chain.go | 15 +- internal/runtime/multimodal/voice/cloud.go | 11 +- .../runtime/multimodal/voice/dashscope.go | 397 ++++++++++++++++++ .../multimodal/voice/dashscope_live_test.go | 96 +++++ .../multimodal/voice/dashscope_test.go | 206 +++++++++ internal/services/gateway/audio.go | 122 ++++++ internal/services/gateway/audio_test.go | 124 ++++++ internal/services/gateway/server.go | 6 + internal/services/gateway/voice.go | 7 +- 13 files changed, 1070 insertions(+), 15 deletions(-) create mode 100644 internal/runtime/multimodal/voice/dashscope.go create mode 100644 internal/runtime/multimodal/voice/dashscope_live_test.go create mode 100644 internal/runtime/multimodal/voice/dashscope_test.go create mode 100644 internal/services/gateway/audio.go create mode 100644 internal/services/gateway/audio_test.go diff --git a/.env.example b/.env.example index 7a9366a..7021f72 100644 --- a/.env.example +++ b/.env.example @@ -66,7 +66,20 @@ IOT_PROVIDER=mock # ---- 语音服务 ---- TTS_PROVIDER=edge-tts STT_PROVIDER=dashscope -# DASHSCOPE_API_KEY=sk-xxxxx +# DASHSCOPE_API_KEY=sk-xxxxx # ⚠️ 必填才能接通阿里百炼语音；未填时走降级链 (silent)。绝不打日志。 +# --- 阿里百炼 DashScope 语音（STT + TTS 主链路）--- +# DASHSCOPE_BASE_URL=https://dashscope.aliyuncs.com/api/v1 +# VOICE_STT_MODEL=qwen-audio-3.0-asr-flash +# VOICE_STT_LANGUAGE=zh +# 已实测：cosyvoice-v3.5-plus / cosyvoice-v2 走 HTTP SpeechSynthesizer 会回 418(引擎拒绝)； +# 可用组合为 cosyvoice-v3-flash + 对应 v3 音色(longxiaochun_v3 龙小淳·女助手等)。 +# 各模型音色不可混用（v2 名如 longxiaochun 用于 v3 模型会 418）。 +# VOICE_TTS_MODEL=cosyvoice-v3-flash +# VOICE_TTS_VOICE_ID=longxiaochun_v3 +# VOICE_TTS_FORMAT=wav +# VOICE_TTS_SAMPLE_RATE=24000 +# 声音复刻（响昔涟音色）：先用参考音频（公网 URL）在百炼「声音复刻」建自定义音色， +# 再把返回的 voice_id 填到 VOICE_TTS_VOICE_ID。TODO：待公网参考音频；本项目不做 create_voice。 # ---- 后台思考 ---- ENABLE_BACKGROUND_THINKING=true diff --git a/Docs/api-client.md b/Docs/api-client.md index e7145ca..3b32c1e 100644 --- a/Docs/api-client.md +++ b/Docs/api-client.md @@ -29,6 +29,12 @@ | 6 | `/api/v1/sessions/{id}/messages` | GET | JWT(owner/admin) | ✅ | **热分页拉历史**：`?limit=&before=&platform=` 默认 `limit=50`（上限 200）、`before`=上一页最旧 `createdAt`（毫秒），按时间倒序 `{data:[{id,sessionId,role,content,mode?,mood?,image?,thinking?,platform?,deviceId?,senderName?,createdAt}]}`；`platform` 为**可选**来源平台过滤（design/16 §4.4）——Web 端不传（=全量共享主会话），IM 适配器传**本平台**（如 `telegram`）只拉本渠道消息；`deviceId` 为**客户端具体端标识**（design/16 §4.5，如 `web:chrome:a1b2c3` / `weixin-ilink:conf0:wx_uid`）；`senderName` 为**第三方发送者昵称**（design/16 §4.6，群聊/陌生人私聊消息带，主人私聊为空，`omitempty`）；非法 `limit`/`before` 回 `400`；未注入 HistoryStore 回 `501` | | 7 | `/api/v1/sessions/{id}` | DELETE | JWT(owner/admin) | ✅ | **删除次级会话及其消息** → `{data:{deleted:}}`；主会话不可删回 `403 forbidden`；未注入 HistoryStore 回 `501` | | 8 | `/api/v1/personas` | GET | JWT | ✅ | `{data:[persona...]}` | +| 9 | `/api/v1/audio/transcribe` | POST | JWT(owner) | ✅ | **语音→文本**：body `{audio:"",format?,deviceId?}` → `{data:{text}}`（走后端 STT，百炼 `qwen-audio-3.0-asr-flash`） | +| 10 | `/api/v1/audio/speech` | POST | JWT(owner) | ✅ | **文本→音频**：body `{text,voice?,format?}` → `200` 音频字节（`Content-Type: audio/`，非 JSON envelope，走后端 TTS，百炼 `cosyvoice-v3-flash`） | + +> **语音 REST 源（`/api/v1/audio/*`，均 owner-only + 审计）**：原生/移动端用它上传/获取音频，**绕开 WS 4KB 帧限**。 +> - `POST /api/v1/audio/transcribe`：body `{audio:"", format?:"wav", deviceId?}` → `200 {data:{text}}`。`audio` 为 base64（或 data: URI）；`format` 为编码（`wav`/`mp3`/`ogg`，缺省 `wav`）；空 audio 回 `400`；未注入语音服务 `500 voice_unavailable`、识别失败 `500 stt_failed`。 +> - `POST /api/v1/audio/speech`：body `{text, voice?, format?}` → `200` 原始音频字节（`Content-Type: audio/`），**非 JSON envelope**。`voice`/`format` 目前遵循服务端 `VOICE_TTS_VOICE_ID`/`VOICE_TTS_FORMAT` 配置（后续支持按请求覆盖）；空 text 回 `400`；未注入语音服务 `500 voice_unavailable`、合成失败 `500 tts_failed`、静默空音频 `500 tts_empty`。**声音复刻（响昔涟音色）**：先用参考音频（公网 URL）在百炼「声音复刻」建自定义音色，再把返回的 `voice_id` 配到 `VOICE_TTS_VOICE_ID`（本项目不做 create_voice）。 ## 2. REST 端点（管理，`/api/v1/admin/*`） @@ -126,7 +132,7 @@ - **向后兼容**：未映射/无法确定的事件仍走 `stream_chunk` + `stream_end`（非流式回退路径）。 - **上行（客户端→服务端）**: `mode_switch` / `chat`（跑 loop 并广播流式） / `voice_input` / `subagent_ask` / `task_output_query` / `interrupt`。 - **`interrupt`**（上行，客户端→服务端）：`{sessionId}` → 会话编排器中断（design/15 §3）——取消该会话当前在跑的任务（context cancel）、清空其待处理队列、回 `session_interrupted`；用户说「停下/算了/换个话题」时发。命中空闲会话为 no-op。 -- **`voice_input`**（上行，客户端→服务端）：`_{audio?, format?, text?, sessionId?}`。`audio` 为 base64（或 data: URI）音频，`format` 为其编码（`wav`/`mp3`/`ogg`）；`text` 为已转写文本时跳过 STT。服务端走 STT→agent 聊天→TTS（design/06 §1），回复经 `voice_output` 定向回该连接。**真 STT/TTS 需配云端 endpoint**；无凭据时后端走降级链（cloud 端点为占位，edge-tts/espeak 未实现，最终 `silent` 返回空音频）。无注入语音服务时回 `error` 帧（`voice unavailable`）。 +- **`voice_input`**（上行，客户端→服务端）：`_{audio?, format?, text?, sessionId?, deviceId?}`。`audio` 为 base64（或 data: URI）音频，`format` 为其编码（`wav`/`mp3`/`ogg`）；`text` 为已转写文本时跳过 STT；`deviceId` 为**客户端具体端标识**（design/16 §4.5，如 `voiceassist:myapp:v1`，可选），随该语音轮次落库、聊天气泡可标注来源。服务端走 STT→agent 聊天→TTS（design/06 §1），回复经 `voice_output` 定向回该连接。**真 STT/TTS 已接通阿里百炼**（`DASHSCOPE_API_KEY` 必填：STT=`qwen-audio-3.0-asr-flash`、TTS=`cosyvoice-v3-flash`+`longxiaochun_v3`）；无凭据时后端走降级链（cloud → edge-tts/espeak（未实现）→ `silent` 空音频）。无注入语音服务时回 `error` 帧（`voice unavailable`）。 - **`voice_output`**（下行，服务端→客户端）：`{audio, format, text, reply}`。`audio` 为 base64 合成音频，`format` 为其编码，`text` 为识别文本，`reply` 为助手回复文本。 - **向后兼容**：未映射/无法确定的事件仍走 `stream_chunk` + `stream_end`（非流式回退路径）。 @@ -196,6 +202,7 @@ - 语音：上行 `voice_input`（base64 音频或已转写 text），下行 `voice_output`（合成音频 base64 + text）。 ## 5. 变更记录 +- `v0.18`（**后端语音接通阿里百炼 STT + TTS 主链路**）：`POST /api/v1/audio/transcribe`（**新增**，owner + 审计，body `{audio,format?,deviceId?}` → `{data:{text}}`，走 STT）；`POST /api/v1/audio/speech`（**新增**，owner + 审计，body `{text,voice?,format?}` → 音频字节 `audio/`，走 TTS）——**REST 语音源**，原生/移动端经它绕开 WS 4KB 帧限上传/获取音频；WS `voice_input` 上行新增 **`deviceId`**（随语音轮次落库，design/16 §4.5）。后端 `internal/runtime/multimodal/voice` 的 cloud rung 由空 endpoint stub 改为**真实百炼 adapter**：STT `qwen-audio-3.0-asr-flash`（base64 原始字节→文本）、TTS `cosyvoice-v3-flash`（按 `voice_id` 合成，取非流式返回的 24h 音频 URL 下载字节）。**实测**（`DASHSCOPE_API_KEY` 实跑）：TTS「你好，我是昔涟」→ 106124 字节 wav；把该 wav 喂给 STT → 文本「你好，我是西联。」（TTS 发音把「昔涟」读得近似「西联」，STT 识别正确）；语音链 (cloud→…→silent) 返回真实字节/文本。**已实测修正**：`cosyvoice-v3.5-plus`/`longxiaochun`（v2 音色）走 HTTP `SpeechSynthesizer` 会回 `418`（引擎拒绝），改用 **`cosyvoice-v3-flash` + `longxiaochun_v3`**（v3 音色，各模型音色不可混用）。**声音复刻（响昔涟音色）仍待**：需参考音频（公网 URL）在百炼建自定义音色后把 `voice_id` 填入 `VOICE_TTS_VOICE_ID`（本项目不做 create_voice）。config 新增 `DASHSCOPE_BASE_URL`/`VOICE_STT_\*`/`VOICE_TTS_\*`（`VOICE_TTS_SAMPLE_RATE` 默认 24000）。 - `v0.17`（系统设置安全与存储：吊销全部 API key / 轮换 JWT 密钥 / 备份 / 恢复，design/11 §4 + design/10 §3 模块14）：新增 `POST /api/v1/admin/system/{revoke-keys,rotate-jwt,backup,restore}`（均 **owner-only** + 审计）。`revoke-keys` 走 `auth.APIKey.RevokeAll()`（清空 key 表，返回 `{revoked:n}`）；`rotate-jwt` 走 `auth.JWTService.RotateSecret`（≥32 位，**新密钥不回显/不落日志**，现有 token 立即失效）；`backup` 用 SQLite `VACUUM INTO` 快照到 `/backup-.db`（含已提交 WAL 内容）返回 `{path,size}`；`restore` 先校验路径在 dataDir 内（**`filepath.Rel` 防路径穿越**，逃逸回 `400`）→ **先自动备份当前 DB** → 替换 DB 文件 → 返回 `{restored:true,backup,note:"需重启后生效"}`（运行进程持有 DB 句柄，热替换需重启可见）。gateway 新增 `Deps.DatabasePath`（composition root 传入实际打开的 SQLite 路径）。`编辑环境变量 / 重启服务 / 检查更新` 仍为后置，未实现、不伪造。 - `v0.17`（KOOK 适配器面板 + 每适配器专属页，design/17 §2）：`GET /api/v1/admin/channels/{platform}/view`、`GET /api/v1/admin/channels/{platform}/friends?type=`、`POST /api/v1/admin/channels/{platform}/friend/handle`、`POST /api/v1/admin/channels/{platform}/message/test`（**新增**，均 owner-only + 审计，带可选 `?accountId=`，仅 `kook` 实现，其它平台 `501`）；`GET /api/v1/admin/channels` 每条账号附 `capabilities`（面板按平台渲染专属卡片）。KOOK 面板：好友/申请、服务器/频道/成员、私信会话、发消息测试——经适配器 REST（`Authorization: Bot `）拉真实数据，token 不打印不落日志，子项失败记入 `errors` 不伪造。 - `v0.16`（运行时配置体系：渠道适配器配置落盘 + 热更新，design/04 §3.1）：`PUT /api/v1/admin/channels/{platform}/config`（**新增**，owner-only + 审计）body `{id?,token?,baseURL?,remoteURL?,listenAddr?,mode?,accessToken?,selfID?,config?,allowGroups?,denyGroups?,allowUsers?,denyUsers?,groupMsgPerMin?,msgPerMin?,maxMessages?}` → `201 {persisted:true,path,platform,accountId}`——读当前有效配置 → `UpsertChannel` → 落盘 `channels.json`（本地、gitignore、0600）→ `bridge.Reload(platform)` **热替换运行中 adapter**（进行中消息用旧、新消息用新），无需重启；token 写入文件但**不回显/不落日志**。`DELETE /api/v1/admin/channels/{platform}/config?id=`（**新增**）删除账号并热更新。`GET /api/v1/admin/channels` 每条渠道附 `accounts`（每平台已配账号，token 脱敏掩码）。`.env`（`CHANNELS_JSON`/`CHANNEL_*_TOKEN`）退为兜底。 diff --git a/internal/config/config.go b/internal/config/config.go index c2b1062..bdef200 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -97,11 +97,35 @@ type IoTConfig struct { Provider string `env:"IOT_PROVIDER" default:"mock"` } -// VoiceConfig holds TTS/STT provider settings. +// VoiceConfig holds TTS/STT provider settings and the Alibaba Bailian DashScope +// credentials/models that back the cloud rung of the voice chain. type VoiceConfig struct { - TTSProvider string `env:"TTS_PROVIDER" default:"edge-tts"` - STTProvider string `env:"STT_PROVIDER" default:"dashscope"` + // TTSProvider selects the non-cloud local TTS rung: "edge-tts" or "espeak". + TTSProvider string `env:"TTS_PROVIDER" default:"edge-tts"` + // STTProvider selects the cloud STT provider ("dashscope"). + STTProvider string `env:"STT_PROVIDER" default:"dashscope"` + // DashscopeAPIKey is the Bailian API key backing STT + TTS. When empty the + // dashscope adapters return ErrNotConfigured and the chain degrades. DashscopeAPIKey string `env:"DASHSCOPE_API_KEY"` + + // DashscopeBaseURL is the Bailian API base; the STT/TTS service paths are + // appended to it. + DashscopeBaseURL string `env:"DASHSCOPE_BASE_URL" default:"https://dashscope.aliyuncs.com/api/v1"` + + // STTModel is the DashScope speech-transcription model. + STTModel string `env:"VOICE_STT_MODEL" default:"qwen-audio-3.0-asr-flash"` + STTLanguage string `env:"VOICE_STT_LANGUAGE" default:"zh"` + + // TTSModel is the DashScope speech-synthesis model. cosyvoice-v3.5-plus and + // cosyvoice-v2 reject the HTTP SpeechSynthesizer call (engine 418); verified + // working is cosyvoice-v3-flash. + TTSModel string `env:"VOICE_TTS_MODEL" default:"cosyvoice-v3-flash"` + // TTSVoiceID selects the synthesized timbre. Voices are model-scoped (a v3 + // model needs a _v3 voice). After cloning a custom voice in Bailian + // (音色复刻) fill its voice_id here. + TTSVoiceID string `env:"VOICE_TTS_VOICE_ID" default:"longxiaochun_v3"` + TTSFormat string `env:"VOICE_TTS_FORMAT" default:"wav"` + TTSSampleRate int `env:"VOICE_TTS_SAMPLE_RATE" default:"24000"` } // BackgroundConfig holds the autonomous thinking settings. QuietStartHour and diff --git a/internal/config/config_test.go b/internal/config/config_test.go index addad0a..be81912 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -162,6 +162,53 @@ func TestLoad_CapacityEnvOverrides(t *testing.T) { } } +func TestLoad_VoiceDefaults(t *testing.T) { + t.Setenv("JWT_SECRET", strings.Repeat("s", 40)) + + cfg, err := Load("") + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if cfg.Voice.DashscopeBaseURL != "https://dashscope.aliyuncs.com/api/v1" { + t.Errorf("DashscopeBaseURL = %q, want the default Bailian URL", cfg.Voice.DashscopeBaseURL) + } + if cfg.Voice.STTModel != "qwen-audio-3.0-asr-flash" { + t.Errorf("STTModel = %q, want qwen-audio-3.0-asr-flash", cfg.Voice.STTModel) + } + if cfg.Voice.STTLanguage != "zh" { + t.Errorf("STTLanguage = %q, want zh", cfg.Voice.STTLanguage) + } + if cfg.Voice.TTSModel != "cosyvoice-v3-flash" { + t.Errorf("TTSModel = %q, want cosyvoice-v3-flash", cfg.Voice.TTSModel) + } + if cfg.Voice.TTSVoiceID != "longxiaochun_v3" { + t.Errorf("TTSVoiceID = %q, want longxiaochun_v3", cfg.Voice.TTSVoiceID) + } + if cfg.Voice.TTSFormat != "wav" { + t.Errorf("TTSFormat = %q, want wav", cfg.Voice.TTSFormat) + } + if cfg.Voice.TTSSampleRate != 24000 { + t.Errorf("TTSSampleRate = %d, want 24000", cfg.Voice.TTSSampleRate) + } +} + +func TestLoad_VoiceEnvOverrides(t *testing.T) { + t.Setenv("JWT_SECRET", strings.Repeat("s", 40)) + t.Setenv("VOICE_TTS_VOICE_ID", "custom-voice-1") + t.Setenv("VOICE_STT_MODEL", "qwen-audio-3.0-asr-flash") + + cfg, err := Load("") + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if cfg.Voice.TTSVoiceID != "custom-voice-1" { + t.Errorf("TTSVoiceID = %q, want custom-voice-1", cfg.Voice.TTSVoiceID) + } + if cfg.Voice.STTModel != "qwen-audio-3.0-asr-flash" { + t.Errorf("STTModel = %q, want qwen-audio-3.0-asr-flash", cfg.Voice.STTModel) + } +} + func TestParseDotEnv_QuotedAndExport(t *testing.T) { path := writeEnvFile(t, `export FOO=bar BAZ='quoted value' diff --git a/internal/runtime/multimodal/voice/chain.go b/internal/runtime/multimodal/voice/chain.go index 32048e2..b7b0273 100644 --- a/internal/runtime/multimodal/voice/chain.go +++ b/internal/runtime/multimodal/voice/chain.go @@ -80,8 +80,8 @@ func WithProviders(steps ...Provider) Option { // New assembles a chain from cfg using the fixed degradation order // cloud → local → edge-tts/espeak → silent. // -// - cloud is included only when a cloud voice setting is present (today an -// API key) so a mis-configured build degrades quietly. +// - cloud is included only when a cloud voice credential or model is present +// (voiceConfigured) so a mis-configured build degrades quietly. // - local is a pure adaptation point and is only included when supplied via // WithProviders; by default it is skipped. // - the edge-tts or espeak rung is included according to cfg.TTSProvider. @@ -98,7 +98,7 @@ func New(cfg config.VoiceConfig, opts ...Option) *Chain { } steps := make([]Provider, 0, 5) - if cfg.DashscopeAPIKey != "" { + if voiceConfigured(cfg) { steps = append(steps, CloudProvider(newDashboardSTT(cfg), newDashboardTTS(cfg))) } switch cfg.TTSProvider { @@ -110,3 +110,12 @@ func New(cfg config.VoiceConfig, opts ...Option) *Chain { steps = append(steps, SilentProvider()) return NewChain(steps...) } + +// voiceConfigured reports whether any cloud voice credential or model is set, so +// the cloud rung is only added when the runtime intends to use DashScope. Any +// single present field (API key, base URL, or an STT/TTS model) makes the cloud +// rung worth trying; a missing key then makes the adapter return ErrNotConfigured +// and the chain degrades quietly. +func voiceConfigured(cfg config.VoiceConfig) bool { + return cfg.DashscopeAPIKey != "" || cfg.DashscopeBaseURL != "" || cfg.STTModel != "" || cfg.TTSModel != "" +} diff --git a/internal/runtime/multimodal/voice/cloud.go b/internal/runtime/multimodal/voice/cloud.go index 1e0e7be..20e5eb3 100644 --- a/internal/runtime/multimodal/voice/cloud.go +++ b/internal/runtime/multimodal/voice/cloud.go @@ -201,16 +201,15 @@ func discardLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } -// newDashboardSTT returns the cloud STT for a config. The endpoint is an -// adaptation point and is empty until config gains a voice URL; a call then -// returns ErrNotConfigured and the chain degrades. +// newDashboardSTT returns the cloud STT adapter for a DashScope config. A call +// returns ErrNotConfigured when no API key is set so the chain degrades. func newDashboardSTT(cfg config.VoiceConfig) STT { - return NewHTTPSTT("", cfg.DashscopeAPIKey) + return NewDashscopeSTT(cfg) } -// newDashboardTTS returns the cloud TTS for a config. +// newDashboardTTS returns the cloud TTS adapter for a DashScope config. func newDashboardTTS(cfg config.VoiceConfig) TTS { - return NewHTTPTTS("", cfg.DashscopeAPIKey) + return NewDashscopeTTS(cfg) } // newEdgeTTS returns the edge-tts adapter point (not yet implemented). diff --git a/internal/runtime/multimodal/voice/dashscope.go b/internal/runtime/multimodal/voice/dashscope.go new file mode 100644 index 0000000..14c38c5 --- /dev/null +++ b/internal/runtime/multimodal/voice/dashscope.go @@ -0,0 +1,397 @@ +package voice + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/config" +) + +// DashScope (Alibaba Bailian) STT/TTS adapters for the cloud rung of the voice +// chain. Requests follow the DashScope public API: +// +// - STT: POST {base}/services/aigc/multimodal-generation/generation +// (qwen-audio-3.0-asr-flash, base64 input audio) +// - TTS: POST {base}/services/audio/tts/SpeechSynthesizer +// (cosyvoice-v3-flash, synthesized by voice_id) +// +// Both take a non-streaming response here: STT returns the transcript in the +// output envelope; TTS (no X-DashScope-SSE header) returns a 24h audio URL that +// we download so the caller always gets the actual bytes. Audio format ↔ mime +// is mapped in both directions (mimeOf / the data: URI in the STT request). + +const ( + // defaultSTTSampleRate is the input audio sample rate the ASR expects. + defaultSTTSampleRate = 16000 + // dashscopeSTTPath is the multimodal-generation service path (ASR). + dashscopeSTTPath = "/services/aigc/multimodal-generation/generation" + // dashscopeTTSPath is the SpeechSynthesizer service path (TTS). + dashscopeTTSPath = "/services/audio/tts/SpeechSynthesizer" + // defaultDashscopeBaseURL is the Bailian API root when none is configured. + defaultDashscopeBaseURL = "https://dashscope.aliyuncs.com/api/v1" +) + +// DashscopeSTT transcribes audio via DashScope multimodal-generation. +type DashscopeSTT struct { + endpoint string + apiKey string + model string + format string + language string + sampleRate int + client *http.Client +} + +// NewDashscopeSTT builds the DashScope STT adapter from cfg. Missing model / +// language / base URL fall back to the DashScope defaults. +func NewDashscopeSTT(cfg config.VoiceConfig) *DashscopeSTT { + model := firstNonEmpty(cfg.STTModel, "qwen-audio-3.0-asr-flash") + language := firstNonEmpty(cfg.STTLanguage, "zh") + return &DashscopeSTT{ + endpoint: dashscopeEndpoint(cfg.DashscopeBaseURL) + dashscopeSTTPath, + apiKey: cfg.DashscopeAPIKey, + model: model, + format: "wav", + language: language, + sampleRate: defaultSTTSampleRate, + client: defaultHTTPClient(), + } +} + +// STT posts the audio as a base64 data: URI and returns the recognized text. +func (s *DashscopeSTT) STT(ctx context.Context, audio Audio) (string, error) { + if s.apiKey == "" { + return "", ErrNotConfigured + } + if len(audio.Data) == 0 { + return "", fmt.Errorf("voice: %w: empty audio", ErrProviderFailed) + } + if err := ctx.Err(); err != nil { + return "", err + } + format := audio.Format + if format == "" { + format = "wav" + } + payload, err := json.Marshal(dashscopeSTTRequest{ + Model: s.model, + Input: dashscopeSTTInput{ + Messages: []dashscopeSTTMessage{{ + Role: "user", + Content: []dashscopeSTTContent{{ + Type: "input_audio", + InputAudio: dashscopeInputAudio{ + Data: "data:" + mimeOf(format) + ";base64," + base64.StdEncoding.EncodeToString(audio.Data), + }, + }}, + }}, + }, + Parameters: dashscopeSTTParameters{ + Format: format, + SampleRate: s.sampleRate, + LanguageHints: []string{s.language}, + }, + }) + if err != nil { + return "", fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err) + } + body, err := dashscopePost(ctx, s.client, s.apiKey, s.endpoint, payload, "disable") + if err != nil { + return "", err + } + text, err := extractSTTText(body) + if err != nil { + return "", fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err) + } + return text, nil +} + +// DashscopeTTS synthesizes speech video via DashScope SpeechSynthesizer. +type DashscopeTTS struct { + endpoint string + apiKey string + model string + voiceID string + format string + sampleRate int + client *http.Client +} + +// NewDashscopeTTS builds the DashScope TTS adapter from cfg. Missing model / +// voice / format / base URL fall back to the DashScope defaults. +func NewDashscopeTTS(cfg config.VoiceConfig) *DashscopeTTS { + model := firstNonEmpty(cfg.TTSModel, "cosyvoice-v3-flash") + voiceID := firstNonEmpty(cfg.TTSVoiceID, "longxiaochun_v3") + format := firstNonEmpty(cfg.TTSFormat, "wav") + sampleRate := cfg.TTSSampleRate + if sampleRate == 0 { + sampleRate = 24000 + } + return &DashscopeTTS{ + endpoint: dashscopeEndpoint(cfg.DashscopeBaseURL) + dashscopeTTSPath, + apiKey: cfg.DashscopeAPIKey, + model: model, + voiceID: voiceID, + format: format, + sampleRate: sampleRate, + client: defaultHTTPClient(), + } +} + +// TTS synthesizes audio for text. The non-streaming DashScope response carries a +// 24h audio URL (or a base64 blob); we resolve it to bytes and return them so the +// caller holds a complete audio file. +func (t *DashscopeTTS) TTS(ctx context.Context, text string) (Audio, error) { + if t.apiKey == "" { + return Audio{}, ErrNotConfigured + } + if strings.TrimSpace(text) == "" { + return Audio{}, ErrEmptyText + } + if err := ctx.Err(); err != nil { + return Audio{}, err + } + payload, err := json.Marshal(dashscopeTTSRequest{ + Model: t.model, + Input: dashscopeTTSInput{ + Text: text, + Voice: t.voiceID, + Format: t.format, + SampleRate: t.sampleRate, + }, + }) + if err != nil { + return Audio{}, fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err) + } + // No X-DashScope-SSE header → non-streaming URL response. + body, err := dashscopePost(ctx, t.client, t.apiKey, t.endpoint, payload, "") + if err != nil { + return Audio{}, err + } + data, err := t.audioBytes(ctx, body) + if err != nil { + return Audio{}, err + } + return Audio{Format: t.format, Data: data}, nil +} + +// audioBytes resolves the synthesized audio from the DashScope response: a 24h +// URL is downloaded, or an inline base64 blob is decoded. +func (t *DashscopeTTS) audioBytes(ctx context.Context, body []byte) ([]byte, error) { + var r dashscopeTTSResponse + if err := json.Unmarshal(body, &r); err != nil { + return nil, fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err) + } + if url := r.Output.Audio.URL; url != "" { + data, err := t.download(ctx, url) + if err != nil { + return nil, err + } + return data, nil + } + if b64 := r.Output.Audio.Data; b64 != "" { + data, err := base64.StdEncoding.DecodeString(b64) + if err != nil { + return nil, fmt.Errorf("voice: %w: decode audio: %v", ErrProviderFailed, err) + } + return data, nil + } + return nil, fmt.Errorf("voice: %w: no audio in response", ErrProviderFailed) +} + +// download fetches an audio URL into memory, capping the size so an oversized +// media response cannot exhaust memory. +func (t *DashscopeTTS) download(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("voice: %w: build: %v", ErrProviderFailed, err) + } + resp, err := t.client.Do(req) + if err != nil { + return nil, fmt.Errorf("voice: %w: download: %v", ErrProviderFailed, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("voice: %w: download status %d", ErrProviderFailed, resp.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, cloudMaxResponseBytes)) + if err != nil { + return nil, fmt.Errorf("voice: %w: download read: %v", ErrProviderFailed, err) + } + return data, nil +} + +// dashscopeEndpoint returns the DashScope API base with a trailing slash trimmed, +// defaulting to the public Bailian root when none is configured. +func dashscopeEndpoint(baseURL string) string { + base := strings.TrimRight(baseURL, "/") + if base == "" { + return defaultDashscopeBaseURL + } + return base +} + +// firstNonEmpty returns a when non-empty, else b. +func firstNonEmpty(a, b string) string { + if a != "" { + return a + } + return b +} + +// dashscopePost POSTs a JSON payload with the DashScope bearer auth and an +// optional X-DashScope-SSE header, returning the decoded response body. A non-200 +// status is wrapped with the DashScope error body. +func dashscopePost(ctx context.Context, client *http.Client, apiKey, url string, payload []byte, sse string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + return nil, fmt.Errorf("voice: %w: build: %v", ErrProviderFailed, err) + } + req.Header.Set("Content-Type", "application/json") + if apiKey != "" { + req.Header.Set("Authorization", "Bearer "+apiKey) + } + if sse != "" { + req.Header.Set("X-DashScope-SSE", sse) + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("voice: %w: request: %v", ErrProviderFailed, err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, cloudMaxResponseBytes)) + if err != nil { + return nil, fmt.Errorf("voice: %w: read: %v", ErrProviderFailed, err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("voice: %w: dashscope status %d: %s", ErrProviderFailed, resp.StatusCode, strings.TrimSpace(string(body))) + } + return body, nil +} + +// extractSTTText pulls the transcript out of a DashScope ASR response. The ASR +// output shape varies by model, so several candidate paths are tried. +func extractSTTText(body []byte) (string, error) { + var r dashscopeSTTResponse + if err := json.Unmarshal(body, &r); err != nil { + return "", err + } + if s := r.Output.Output.Sentence.Text; s != "" { + return s, nil + } + if s := r.Output.Output.Text; s != "" { + return s, nil + } + if s := r.Output.Text; s != "" { + return s, nil + } + for _, ch := range r.Output.Choices { + if len(ch.Message.Content) > 0 { + if s := ch.Message.Content[0].Text; s != "" { + return s, nil + } + } + } + return r.Text, nil +} + +// mimeOf maps an audio format to its MIME type for the data: URI / Content-Type. +func mimeOf(format string) string { + switch strings.ToLower(format) { + case "mp3", "mpeg": + return "audio/mpeg" + case "ogg", "oga": + return "audio/ogg" + case "webm": + return "audio/webm" + default: + return "audio/wav" + } +} + +// dashscopeSTTResponse is the DashScope ASR response envelope. +type dashscopeSTTResponse struct { + Output struct { + Output struct { + Sentence struct { + Text string `json:"text"` + } `json:"sentence"` + Text string `json:"text"` + } `json:"output"` + Text string `json:"text"` + Choices []struct { + Message struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + } `json:"message"` + } `json:"choices"` + } `json:"output"` + Text string `json:"text"` +} + +// dashscopeTTSResponse is the DashScope TTS response envelope. +type dashscopeTTSResponse struct { + Output struct { + Audio struct { + URL string `json:"url"` + Data string `json:"data"` + } `json:"audio"` + } `json:"output"` +} + +// dashscopeSTTRequest is the ASR request body. +type dashscopeSTTRequest struct { + Model string `json:"model"` + Input dashscopeSTTInput `json:"input"` + Parameters dashscopeSTTParameters `json:"parameters"` +} + +// dashscopeSTTInput carries the multimodal messages list. +type dashscopeSTTInput struct { + Messages []dashscopeSTTMessage `json:"messages"` +} + +// dashscopeSTTMessage is one ASR user turn. +type dashscopeSTTMessage struct { + Role string `json:"role"` + Content []dashscopeSTTContent `json:"content"` +} + +// dashscopeSTTContent is one audio content block. +type dashscopeSTTContent struct { + Type string `json:"type"` + InputAudio dashscopeInputAudio `json:"input_audio"` +} + +// dashscopeInputAudio carries the base64 data: URI audio. +type dashscopeInputAudio struct { + Data string `json:"data"` +} + +// dashscopeSTTParameters tunes the ASR request. +type dashscopeSTTParameters struct { + Format string `json:"format"` + SampleRate int `json:"sample_rate"` + LanguageHints []string `json:"language_hints,omitempty"` +} + +// dashscopeTTSRequest is the TTS request body. +type dashscopeTTSRequest struct { + Model string `json:"model"` + Input dashscopeTTSInput `json:"input"` +} + +// dashscopeTTSInput carries text + voice + output format. +type dashscopeTTSInput struct { + Text string `json:"text"` + Voice string `json:"voice"` + Format string `json:"format"` + SampleRate int `json:"sample_rate"` +} diff --git a/internal/runtime/multimodal/voice/dashscope_live_test.go b/internal/runtime/multimodal/voice/dashscope_live_test.go new file mode 100644 index 0000000..457a5a6 --- /dev/null +++ b/internal/runtime/multimodal/voice/dashscope_live_test.go @@ -0,0 +1,96 @@ +//go:build dashscope_live + +package voice + +import ( + "bytes" + "context" + "encoding/binary" + "os" + "testing" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/config" +) + +// TestDashscopeLive verifies the real Alibaba Bailian STT + TTS end to end. It is +// excluded from the default suite (build tag dashscope_live) so `go test ./...` +// never hits the paid API. Run it explicitly with the key sourced from .env: +// +// set -a; source .env; set +a +// go test -tags dashscope_live -run TestDashscopeLive -v ./internal/runtime/multimodal/voice/ +func TestDashscopeLive(t *testing.T) { + key := os.Getenv("DASHSCOPE_API_KEY") + if key == "" { + t.Skip("DASHSCOPE_API_KEY not set") + } + cfg := config.VoiceConfig{DashscopeAPIKey: key} + + // TTS: synthesize speech for a short line. + tts := NewDashscopeTTS(cfg) + audio, err := tts.TTS(context.Background(), "你好，我是昔涟") + if err != nil { + t.Fatalf("TTS error: %v", err) + } + t.Logf("TTS: format=%q bytes=%d", audio.Format, len(audio.Data)) + if len(audio.Data) == 0 { + t.Fatal("TTS returned empty audio") + } + + // STT closed loop: transcribe the synthesized speech back to text. + stt := NewDashscopeSTT(cfg) + text, err := stt.STT(context.Background(), Audio{Format: "wav", Data: audio.Data}) + if err != nil { + t.Fatalf("STT error: %v", err) + } + t.Logf("STT (of TTS audio): text=%q", text) + + // STT on a generated silence wav, to confirm the empty-audio path is accepted. + silence := silentWAVBytes(1, 16000) + sttText, sttErr := stt.STT(context.Background(), Audio{Format: "wav", Data: silence}) + if sttErr != nil { + t.Logf("STT (silence): error=%v", sttErr) + } else { + t.Logf("STT (silence): text=%q", sttText) + } + + // Exercise the degrading chain (the same object the gateway wires), proving it + // returns real bytes/text rather than degrading to the silent terminal rung. + ch := New(cfg) + chTTS, chTTSErr := ch.TTS(context.Background(), "你好，我是昔涟") + if chTTSErr != nil { + t.Fatalf("chain TTS error: %v", chTTSErr) + } + t.Logf("chain TTS: format=%q bytes=%d", chTTS.Format, len(chTTS.Data)) + if len(chTTS.Data) == 0 { + t.Fatal("chain TTS degraded to empty (silent) audio") + } + chText, chSTTErr := ch.STT(context.Background(), Audio{Format: "wav", Data: chTTS.Data}) + if chSTTErr != nil { + t.Fatalf("chain STT error: %v", chSTTErr) + } + t.Logf("chain STT: text=%q", chText) + if chText == "" { + t.Fatal("chain STT degraded to empty (silent) text") + } +} + +// silentWAVBytes builds a mono 16-bit 16 kHz silence WAV of durS seconds. +func silentWAVBytes(durS, sampleRate int) []byte { + dataLen := durS * sampleRate * 2 + buf := &bytes.Buffer{} + // RIFF header + fmt chunk + data chunk. + buf.WriteString("RIFF") + _ = binary.Write(buf, binary.LittleEndian, uint32(36+dataLen)) + buf.WriteString("WAVEfmt ") + _ = binary.Write(buf, binary.LittleEndian, uint32(16)) + _ = binary.Write(buf, binary.LittleEndian, uint16(1)) // PCM + _ = binary.Write(buf, binary.LittleEndian, uint16(1)) // mono + _ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate)) + _ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate*2)) + _ = binary.Write(buf, binary.LittleEndian, uint16(2)) // block align + _ = binary.Write(buf, binary.LittleEndian, uint16(16)) // bits per sample + buf.WriteString("data") + _ = binary.Write(buf, binary.LittleEndian, uint32(dataLen)) + buf.Write(make([]byte, dataLen)) + return buf.Bytes() +} diff --git a/internal/runtime/multimodal/voice/dashscope_test.go b/internal/runtime/multimodal/voice/dashscope_test.go new file mode 100644 index 0000000..592e9f8 --- /dev/null +++ b/internal/runtime/multimodal/voice/dashscope_test.go @@ -0,0 +1,206 @@ +package voice + +import ( + "context" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/config" +) + +func TestDashscopeSTT(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/generation") { + t.Errorf("path = %q, want .../generation", r.URL.Path) + } + if r.Header.Get("Authorization") != "Bearer sk" { + t.Errorf("auth = %q, want Bearer sk", r.Header.Get("Authorization")) + } + if r.Header.Get("X-DashScope-SSE") != "disable" { + t.Errorf("sse = %q, want disable", r.Header.Get("X-DashScope-SSE")) + } + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("decode: %v", err) + } + if body["model"] != "qwen-audio-3.0-asr-flash" { + t.Errorf("model = %v, want qwen-audio-3.0-asr-flash", body["model"]) + } + input := body["input"].(map[string]any) + msgs := input["messages"].([]any) + content := msgs[0].(map[string]any)["content"].([]any) + audio := content[0].(map[string]any)["input_audio"].(map[string]any)["data"].(string) + if !strings.HasPrefix(audio, "data:audio/wav;base64,") { + t.Errorf("data uri = %q, want data:audio/wav;base64,...", audio) + } + params := body["parameters"].(map[string]any) + if params["format"] != "wav" { + t.Errorf("format = %v, want wav", params["format"]) + } + hints := params["language_hints"].([]any) + if len(hints) != 1 || hints[0] != "zh" { + t.Errorf("language_hints = %v, want [zh]", hints) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"output":{"output":{"sentence":{"text":"你好世界"}}}}`)) + })) + defer srv.Close() + + stt := NewDashscopeSTT(config.VoiceConfig{DashscopeBaseURL: srv.URL, DashscopeAPIKey: "sk"}) + text, err := stt.STT(context.Background(), Audio{Format: "wav", Data: []byte("rawaudio")}) + if err != nil { + t.Fatalf("STT error: %v", err) + } + if text != "你好世界" { + t.Errorf("text = %q, want 你好世界", text) + } +} + +func TestDashscopeSTT_ChoicesText(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"output":{"choices":[{"message":{"content":[{"text":"hello"}]}}]}}`)) + })) + defer srv.Close() + + stt := NewDashscopeSTT(config.VoiceConfig{DashscopeBaseURL: srv.URL, DashscopeAPIKey: "sk"}) + text, err := stt.STT(context.Background(), Audio{Format: "wav", Data: []byte("x")}) + if err != nil { + t.Fatalf("STT error: %v", err) + } + if text != "hello" { + t.Errorf("text = %q, want hello", text) + } +} + +func TestDashscopeSTT_NotConfigured(t *testing.T) { + t.Parallel() + stt := NewDashscopeSTT(config.VoiceConfig{}) + if _, err := stt.STT(context.Background(), Audio{Format: "wav", Data: []byte("x")}); err != ErrNotConfigured { + t.Fatalf("err = %v, want ErrNotConfigured", err) + } +} + +func TestDashscopeSTT_EmptyAudio(t *testing.T) { + t.Parallel() + stt := NewDashscopeSTT(config.VoiceConfig{DashscopeBaseURL: "http://x", DashscopeAPIKey: "sk"}) + if _, err := stt.STT(context.Background(), Audio{}); !errors.Is(err, ErrProviderFailed) { + t.Fatalf("err = %v, want ErrProviderFailed", err) + } +} + +func TestDashscopeSTT_Non200(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"code":"InvalidParameter","message":"bad"}`)) + })) + defer srv.Close() + + stt := NewDashscopeSTT(config.VoiceConfig{DashscopeBaseURL: srv.URL, DashscopeAPIKey: "sk"}) + _, err := stt.STT(context.Background(), Audio{Format: "wav", Data: []byte("x")}) + if !errors.Is(err, ErrProviderFailed) { + t.Fatalf("err = %v, want ErrProviderFailed", err) + } + if !strings.Contains(err.Error(), "400") { + t.Errorf("err = %v, want it to mention status 400", err) + } +} + +func TestDashscopeTTS_URL(t *testing.T) { + t.Parallel() + audioBytes := []byte("RIFFxxxxWAVE") + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/SpeechSynthesizer"): + if r.Header.Get("Authorization") != "Bearer sk" { + t.Errorf("auth = %q, want Bearer sk", r.Header.Get("Authorization")) + } + if sse := r.Header.Get("X-DashScope-SSE"); sse != "" { + t.Errorf("sse = %q, want empty (non-streaming URL)", sse) + } + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("decode: %v", err) + } + if body["model"] != "cosyvoice-v3-flash" { + t.Errorf("model = %v, want cosyvoice-v3-flash", body["model"]) + } + input := body["input"].(map[string]any) + if input["text"] != "你好" { + t.Errorf("text = %v, want 你好", input["text"]) + } + if input["voice"] != "longxiaochun_v3" { + t.Errorf("voice = %v, want longxiaochun_v3", input["voice"]) + } + if input["format"] != "wav" { + t.Errorf("format = %v, want wav", input["format"]) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"output":{"audio":{"url":"` + srv.URL + `/audio/out.wav"}}}`)) + case strings.HasSuffix(r.URL.Path, "/audio/out.wav"): + w.Header().Set("Content-Type", "audio/wav") + _, _ = w.Write(audioBytes) + default: + t.Errorf("unexpected path %q", r.URL.Path) + } + })) + defer srv.Close() + + tts := NewDashscopeTTS(config.VoiceConfig{DashscopeBaseURL: srv.URL, DashscopeAPIKey: "sk"}) + audio, err := tts.TTS(context.Background(), "你好") + if err != nil { + t.Fatalf("TTS error: %v", err) + } + if audio.Format != "wav" { + t.Errorf("format = %q, want wav", audio.Format) + } + if hex.EncodeToString(audio.Data) != hex.EncodeToString(audioBytes) { + t.Errorf("data = %s, want %s", hex.EncodeToString(audio.Data), hex.EncodeToString(audioBytes)) + } +} + +func TestDashscopeTTS_DataBlob(t *testing.T) { + t.Parallel() + audioBytes := []byte{9, 8, 7} + b64 := base64.StdEncoding.EncodeToString(audioBytes) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"output":{"audio":{"data":"` + b64 + `"}}}`)) + })) + defer srv.Close() + + tts := NewDashscopeTTS(config.VoiceConfig{DashscopeBaseURL: srv.URL, DashscopeAPIKey: "sk"}) + audio, err := tts.TTS(context.Background(), "hi") + if err != nil { + t.Fatalf("TTS error: %v", err) + } + if hex.EncodeToString(audio.Data) != hex.EncodeToString(audioBytes) { + t.Errorf("data = %s, want %s", hex.EncodeToString(audio.Data), hex.EncodeToString(audioBytes)) + } +} + +func TestDashscopeTTS_EmptyText(t *testing.T) { + t.Parallel() + tts := NewDashscopeTTS(config.VoiceConfig{DashscopeBaseURL: "http://x", DashscopeAPIKey: "sk"}) + if _, err := tts.TTS(context.Background(), " "); err != ErrEmptyText { + t.Fatalf("err = %v, want ErrEmptyText", err) + } +} + +func TestDashscopeTTS_NotConfigured(t *testing.T) { + t.Parallel() + tts := NewDashscopeTTS(config.VoiceConfig{}) + if _, err := tts.TTS(context.Background(), "hi"); err != ErrNotConfigured { + t.Fatalf("err = %v, want ErrNotConfigured", err) + } +} diff --git a/internal/services/gateway/audio.go b/internal/services/gateway/audio.go new file mode 100644 index 0000000..acfe8b7 --- /dev/null +++ b/internal/services/gateway/audio.go @@ -0,0 +1,122 @@ +package gateway + +import ( + "encoding/json" + "net/http" + "strconv" + "strings" + + apperr "git.yeij.top/AskaEth/Cyrene_v2/internal/errors" +) + +// maxAudioRequestBytes caps the JSON body a /api/v1/audio/* request may carry so +// a huge base64 audio payload cannot exhaust memory. +const maxAudioRequestBytes = 32 << 20 + +// transcribeRequest is the POST /api/v1/audio/transcribe body. Audio is a base64 +// (or data: URI) blob; Format is its encoding ("wav"/"mp3"/"ogg"). +type transcribeRequest struct { + Audio string `json:"audio"` + Format string `json:"format"` + DeviceID string `json:"deviceId,omitempty"` +} + +// speechRequest is the POST /api/v1/audio/speech body. Voice/format are accepted +// for forward compatibility; synthesis currently follows the server's +// VOICE_TTS_VOICE_ID / VOICE_TTS_FORMAT config. +type speechRequest struct { + Text string `json:"text"` + Voice string `json:"voice,omitempty"` + Format string `json:"format,omitempty"` +} + +// handleTranscribe runs STT on the uploaded audio and returns the recognized +// text. It is the REST source for native clients, bypassing the 4KB WebSocket +// frame limit that would truncate a base64 audio blob. +func (s *Server) handleTranscribe(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxAudioRequestBytes) + var req transcribeRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + s.auditClaims(r, "audio.transcribe", "error") + writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "invalid request body")) + return + } + if s.voice == nil { + s.auditClaims(r, "audio.transcribe", "error") + writeError(w, apperr.Wrap(apperr.ErrInternal, "voice_unavailable", "voice service is not configured")) + return + } + audio, err := decodeVoiceAudio(req.Audio, req.Format) + if err != nil { + s.auditClaims(r, "audio.transcribe", "error") + writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "audio could not be decoded")) + return + } + if len(audio.Data) == 0 { + s.auditClaims(r, "audio.transcribe", "error") + writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "audio is required")) + return + } + text, err := s.voice.STT(r.Context(), audio) + if err != nil { + s.auditClaims(r, "audio.transcribe", "error") + s.log.Warn("voice transcribe", "err", err) + writeError(w, apperr.Wrap(apperr.ErrInternal, "stt_failed", "speech transcription failed")) + return + } + s.auditClaims(r, "audio.transcribe", "ok") + writeJSON(w, http.StatusOK, map[string]any{"text": text}) +} + +// handleSpeech synthesizes speech for the given text and writes the raw audio +// bytes (audio/) back — a binary response, not a JSON envelope. +func (s *Server) handleSpeech(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxAudioRequestBytes) + var req speechRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + s.auditClaims(r, "audio.speech", "error") + writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "invalid request body")) + return + } + if strings.TrimSpace(req.Text) == "" { + s.auditClaims(r, "audio.speech", "error") + writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "text is required")) + return + } + if s.voice == nil { + s.auditClaims(r, "audio.speech", "error") + writeError(w, apperr.Wrap(apperr.ErrInternal, "voice_unavailable", "voice service is not configured")) + return + } + audi, err := s.voice.TTS(r.Context(), req.Text) + if err != nil { + s.auditClaims(r, "audio.speech", "error") + s.log.Warn("voice speech", "err", err) + writeError(w, apperr.Wrap(apperr.ErrInternal, "tts_failed", "speech synthesis failed")) + return + } + if len(audi.Data) == 0 { + s.auditClaims(r, "audio.speech", "error") + writeError(w, apperr.Wrap(apperr.ErrInternal, "tts_empty", "no audio synthesized")) + return + } + s.auditClaims(r, "audio.speech", "ok") + w.Header().Set("Content-Type", audioMime(audi.Format)) + w.Header().Set("Content-Length", strconv.Itoa(len(audi.Data))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(audi.Data) +} + +// audioMime maps an audio format to its Content-Type for the speech response. +func audioMime(format string) string { + switch strings.ToLower(format) { + case "mp3", "mpeg": + return "audio/mpeg" + case "ogg", "oga": + return "audio/ogg" + case "webm": + return "audio/webm" + default: + return "audio/wav" + } +} diff --git a/internal/services/gateway/audio_test.go b/internal/services/gateway/audio_test.go new file mode 100644 index 0000000..a7d38c5 --- /dev/null +++ b/internal/services/gateway/audio_test.go @@ -0,0 +1,124 @@ +package gateway + +import ( + "context" + "encoding/base64" + "net/http" + "testing" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/auth" + "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/multimodal/voice" +) + +// audioStubVoice is a VoiceService returning fixed STT/TTS results for the +// REST /api/v1/audio/* handler tests. +type audioStubVoice struct { + sttText string + sttErr error + ttsAudio voice.Audio + ttsErr error + gotAudio voice.Audio + gotText string + sttCalled bool + ttsCalled bool +} + +func (s *audioStubVoice) STT(ctx context.Context, audio voice.Audio) (string, error) { + s.sttCalled = true + s.gotAudio = audio + return s.sttText, s.sttErr +} + +func (s *audioStubVoice) TTS(ctx context.Context, text string) (voice.Audio, error) { + s.ttsCalled = true + s.gotText = text + return s.ttsAudio, s.ttsErr +} + +func TestHandleTranscribe(t *testing.T) { + env := newEnv(t, nil, false) + sv := &audioStubVoice{sttText: "你好世界"} + env.srv.SetVoice(sv) + + token := env.token(t, auth.RoleOwner) + resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/transcribe", map[string]any{ + "audio": base64.StdEncoding.EncodeToString([]byte("raw")), + "format": "wav", + }, token) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, body = %s", resp.StatusCode, body) + } + if !sv.sttCalled { + t.Fatal("STT not called") + } + if string(sv.gotAudio.Data) != "raw" || sv.gotAudio.Format != "wav" { + t.Errorf("audio = %+v, want format wav / data raw", sv.gotAudio) + } + env2 := decodeBody(t, body) + if got := string(env2["data"]); got == "" { + t.Errorf("data envelope empty: %s", body) + } +} + +func TestHandleTranscribe_AuthRequired(t *testing.T) { + env := newEnv(t, nil, false) + sv := &audioStubVoice{sttText: "x"} + env.srv.SetVoice(sv) + + resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/transcribe", map[string]any{ + "audio": base64.StdEncoding.EncodeToString([]byte("raw")), + }, "") + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 (body=%s)", resp.StatusCode, body) + } +} + +func TestHandleTranscribe_EmptyAudio(t *testing.T) { + env := newEnv(t, nil, false) + sv := &audioStubVoice{sttText: "x"} + env.srv.SetVoice(sv) + + token := env.token(t, auth.RoleOwner) + resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/transcribe", map[string]any{ + "audio": base64.StdEncoding.EncodeToString([]byte("")), + }, token) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (body=%s)", resp.StatusCode, body) + } +} + +func TestHandleSpeech(t *testing.T) { + env := newEnv(t, nil, false) + sv := &audioStubVoice{ttsAudio: voice.Audio{Format: "wav", Data: []byte("audio-bytes")}} + env.srv.SetVoice(sv) + + token := env.token(t, auth.RoleOwner) + resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/speech", map[string]any{ + "text": "你好", + }, token) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, body = %s", resp.StatusCode, body) + } + if ct := resp.Header.Get("Content-Type"); ct != "audio/wav" { + t.Errorf("Content-Type = %q, want audio/wav", ct) + } + if string(body) != "audio-bytes" { + t.Errorf("body = %q, want audio-bytes", body) + } + if !sv.ttsCalled || sv.gotText != "你好" { + t.Errorf("TTS got text = %q (called=%v), want 你好", sv.gotText, sv.ttsCalled) + } +} + +func TestHandleSpeech_EmptyText(t *testing.T) { + env := newEnv(t, nil, false) + env.srv.SetVoice(&audioStubVoice{}) + + token := env.token(t, auth.RoleOwner) + resp, body := doReq(t, http.MethodPost, env.ts.URL+"/api/v1/audio/speech", map[string]any{ + "text": " ", + }, token) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (body=%s)", resp.StatusCode, body) + } +} diff --git a/internal/services/gateway/server.go b/internal/services/gateway/server.go index bd97570..a989a12 100644 --- a/internal/services/gateway/server.go +++ b/internal/services/gateway/server.go @@ -385,6 +385,12 @@ func (s *Server) Handler() http.Handler { mux.Handle("GET /api/v1/sessions", readAuth(http.HandlerFunc(s.handleListSessions))) mux.Handle("GET /api/v1/personas", readAuth(http.HandlerFunc(s.handleListPersonas))) + // Audio (voice) STT/TTS. Owner-only (the owner's private voice assistant), + // audited. These are the REST surface for native clients to upload audio and + // fetch synthesized speech, bypassing the 4KB WebSocket frame limit. + mux.Handle("POST /api/v1/audio/transcribe", s.auth.Authorize(auth.RoleOwner)(http.HandlerFunc(s.handleTranscribe))) + mux.Handle("POST /api/v1/audio/speech", s.auth.Authorize(auth.RoleOwner)(http.HandlerFunc(s.handleSpeech))) + // Session/model reads and writes are the owner's private history; owner/admin only // (chat uses the same roles). Session metadata in GET /sessions stays readAuth for // any authenticated role, but the message content (and thinking) is owner/admin-only diff --git a/internal/services/gateway/voice.go b/internal/services/gateway/voice.go index de8cd22..135da8a 100644 --- a/internal/services/gateway/voice.go +++ b/internal/services/gateway/voice.go @@ -26,12 +26,14 @@ type VoiceService interface { // voiceInputRequest is the decoded payload of a voice_input frame. Audio is a // base64 (or data: URI) audio blob; Format is its encoding ("wav"/"mp3"/"ogg"); // Text overrides STT when present (a client that already transcribed can skip -// recognition); SessionID defaults to "web". +// recognition); SessionID defaults to "web"; DeviceID is the client-produced +// end identifier (design/16 §4.5) threaded into the persisted voice turn. type voiceInputRequest struct { Audio string `json:"audio"` Format string `json:"format"` Text string `json:"text"` SessionID string `json:"sessionId"` + DeviceID string `json:"deviceId,omitempty"` } // SetVoice injects the voice STT/TTS service used by the voice_input WS route. @@ -66,6 +68,9 @@ func (c *wsConn) handleVoiceInput(in wsInbound) { // carrying the base64 audio plus the recognized transcript and assistant reply. func (h *hub) runVoiceTurn(c *wsConn, req voiceInputRequest) { ctx := context.Background() + // Thread the client's deviceID so the persisted voice turn is tagged with the + // sending end (design/16 §4.5), mirroring ws_hub.go:240. + ctx = withDeviceID(ctx, req.DeviceID) if req.SessionID == "" { req.SessionID = "web" }