feat(voice): real TTS fallback chain and whisper-compatible STT

- TTS providers: cloud (OpenAI-compatible /audio/speech, e.g. Bailian
  compatible mode with cosyvoice), local self-hosted HTTP server,
  edge-tts CLI, espeak CLI, silent — ordered by TTS_PROVIDER list,
  skipping providers whose binary/config is missing
- STT: whisper-compatible /audio/transcriptions endpoint (cloud or
  self-hosted faster-whisper), replacing the empty dashscope stub
- config: TTS_CLOUD_URL/KEY/MODEL/VOICE, TTS_LOCAL_URL, STT_API_URL/
  KEY/MODEL; cloud keys fall back to DASHSCOPE_API_KEY
- gateway /voice/tts now returns real audio/mpeg bytes instead of a
  byte count
- tests: cloud/local/whisper round-trips via httptest, chain skip
  behavior, fallback order; kept and adapted prior service tests
This commit is contained in:
Aska committed 2026-08-21 18:28:39 +08:00
1 parent b8fb7feb86
commit a0357a7bbd
7 files changed
+410 -63

No files matched your search

+28 -4
View File
@@ -10,6 +10,7 @@ import (
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
@@ -124,12 +125,24 @@ func main() {
logger.Infof("[plugin] loaded %d plugins: %v", reg.Count(), reg.ListNames())
// === 5. Voice Service ===
orEmpty := func(v, fallback string) string {
if v != "" {
return v
}
return fallback
}
voiceSvc := voicesvc.New(voicesvc.Config{
STTProvider: cfg.ASRProvider,
TTSProviders: []string{cfg.TTSProvider},
DashScopeKey: cfg.DashScopeAPIKey,
STTApiURL: cfg.STTApiURL,
STTApiKey: orEmpty(cfg.STTApiKey, cfg.DashScopeAPIKey),
STTModel: cfg.STTModel,
TTSProviders: splitComma(cfg.TTSProvider),
TTSCloudURL: cfg.TTSCloudURL,
TTSCloudKey: orEmpty(cfg.TTSCloudKey, cfg.DashScopeAPIKey),
TTSCloudModel: cfg.TTSCloudModel,
TTSCloudVoice: cfg.TTSCloudVoice,
TTSLocalURL: cfg.TTSLocalURL,
})
logger.Infof("[voice] ready")
logger.Infof("[voice] ready (tts chain: %v)", voiceSvc.TTSProviderNames())
// === 6. Memory Service ===
var memSvc *memory.Service
@@ -407,6 +420,17 @@ type llmLoggerAdapter struct{ *llmlog.CallLogger }
func (a *llmLoggerAdapter) Recent(n int) []map[string]any { return a.CallLogger.RecentMaps(n) }
// splitComma splits a comma-separated config value into trimmed non-empty parts.
func splitComma(s string) []string {
var out []string
for _, p := range strings.Split(s, ",") {
if p = strings.TrimSpace(p); p != "" {
out = append(out, p)
}
}
return out
}
func pluginNamesToMap(names []string) []map[string]string {
result := make([]map[string]string, len(names))
for i, n := range names {
+14 -3
View File
@@ -53,8 +53,16 @@ type Config struct {
// Voice
DashScopeAPIKey string
TTSProvider string // "edge-tts" | "espeak"
ASRProvider string // "dashscope" | "whisper-local"
TTSProvider string // comma-separated fallback chain: "cloud,local,edge-tts,espeak,silent"
TTSCloudURL string // OpenAI-compatible base URL for cloud TTS (e.g. Bailian compatible mode)
TTSCloudKey string // falls back to DashScopeAPIKey
TTSCloudModel string
TTSCloudVoice string
TTSLocalURL string // self-hosted TTS POST endpoint
ASRProvider string // legacy label
STTApiURL string // Whisper-compatible base URL (cloud or self-hosted)
STTApiKey string // falls back to DashScopeAPIKey
STTModel string
// Auth
JWTSecret string
@@ -139,7 +147,10 @@ func DefaultConfig() *Config {
EmbeddingModel: "text-embedding-3-small",
TTSProvider: "edge-tts",
TTSProvider: "cloud,local,edge-tts,silent",
TTSCloudURL: "https://dashscope.aliyuncs.com/compatible-mode/v1",
TTSCloudModel: "cosyvoice-v1",
TTSCloudVoice: "longxiaochun",
ASRProvider: "dashscope",
JWTExpiryHours: 720,
+8
View File
@@ -93,7 +93,15 @@ func (c *Config) mergeEnv() {
// Voice
c.DashScopeAPIKey = getEnv("DASHSCOPE_API_KEY", c.DashScopeAPIKey)
c.TTSProvider = getEnv("TTS_PROVIDER", c.TTSProvider)
c.TTSCloudURL = getEnv("TTS_CLOUD_URL", c.TTSCloudURL)
c.TTSCloudKey = getEnv("TTS_CLOUD_KEY", c.TTSCloudKey)
c.TTSCloudModel = getEnv("TTS_CLOUD_MODEL", c.TTSCloudModel)
c.TTSCloudVoice = getEnv("TTS_CLOUD_VOICE", c.TTSCloudVoice)
c.TTSLocalURL = getEnv("TTS_LOCAL_URL", c.TTSLocalURL)
c.ASRProvider = getEnv("ASR_PROVIDER", c.ASRProvider)
c.STTApiURL = getEnv("STT_API_URL", c.STTApiURL)
c.STTApiKey = getEnv("STT_API_KEY", c.STTApiKey)
c.STTModel = getEnv("STT_MODEL", c.STTModel)
// Auth
c.JWTSecret = getEnv("JWT_SECRET", c.JWTSecret)
+7 -1
View File
@@ -476,7 +476,13 @@ func (r *Router) voiceTTSHandler(w http.ResponseWriter, req *http.Request) {
writeJSON(w, 500, M{"error": err.Error()})
return
}
writeJSON(w, 200, M{"audio_size": len(audio)})
if len(audio) == 0 {
writeJSON(w, 503, M{"error": "no tts provider available"})
return
}
w.Header().Set("Content-Type", "audio/mpeg")
w.WriteHeader(200)
w.Write(audio)
}
// --- Admin ---
+192
View File
@@ -0,0 +1,192 @@
package voice
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// --- Cloud TTS (OpenAI-compatible /audio/speech, e.g. Alibaba Bailian compatible mode) ---
type cloudTTS struct {
baseURL, apiKey, model, defaultVoice string
client *http.Client
}
func (p *cloudTTS) Name() string { return "cloud" }
func (p *cloudTTS) Synthesize(ctx context.Context, text string, voice string) ([]byte, error) {
if voice == "" {
voice = p.defaultVoice
}
body, _ := json.Marshal(map[string]any{
"model": p.model,
"voice": voice,
"input": text,
})
req, err := http.NewRequestWithContext(ctx, "POST", p.baseURL+"/audio/speech", bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+p.apiKey)
req.Header.Set("Content-Type", "application/json")
resp, err := p.client.Do(req)
if err != nil {
return nil, fmt.Errorf("cloud tts: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return nil, fmt.Errorf("cloud tts %d: %s", resp.StatusCode, msg)
}
return io.ReadAll(resp.Body)
}
// --- Local TTS (self-hosted HTTP TTS server, e.g. GPT-SoVITS / edge-tts-server / OpenTTS) ---
type localTTS struct {
url string // POST endpoint accepting {"text","voice"} and returning audio bytes
client *http.Client
}
func (p *localTTS) Name() string { return "local" }
func (p *localTTS) Synthesize(ctx context.Context, text string, voice string) ([]byte, error) {
body, _ := json.Marshal(map[string]any{"text": text, "voice": voice})
req, err := http.NewRequestWithContext(ctx, "POST", p.url, bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
resp, err := p.client.Do(req)
if err != nil {
return nil, fmt.Errorf("local tts: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return nil, fmt.Errorf("local tts %d", resp.StatusCode)
}
return io.ReadAll(resp.Body)
}
// --- edge-tts CLI ---
type edgeTTS struct{ bin string }
// newEdgeTTS locates the edge-tts binary: EDGETTS_BIN env, PATH, then ./.venv.
func newEdgeTTS() *edgeTTS {
if bin := os.Getenv("EDGETTS_BIN"); bin != "" {
return &edgeTTS{bin: bin}
}
if bin, err := exec.LookPath("edge-tts"); err == nil {
return &edgeTTS{bin: bin}
}
if _, err := os.Stat(".venv/bin/edge-tts"); err == nil {
abs, _ := filepath.Abs(".venv/bin/edge-tts")
return &edgeTTS{bin: abs}
}
return &edgeTTS{}
}
func (p *edgeTTS) Name() string { return "edge-tts" }
func (p *edgeTTS) available() bool { return p.bin != "" }
func (p *edgeTTS) Synthesize(ctx context.Context, text string, voice string) ([]byte, error) {
if !p.available() {
return nil, fmt.Errorf("edge-tts binary not found")
}
if voice == "" {
voice = "zh-CN-XiaoxiaoNeural"
}
out := filepath.Join(os.TempDir(), fmt.Sprintf("cyrene-tts-%d.mp3", time.Now().UnixNano()))
defer os.Remove(out)
cmd := exec.CommandContext(ctx, p.bin, "--voice", voice, "--text", text, "--write-media", out)
if combined, err := cmd.CombinedOutput(); err != nil {
return nil, fmt.Errorf("edge-tts: %w: %s", err, strings.TrimSpace(string(combined)))
}
return os.ReadFile(out)
}
// --- espeak CLI ---
type espeakTTS struct{ bin string }
func newEspeakTTS() *espeakTTS {
for _, name := range []string{"espeak-ng", "espeak"} {
if bin, err := exec.LookPath(name); err == nil {
return &espeakTTS{bin: bin}
}
}
return &espeakTTS{}
}
func (p *espeakTTS) Name() string { return "espeak" }
func (p *espeakTTS) Synthesize(ctx context.Context, text string, _ string) ([]byte, error) {
if p.bin == "" {
return nil, fmt.Errorf("espeak binary not found")
}
cmd := exec.CommandContext(ctx, p.bin, "-v", "zh", "--stdout", text)
return cmd.Output()
}
// --- Whisper-compatible STT (OpenAI /audio/transcriptions; local servers like faster-whisper too) ---
type whisperSTT struct {
baseURL, apiKey, model string
client *http.Client
}
func (p *whisperSTT) Name() string { return "whisper-api" }
func (p *whisperSTT) Recognize(ctx context.Context, audioData []byte) (string, error) {
var buf bytes.Buffer
w := multipart.NewWriter(&buf)
fw, err := w.CreateFormFile("file", "audio.wav")
if err != nil {
return "", err
}
if _, err := fw.Write(audioData); err != nil {
return "", err
}
w.WriteField("model", p.model)
w.Close()
req, err := http.NewRequestWithContext(ctx, "POST", p.baseURL+"/audio/transcriptions", &buf)
if err != nil {
return "", err
}
req.Header.Set("Authorization", "Bearer "+p.apiKey)
req.Header.Set("Content-Type", w.FormDataContentType())
resp, err := p.client.Do(req)
if err != nil {
return "", fmt.Errorf("stt: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return "", fmt.Errorf("stt %d: %s", resp.StatusCode, msg)
}
var result struct {
Text string `json:"text"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return "", fmt.Errorf("parse stt: %w", err)
}
return result.Text, nil
}
+64 -31
View File
@@ -1,13 +1,20 @@
// Package voice provides speech-to-text and text-to-speech capabilities.
// It defines provider interfaces allowing pluggable backends (DashScope, Whisper, Edge-TTS, espeak).
// TTS providers form a fallback chain: cloud (Bailian/OpenAI-compatible),
// local self-hosted HTTP server, edge-tts CLI, espeak CLI, silent.
// STT uses a Whisper-compatible /audio/transcriptions endpoint
// (cloud or self-hosted, e.g. faster-whisper).
package voice
import "context"
import (
"context"
"net/http"
"time"
)
// STTProvider converts speech audio to text.
type STTProvider interface {
// Name returns the provider identifier.
Name() string // "dashscope", "whisper-local"
Name() string // "whisper-api", "silent"
// Recognize converts audio data to text.
Recognize(ctx context.Context, audioData []byte) (string, error)
@@ -16,7 +23,7 @@ type STTProvider interface {
// TTSProvider converts text to speech audio.
type TTSProvider interface {
// Name returns the provider identifier.
Name() string // "edge-tts", "espeak", "silent"
Name() string // "cloud", "local", "edge-tts", "espeak", "silent"
// Synthesize converts text to audio bytes.
Synthesize(ctx context.Context, text string, voice string) ([]byte, error)
@@ -30,18 +37,35 @@ type Service struct {
// Config holds voice service configuration.
type Config struct {
STTProvider string // "dashscope" | "whisper-local"
TTSProviders []string // ["edge-tts", "espeak", "silent"] ordered by priority
DashScopeKey string
// STT (Whisper-compatible endpoint; works with Bailian compatible mode
// or a self-hosted faster-whisper server)
STTApiURL string // e.g. "https://dashscope.aliyuncs.com/compatible-mode/v1"
STTApiKey string
STTModel string // e.g. "whisper-large-v3"
// TTS fallback chain, e.g. ["cloud", "local", "edge-tts", "silent"]
TTSProviders []string
TTSCloudURL string // OpenAI-compatible base URL for cloud TTS
TTSCloudKey string
TTSCloudModel string // e.g. "cosyvoice-v1" (Bailian) / "tts-1" (OpenAI)
TTSCloudVoice string // e.g. "longxiaochun"
TTSLocalURL string // self-hosted TTS POST endpoint
}
// New creates a new voice service.
func New(cfg Config) *Service {
svc := &Service{}
// STT: for now, use a simple placeholder
if cfg.STTProvider == "dashscope" && cfg.DashScopeKey != "" {
svc.sttProvider = &dashscopeSTT{apiKey: cfg.DashScopeKey}
// STT
if cfg.STTApiURL != "" && cfg.STTApiKey != "" {
model := cfg.STTModel
if model == "" {
model = "whisper-large-v3"
}
svc.sttProvider = &whisperSTT{
baseURL: cfg.STTApiURL, apiKey: cfg.STTApiKey, model: model,
client: &http.Client{Timeout: 60 * time.Second},
}
} else {
svc.sttProvider = &silentSTT{}
}
@@ -49,10 +73,28 @@ func New(cfg Config) *Service {
// TTS: build fallback chain
for _, name := range cfg.TTSProviders {
switch name {
case "cloud":
if cfg.TTSCloudURL != "" && cfg.TTSCloudKey != "" {
svc.ttsProviders = append(svc.ttsProviders, &cloudTTS{
baseURL: cfg.TTSCloudURL, apiKey: cfg.TTSCloudKey,
model: cfg.TTSCloudModel, defaultVoice: cfg.TTSCloudVoice,
client: &http.Client{Timeout: 60 * time.Second},
})
}
case "local":
if cfg.TTSLocalURL != "" {
svc.ttsProviders = append(svc.ttsProviders, &localTTS{
url: cfg.TTSLocalURL, client: &http.Client{Timeout: 60 * time.Second},
})
}
case "edge-tts":
svc.ttsProviders = append(svc.ttsProviders, &edgeTTS{})
if e := newEdgeTTS(); e.available() {
svc.ttsProviders = append(svc.ttsProviders, e)
}
case "espeak":
svc.ttsProviders = append(svc.ttsProviders, &espeakTTS{})
if e := newEspeakTTS(); e.bin != "" {
svc.ttsProviders = append(svc.ttsProviders, e)
}
case "silent":
svc.ttsProviders = append(svc.ttsProviders, &silentTTS{})
}
@@ -64,6 +106,15 @@ func New(cfg Config) *Service {
return svc
}
// TTSProviderNames returns the active TTS fallback chain.
func (s *Service) TTSProviderNames() []string {
names := make([]string, len(s.ttsProviders))
for i, p := range s.ttsProviders {
names[i] = p.Name()
}
return names
}
// Recognize performs speech recognition with the configured STT provider.
func (s *Service) Recognize(ctx context.Context, audioData []byte) (string, error) {
return s.sttProvider.Recognize(ctx, audioData)
@@ -82,13 +133,7 @@ func (s *Service) Synthesize(ctx context.Context, text string, voice string) ([]
return nil, lastErr
}
// --- Provider stubs ---
type dashscopeSTT struct{ apiKey string }
func (p *dashscopeSTT) Name() string { return "dashscope" }
func (p *dashscopeSTT) Recognize(_ context.Context, _ []byte) (string, error) {
return "", nil // Placeholder: real implementation requires DashScope SDK
}
// --- Silent fallbacks ---
type silentSTT struct{}
func (p *silentSTT) Name() string { return "silent" }
@@ -96,18 +141,6 @@ func (p *silentSTT) Recognize(_ context.Context, _ []byte) (string, error) {
return "", nil
}
type edgeTTS struct{}
func (p *edgeTTS) Name() string { return "edge-tts" }
func (p *edgeTTS) Synthesize(_ context.Context, text string, _ string) ([]byte, error) {
return []byte(text), nil // Placeholder
}
type espeakTTS struct{}
func (p *espeakTTS) Name() string { return "espeak" }
func (p *espeakTTS) Synthesize(_ context.Context, text string, _ string) ([]byte, error) {
return []byte(text), nil
}
type silentTTS struct{}
func (p *silentTTS) Name() string { return "silent" }
func (p *silentTTS) Synthesize(_ context.Context, _ string, _ string) ([]byte, error) {
+97 -24
View File
@@ -2,14 +2,15 @@ package voice
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestNewServiceDefaultConfig(t *testing.T) {
svc := New(Config{
STTProvider: "silent",
TTSProviders: []string{"silent"},
})
svc := New(Config{TTSProviders: []string{"silent"}})
if svc.sttProvider == nil {
t.Fatal("expected STT provider")
@@ -28,6 +29,9 @@ func TestNewServiceEmptyConfig(t *testing.T) {
if len(svc.ttsProviders) == 0 {
t.Fatal("expected at least one fallback TTS provider")
}
if got := svc.TTSProviderNames(); len(got) != 1 || got[0] != "silent" {
t.Errorf("expected [silent] chain, got %v", got)
}
}
func TestServiceRecognize(t *testing.T) {
@@ -54,37 +58,106 @@ func TestServiceSynthesize(t *testing.T) {
}
}
func TestTTSProviderChain(t *testing.T) {
svc := New(Config{
TTSProviders: []string{"edge-tts", "espeak", "silent"},
})
func TestTTSProviderChainSkipsUnavailable(t *testing.T) {
// edge-tts/espeak binaries are usually absent in CI — they must be
// skipped, leaving at least the silent provider.
svc := New(Config{TTSProviders: []string{"edge-tts", "espeak", "silent"}})
if len(svc.ttsProviders) != 3 {
t.Errorf("expected 3 TTS providers, got %d", len(svc.ttsProviders))
names := svc.TTSProviderNames()
if names[len(names)-1] != "silent" {
t.Errorf("chain must end with silent, got %v", names)
}
names := []string{"edge-tts", "espeak", "silent"}
for i, p := range svc.ttsProviders {
if p.Name() != names[i] {
t.Errorf("provider[%d]: expected %s, got %s", i, names[i], p.Name())
for _, n := range names {
if n == "edge-tts" && newEdgeTTS().bin == "" {
t.Error("edge-tts in chain but binary not found")
}
}
}
func TestProviderNames(t *testing.T) {
if n := (&dashscopeSTT{}).Name(); n != "dashscope" {
t.Errorf("got %s", n)
cases := map[string]TTSProvider{
"cloud": &cloudTTS{},
"local": &localTTS{},
"edge-tts": &edgeTTS{},
"espeak": &espeakTTS{},
"silent": &silentTTS{},
}
for want, p := range cases {
if p.Name() != want {
t.Errorf("provider name = %s, want %s", p.Name(), want)
}
}
if n := (STTProvider)(&whisperSTT{}).Name(); n != "whisper-api" {
t.Errorf("stt name = %s", n)
}
if n := (&silentSTT{}).Name(); n != "silent" {
t.Errorf("got %s", n)
t.Errorf("stt name = %s", n)
}
if n := (&edgeTTS{}).Name(); n != "edge-tts" {
t.Errorf("got %s", n)
}
func TestCloudTTS(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/audio/speech" {
t.Errorf("unexpected path: %s", r.URL.Path)
}
body, _ := io.ReadAll(r.Body)
if !strings.Contains(string(body), "cosyvoice-v1") || !strings.Contains(string(body), "你好") {
t.Errorf("unexpected body: %s", body)
}
w.Write([]byte("fake-mp3-bytes"))
}))
defer srv.Close()
p := &cloudTTS{baseURL: srv.URL, apiKey: "k", model: "cosyvoice-v1", defaultVoice: "longxiaochun", client: srv.Client()}
audio, err := p.Synthesize(context.Background(), "你好", "")
if err != nil {
t.Fatalf("Synthesize: %v", err)
}
if n := (&espeakTTS{}).Name(); n != "espeak" {
t.Errorf("got %s", n)
if string(audio) != "fake-mp3-bytes" {
t.Errorf("audio = %q", audio)
}
if n := (&silentTTS{}).Name(); n != "silent" {
t.Errorf("got %s", n)
}
func TestLocalTTS(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("local-audio"))
}))
defer srv.Close()
p := &localTTS{url: srv.URL, client: srv.Client()}
audio, err := p.Synthesize(context.Background(), "文本", "")
if err != nil || string(audio) != "local-audio" {
t.Errorf("audio=%q err=%v", audio, err)
}
}
func TestWhisperSTT(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/audio/transcriptions" {
t.Errorf("unexpected path: %s", r.URL.Path)
}
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Errorf("parse multipart: %v", err)
}
w.Write([]byte(`{"text":"识别结果"}`))
}))
defer srv.Close()
p := &whisperSTT{baseURL: srv.URL, apiKey: "k", model: "whisper-large-v3", client: srv.Client()}
text, err := p.Recognize(context.Background(), []byte("RIFF....fake-wav"))
if err != nil || text != "识别结果" {
t.Errorf("text=%q err=%v", text, err)
}
}
func TestFallbackChain(t *testing.T) {
// First provider unreachable, second (silent) succeeds
svc := &Service{ttsProviders: []TTSProvider{
&localTTS{url: "http://127.0.0.1:1", client: &http.Client{}},
&silentTTS{},
}}
audio, err := svc.Synthesize(context.Background(), "text", "")
if err != nil || len(audio) != 0 {
t.Errorf("expected silent fallback empty audio, got %d bytes, err=%v", len(audio), err)
}
}