package voice import ( "bytes" "context" "encoding/base64" "encoding/json" "fmt" "io" "net/http" "strings" "unicode" "git.yeij.top/AskaEth/Cyrene_v2/internal/config" ) // DashScope (Alibaba Bailian) STT/TTS adapters for the cloud rung of the voice // chain. Requests follow the DashScope public API: // // - STT: POST {base}/services/aigc/multimodal-generation/generation // (qwen-audio-3.0-asr-flash, base64 input audio) // - TTS: POST {base}/services/audio/tts/SpeechSynthesizer // (cosyvoice-v3-flash, synthesized by voice_id) // // Both take a non-streaming response here: STT returns the transcript in the // output envelope; TTS (no X-DashScope-SSE header) returns a 24h audio URL that // we download so the caller always gets the actual bytes. Audio format ↔ mime // is mapped in both directions (mimeOf / the data: URI in the STT request). const ( // defaultSTTSampleRate is the input audio sample rate the ASR expects. defaultSTTSampleRate = 16000 // dashscopeSTTPath is the multimodal-generation service path (ASR). dashscopeSTTPath = "/services/aigc/multimodal-generation/generation" // dashscopeTTSPath is the SpeechSynthesizer service path (TTS). dashscopeTTSPath = "/services/audio/tts/SpeechSynthesizer" // dashscopeClonePath is the voice-customization (声音复刻 create_voice) path. dashscopeClonePath = "/services/audio/tts/customization" // defaultDashscopeBaseURL is the Bailian API root when none is configured. defaultDashscopeBaseURL = "https://dashscope.aliyuncs.com/api/v1" // defaultClonePrefix is the voice-name prefix when none is supplied. defaultClonePrefix = "cyrene" // voiceEnrollmentModel is the DashScope voice-enrollment model ("create_voice"). voiceEnrollmentModel = "voice-enrollment" ) // DashscopeSTT transcribes audio via DashScope multimodal-generation. type DashscopeSTT struct { endpoint string apiKey string model string format string language string sampleRate int client *http.Client } // NewDashscopeSTT builds the DashScope STT adapter from cfg. Missing model / // language / base URL fall back to the DashScope defaults. func NewDashscopeSTT(cfg config.VoiceConfig) *DashscopeSTT { model := firstNonEmpty(cfg.STTModel, "qwen-audio-3.0-asr-flash") language := firstNonEmpty(cfg.STTLanguage, "zh") return &DashscopeSTT{ endpoint: dashscopeEndpoint(cfg.DashscopeBaseURL) + dashscopeSTTPath, apiKey: cfg.DashscopeAPIKey, model: model, format: "wav", language: language, sampleRate: defaultSTTSampleRate, client: defaultHTTPClient(), } } // STT posts the audio as a base64 data: URI and returns the recognized text. func (s *DashscopeSTT) STT(ctx context.Context, audio Audio) (string, error) { if s.apiKey == "" { return "", ErrNotConfigured } if len(audio.Data) == 0 { return "", fmt.Errorf("voice: %w: empty audio", ErrProviderFailed) } if err := ctx.Err(); err != nil { return "", err } format := audio.Format if format == "" { format = "wav" } payload, err := json.Marshal(dashscopeSTTRequest{ Model: s.model, Input: dashscopeSTTInput{ Messages: []dashscopeSTTMessage{{ Role: "user", Content: []dashscopeSTTContent{{ Type: "input_audio", InputAudio: dashscopeInputAudio{ Data: "data:" + mimeOf(format) + ";base64," + base64.StdEncoding.EncodeToString(audio.Data), }, }}, }}, }, Parameters: dashscopeSTTParameters{ Format: format, SampleRate: s.sampleRate, LanguageHints: []string{s.language}, }, }) if err != nil { return "", fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err) } body, err := dashscopePost(ctx, s.client, s.apiKey, s.endpoint, payload, "disable") if err != nil { return "", err } text, err := extractSTTText(body) if err != nil { return "", fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err) } return text, nil } // DashscopeTTS synthesizes speech video via DashScope SpeechSynthesizer. type DashscopeTTS struct { endpoint string apiKey string model string voiceID string format string sampleRate int instruction string client *http.Client } // NewDashscopeTTS builds the DashScope TTS adapter from cfg. Missing model / // voice / format / base URL fall back to the DashScope defaults. func NewDashscopeTTS(cfg config.VoiceConfig) *DashscopeTTS { model := firstNonEmpty(cfg.TTSModel, "cosyvoice-v3-flash") voiceID := firstNonEmpty(cfg.TTSVoiceID, "longxiaochun_v3") format := firstNonEmpty(cfg.TTSFormat, "wav") sampleRate := cfg.TTSSampleRate if sampleRate == 0 { sampleRate = 24000 } return &DashscopeTTS{ endpoint: dashscopeEndpoint(cfg.DashscopeBaseURL) + dashscopeTTSPath, apiKey: cfg.DashscopeAPIKey, model: model, voiceID: voiceID, format: format, sampleRate: sampleRate, instruction: cfg.VoiceTTSInstruction, client: defaultHTTPClient(), } } // TTS synthesizes audio for text. The non-streaming DashScope response carries a // 24h audio URL (or a base64 blob); we resolve it to bytes and return them so the // caller holds a complete audio file. func (t *DashscopeTTS) TTS(ctx context.Context, text string) (Audio, error) { if t.apiKey == "" { return Audio{}, ErrNotConfigured } if strings.TrimSpace(text) == "" { return Audio{}, ErrEmptyText } if err := ctx.Err(); err != nil { return Audio{}, err } req := dashscopeTTSRequest{ Model: t.model, Input: dashscopeTTSInput{ Text: text, Voice: t.voiceID, Format: t.format, SampleRate: t.sampleRate, }, } if instruction := t.EffectiveInstruction(ctx); instruction != "" { req.Input.Instruction = instruction } payload, err := json.Marshal(req) if err != nil { return Audio{}, fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err) } // No X-DashScope-SSE header → non-streaming URL response. body, err := dashscopePost(ctx, t.client, t.apiKey, t.endpoint, payload, "") if err != nil { return Audio{}, err } data, err := t.audioBytes(ctx, body) if err != nil { return Audio{}, err } return Audio{Format: t.format, Data: data}, nil } // EffectiveInstruction returns the instruction to send for this synthesis: a // per-request override threaded through ctx wins, else the configured default. An // empty result means the request carries no instruction. A system voice never gets // a free-form instruction (see canInstruct), because cosyvoice v3 system voices // require a fixed instruction format. func (t *DashscopeTTS) EffectiveInstruction(ctx context.Context) string { if !canInstruct(t.voiceID) { return "" } if override := TTSInstructionFromContext(ctx); override != "" { return override } return t.instruction } // canInstruct reports whether voiceID accepts a free-form instruction. cosyvoice v3 // system voices (the long*_v3 family, e.g. longxiaochun_v3) require a fixed // instruction format; a cloned/designed voice (声音复刻, whose id never ends in // "_v3") accepts an arbitrary instruction. func canInstruct(voiceID string) bool { return !strings.HasSuffix(voiceID, "_v3") } // audioBytes resolves the synthesized audio from the DashScope response: a 24h // URL is downloaded, or an inline base64 blob is decoded. func (t *DashscopeTTS) audioBytes(ctx context.Context, body []byte) ([]byte, error) { var r dashscopeTTSResponse if err := json.Unmarshal(body, &r); err != nil { return nil, fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err) } if url := r.Output.Audio.URL; url != "" { data, err := t.download(ctx, url) if err != nil { return nil, err } return data, nil } if b64 := r.Output.Audio.Data; b64 != "" { data, err := base64.StdEncoding.DecodeString(b64) if err != nil { return nil, fmt.Errorf("voice: %w: decode audio: %v", ErrProviderFailed, err) } return data, nil } return nil, fmt.Errorf("voice: %w: no audio in response", ErrProviderFailed) } // download fetches an audio URL into memory, capping the size so an oversized // media response cannot exhaust memory. func (t *DashscopeTTS) download(ctx context.Context, url string) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, fmt.Errorf("voice: %w: build: %v", ErrProviderFailed, err) } resp, err := t.client.Do(req) if err != nil { return nil, fmt.Errorf("voice: %w: download: %v", ErrProviderFailed, err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("voice: %w: download status %d", ErrProviderFailed, resp.StatusCode) } data, err := io.ReadAll(io.LimitReader(resp.Body, cloudMaxResponseBytes)) if err != nil { return nil, fmt.Errorf("voice: %w: download read: %v", ErrProviderFailed, err) } return data, nil } // DashscopeCloner creates custom TTS voices via the DashScope voice-enrollment // API (声音复刻 create_voice). The reference audio must be a public URL — the // create_voice action only accepts input.url (no base64). The returned voice_id is // bound to TTSModel, so target_model must match the model used at synthesis time. type DashscopeCloner struct { endpoint string apiKey string model string client *http.Client } // NewDashscopeCloner builds the DashScope voice cloner from cfg. Missing model / // base URL fall back to the DashScope defaults; the model tracks TTSModel so the // cloned voice is usable by the TTS rung (target_model must match). func NewDashscopeCloner(cfg config.VoiceConfig) *DashscopeCloner { return &DashscopeCloner{ endpoint: dashscopeEndpoint(cfg.DashscopeBaseURL) + dashscopeClonePath, apiKey: cfg.DashscopeAPIKey, model: firstNonEmpty(cfg.TTSModel, "cosyvoice-v3-flash"), client: defaultHTTPClient(), } } // CloneVoice submits a create_voice request and returns the created voice_id. It // is the adapter behind the REST /api/v1/audio/voice-clone endpoint. A missing API // key returns ErrNotConfigured; an empty URL or an invalid prefix is rejected. func (d *DashscopeCloner) CloneVoice(ctx context.Context, referenceURL, prefix string) (string, error) { if d.apiKey == "" { return "", ErrNotConfigured } referenceURL = strings.TrimSpace(referenceURL) if referenceURL == "" { return "", fmt.Errorf("voice: %w: reference audio url is required", ErrProviderFailed) } prefix, err := normalizeClonePrefix(prefix) if err != nil { return "", err } if err := ctx.Err(); err != nil { return "", err } payload, err := json.Marshal(dashscopeCloneRequest{ Model: voiceEnrollmentModel, Input: dashscopeCloneInput{ Action: "create_voice", TargetModel: d.model, Prefix: prefix, URL: referenceURL, }, }) if err != nil { return "", fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err) } body, err := dashscopePost(ctx, d.client, d.apiKey, d.endpoint, payload, "") if err != nil { return "", err } var resp dashscopeCloneResponse if err := json.Unmarshal(body, &resp); err != nil { return "", fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err) } if resp.Message != "" { return "", fmt.Errorf("voice: %w: %s", ErrProviderFailed, resp.Message) } if resp.Output.VoiceID == "" { return "", fmt.Errorf("voice: %w: no voice_id in response", ErrProviderFailed) } return resp.Output.VoiceID, nil } // normalizeClonePrefix trims and validates the voice-name prefix. An empty value // falls back to defaultClonePrefix; the result must be ≤10 letter/digit chars // (the voice-enrollment prefix constraint). func normalizeClonePrefix(prefix string) (string, error) { prefix = strings.TrimSpace(prefix) if prefix == "" { return defaultClonePrefix, nil } if len(prefix) > 10 { return "", fmt.Errorf("voice: %w: prefix %q exceeds 10 chars", ErrProviderFailed, prefix) } for _, r := range prefix { if !unicode.IsLetter(r) && !unicode.IsDigit(r) { return "", fmt.Errorf("voice: %w: prefix must be alphanumeric", ErrProviderFailed) } } return prefix, nil } // dashscopeEndpoint returns the DashScope API base with a trailing slash trimmed, // defaulting to the public Bailian root when none is configured. func dashscopeEndpoint(baseURL string) string { base := strings.TrimRight(baseURL, "/") if base == "" { return defaultDashscopeBaseURL } return base } // firstNonEmpty returns a when non-empty, else b. func firstNonEmpty(a, b string) string { if a != "" { return a } return b } // dashscopePost POSTs a JSON payload with the DashScope bearer auth and an // optional X-DashScope-SSE header, returning the decoded response body. A non-200 // status is wrapped with the DashScope error body. func dashscopePost(ctx context.Context, client *http.Client, apiKey, url string, payload []byte, sse string) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) if err != nil { return nil, fmt.Errorf("voice: %w: build: %v", ErrProviderFailed, err) } req.Header.Set("Content-Type", "application/json") if apiKey != "" { req.Header.Set("Authorization", "Bearer "+apiKey) } if sse != "" { req.Header.Set("X-DashScope-SSE", sse) } resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("voice: %w: request: %v", ErrProviderFailed, err) } defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(io.LimitReader(resp.Body, cloudMaxResponseBytes)) if err != nil { return nil, fmt.Errorf("voice: %w: read: %v", ErrProviderFailed, err) } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("voice: %w: dashscope status %d: %s", ErrProviderFailed, resp.StatusCode, strings.TrimSpace(string(body))) } return body, nil } // extractSTTText pulls the transcript out of a DashScope ASR response. The ASR // output shape varies by model, so several candidate paths are tried. func extractSTTText(body []byte) (string, error) { var r dashscopeSTTResponse if err := json.Unmarshal(body, &r); err != nil { return "", err } if s := r.Output.Output.Sentence.Text; s != "" { return s, nil } if s := r.Output.Output.Text; s != "" { return s, nil } if s := r.Output.Text; s != "" { return s, nil } for _, ch := range r.Output.Choices { if len(ch.Message.Content) > 0 { if s := ch.Message.Content[0].Text; s != "" { return s, nil } } } return r.Text, nil } // mimeOf maps an audio format to its MIME type for the data: URI / Content-Type. func mimeOf(format string) string { switch strings.ToLower(format) { case "mp3", "mpeg": return "audio/mpeg" case "ogg", "oga": return "audio/ogg" case "webm": return "audio/webm" default: return "audio/wav" } } // dashscopeSTTResponse is the DashScope ASR response envelope. type dashscopeSTTResponse struct { Output struct { Output struct { Sentence struct { Text string `json:"text"` } `json:"sentence"` Text string `json:"text"` } `json:"output"` Text string `json:"text"` Choices []struct { Message struct { Content []struct { Text string `json:"text"` } `json:"content"` } `json:"message"` } `json:"choices"` } `json:"output"` Text string `json:"text"` } // dashscopeTTSResponse is the DashScope TTS response envelope. type dashscopeTTSResponse struct { Output struct { Audio struct { URL string `json:"url"` Data string `json:"data"` } `json:"audio"` } `json:"output"` } // dashscopeSTTRequest is the ASR request body. type dashscopeSTTRequest struct { Model string `json:"model"` Input dashscopeSTTInput `json:"input"` Parameters dashscopeSTTParameters `json:"parameters"` } // dashscopeSTTInput carries the multimodal messages list. type dashscopeSTTInput struct { Messages []dashscopeSTTMessage `json:"messages"` } // dashscopeSTTMessage is one ASR user turn. type dashscopeSTTMessage struct { Role string `json:"role"` Content []dashscopeSTTContent `json:"content"` } // dashscopeSTTContent is one audio content block. type dashscopeSTTContent struct { Type string `json:"type"` InputAudio dashscopeInputAudio `json:"input_audio"` } // dashscopeInputAudio carries the base64 data: URI audio. type dashscopeInputAudio struct { Data string `json:"data"` } // dashscopeSTTParameters tunes the ASR request. type dashscopeSTTParameters struct { Format string `json:"format"` SampleRate int `json:"sample_rate"` LanguageHints []string `json:"language_hints,omitempty"` } // dashscopeTTSRequest is the TTS request body. type dashscopeTTSRequest struct { Model string `json:"model"` Input dashscopeTTSInput `json:"input"` } // dashscopeTTSInput carries text + voice + output format. type dashscopeTTSInput struct { Text string `json:"text"` Voice string `json:"voice"` Format string `json:"format"` SampleRate int `json:"sample_rate"` // Instruction is the CosyVoice emotion/tone instruction; omitempty keeps a // system voice (which must use a fixed instruction) free of a free-form one. Instruction string `json:"instruction,omitempty"` } // dashscopeCloneRequest is the voice-enrollment create_voice request body. type dashscopeCloneRequest struct { Model string `json:"model"` Input dashscopeCloneInput `json:"input"` } // dashscopeCloneInput carries the create_voice action parameters. type dashscopeCloneInput struct { Action string `json:"action"` TargetModel string `json:"target_model"` Prefix string `json:"prefix"` URL string `json:"url"` } // dashscopeCloneResponse is the voice-enrollment response envelope. A top-level // message marks a model-level error surfaced even on an HTTP 200. type dashscopeCloneResponse struct { Output struct { VoiceID string `json:"voice_id"` } `json:"output"` Message string `json:"message"` }