feat: 语音流式输入管线 + VAD前端集成 + 插件-工具合并清理

- 前端: VAD语音检测(@ricky0123/vad-web) + useVoiceInput双模式(流式WS/REST)
- Gateway: VoiceStreamManager代理WS流式STT到voice-service
- Voice-service: DashScope REST → Realtime WS → Whisper三级引擎 + ffmpeg转码
- 共享模块: pkg/audio(音频转换) + pkg/dashscope(ASR REST客户端)
- 清理: 移除旧plugin-manager和pkg/plugins,完成插件→工具合并
- 文档: 完善gateway-api.md和voice-service.md语音API文档
- 工具: scripts/voice/ 语音转换脚本集

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-06-06 11:50:40 +08:00
parent 258cf81b25
commit 6ef9e082a6
91 changed files with 4091 additions and 3929 deletions
+20 -93
View File
@@ -1,31 +1,30 @@
package llm
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"git.yeij.top/AskaEth/Cyrene/pkg/audio"
"git.yeij.top/AskaEth/Cyrene/pkg/dashscope"
)
// ASRProvider handles speech-to-text transcription.
type ASRProvider interface {
Transcribe(ctx context.Context, audioURL string) (string, error)
Transcribe(ctx context.Context, audioURL, language string) (string, error)
IsAvailable() bool
ModelName() string
}
// DashScopeASRProvider uses DashScope Paraformer API for offline speech recognition.
type DashScopeASRProvider struct {
apiKey string
baseURL string
model string
client *http.Client
model string
client *dashscope.RESTClient
http *http.Client
}
// NewDashScopeASRProvider creates a DashScope ASR provider.
@@ -34,16 +33,15 @@ func NewDashScopeASRProvider(baseURL, apiKey, model string) *DashScopeASRProvide
model = "qwen3-asr-flash-2026-02-10"
}
return &DashScopeASRProvider{
apiKey: apiKey,
baseURL: baseURL,
model: model,
client: &http.Client{Timeout: 60 * time.Second},
model: model,
client: dashscope.NewRESTClient(apiKey),
http: &http.Client{Timeout: 60 * time.Second},
}
}
// IsAvailable returns true if the API key is configured.
func (p *DashScopeASRProvider) IsAvailable() bool {
return p.apiKey != ""
return p.client.IsAvailable()
}
// ModelName returns the ASR model name.
@@ -51,34 +49,6 @@ func (p *DashScopeASRProvider) ModelName() string {
return p.model
}
type asrRequest struct {
Model string `json:"model"`
Input asrInput `json:"input"`
Parameters asrParams `json:"parameters"`
}
type asrInput struct {
Audio string `json:"audio"`
}
type asrParams struct {
Format string `json:"format,omitempty"`
SampleRate int `json:"sample_rate,omitempty"`
Language string `json:"language,omitempty"`
}
type asrResponse struct {
Output struct {
Text string `json:"text"`
} `json:"output"`
Usage struct {
TotalTokens int `json:"total_tokens"`
} `json:"usage"`
RequestID string `json:"request_id"`
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
}
// downloadAudio fetches audio data from a URL and returns the bytes with inferred format.
func (p *DashScopeASRProvider) downloadAudio(ctx context.Context, audioURL string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, "GET", audioURL, nil)
@@ -86,7 +56,7 @@ func (p *DashScopeASRProvider) downloadAudio(ctx context.Context, audioURL strin
return nil, "", fmt.Errorf("create download request: %w", err)
}
resp, err := p.client.Do(req)
resp, err := p.http.Do(req)
if err != nil {
return nil, "", fmt.Errorf("download failed: %w", err)
}
@@ -103,7 +73,6 @@ func (p *DashScopeASRProvider) downloadAudio(ctx context.Context, audioURL strin
// inferAudioFormat determines the audio format from URL extension or Content-Type header.
func inferAudioFormat(urlStr, contentType string) string {
// Try URL extension first
u, err := url.Parse(urlStr)
if err == nil {
path := u.Path
@@ -115,7 +84,6 @@ func inferAudioFormat(urlStr, contentType string) string {
}
}
}
// Fallback: use Content-Type
if strings.Contains(contentType, "audio/amr") || strings.Contains(contentType, "amr") {
return "amr"
}
@@ -130,14 +98,8 @@ func inferAudioFormat(urlStr, contentType string) string {
}
return "amr" // default for QQ voice messages
}
// asrEndpoint derives the DashScope ASR REST endpoint from the provider base URL.
func asrEndpoint(baseURL string) string {
if u, err := url.Parse(baseURL); err == nil {
return fmt.Sprintf("%s://%s/api/v1/services/audio/asr/asr", u.Scheme, u.Host)
}
return strings.TrimRight(baseURL, "/") + "/api/v1/services/audio/asr/asr"
}
func (p *DashScopeASRProvider) Transcribe(ctx context.Context, audioURL string) (string, error) {
func (p *DashScopeASRProvider) Transcribe(ctx context.Context, audioURL, language string) (string, error) {
if !p.IsAvailable() {
return "", fmt.Errorf("DashScope ASR API key not configured")
}
@@ -147,50 +109,15 @@ func (p *DashScopeASRProvider) Transcribe(ctx context.Context, audioURL string)
return "", fmt.Errorf("download audio: %w", err)
}
audioB64 := base64.StdEncoding.EncodeToString(audioData)
reqBody := asrRequest{
Model: p.model,
Input: asrInput{
Audio: fmt.Sprintf("data:audio/%s;base64,%s", format, audioB64),
},
Parameters: asrParams{
Format: format,
Language: "zh",
},
}
bodyBytes, err := json.Marshal(reqBody)
// 转码为 16kHz mono PCM,提升识别兼容性
pcmData, err := audio.ConvertToPCM16(audioData, format)
if err != nil {
return "", fmt.Errorf("marshal ASR request: %w", err)
return "", fmt.Errorf("audio transcode: %w", err)
}
asrURL := asrEndpoint(p.baseURL)
req, err := http.NewRequestWithContext(ctx, "POST", asrURL, bytes.NewReader(bodyBytes))
if err != nil {
return "", fmt.Errorf("create ASR request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+p.apiKey)
resp, err := p.client.Do(req)
if err != nil {
return "", fmt.Errorf("ASR request failed: %w", err)
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("read ASR response: %w", err)
if language == "" || language == "auto" {
language = "zh"
}
var asrResp asrResponse
if err := json.Unmarshal(respBytes, &asrResp); err != nil {
return "", fmt.Errorf("parse ASR response: %w", err)
}
if asrResp.Code != "" && asrResp.Code != "0" {
return "", fmt.Errorf("ASR error: %s (code=%s)", asrResp.Message, asrResp.Code)
}
return asrResp.Output.Text, nil
return p.client.Transcribe(ctx, p.model, pcmData, "pcm", 16000, language)
}