feat: 语音流式输入管线 + VAD前端集成 + 插件-工具合并清理
- 前端: VAD语音检测(@ricky0123/vad-web) + useVoiceInput双模式(流式WS/REST) - Gateway: VoiceStreamManager代理WS流式STT到voice-service - Voice-service: DashScope REST → Realtime WS → Whisper三级引擎 + ffmpeg转码 - 共享模块: pkg/audio(音频转换) + pkg/dashscope(ASR REST客户端) - 清理: 移除旧plugin-manager和pkg/plugins,完成插件→工具合并 - 文档: 完善gateway-api.md和voice-service.md语音API文档 - 工具: scripts/voice/ 语音转换脚本集 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1,31 +1,30 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene/pkg/audio"
|
||||
"git.yeij.top/AskaEth/Cyrene/pkg/dashscope"
|
||||
)
|
||||
|
||||
// ASRProvider handles speech-to-text transcription.
|
||||
type ASRProvider interface {
|
||||
Transcribe(ctx context.Context, audioURL string) (string, error)
|
||||
Transcribe(ctx context.Context, audioURL, language string) (string, error)
|
||||
IsAvailable() bool
|
||||
ModelName() string
|
||||
}
|
||||
|
||||
// DashScopeASRProvider uses DashScope Paraformer API for offline speech recognition.
|
||||
type DashScopeASRProvider struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
model string
|
||||
client *http.Client
|
||||
model string
|
||||
client *dashscope.RESTClient
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// NewDashScopeASRProvider creates a DashScope ASR provider.
|
||||
@@ -34,16 +33,15 @@ func NewDashScopeASRProvider(baseURL, apiKey, model string) *DashScopeASRProvide
|
||||
model = "qwen3-asr-flash-2026-02-10"
|
||||
}
|
||||
return &DashScopeASRProvider{
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
model: model,
|
||||
client: &http.Client{Timeout: 60 * time.Second},
|
||||
model: model,
|
||||
client: dashscope.NewRESTClient(apiKey),
|
||||
http: &http.Client{Timeout: 60 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// IsAvailable returns true if the API key is configured.
|
||||
func (p *DashScopeASRProvider) IsAvailable() bool {
|
||||
return p.apiKey != ""
|
||||
return p.client.IsAvailable()
|
||||
}
|
||||
|
||||
// ModelName returns the ASR model name.
|
||||
@@ -51,34 +49,6 @@ func (p *DashScopeASRProvider) ModelName() string {
|
||||
return p.model
|
||||
}
|
||||
|
||||
type asrRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input asrInput `json:"input"`
|
||||
Parameters asrParams `json:"parameters"`
|
||||
}
|
||||
|
||||
type asrInput struct {
|
||||
Audio string `json:"audio"`
|
||||
}
|
||||
|
||||
type asrParams struct {
|
||||
Format string `json:"format,omitempty"`
|
||||
SampleRate int `json:"sample_rate,omitempty"`
|
||||
Language string `json:"language,omitempty"`
|
||||
}
|
||||
|
||||
type asrResponse struct {
|
||||
Output struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"output"`
|
||||
Usage struct {
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
} `json:"usage"`
|
||||
RequestID string `json:"request_id"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
// downloadAudio fetches audio data from a URL and returns the bytes with inferred format.
|
||||
func (p *DashScopeASRProvider) downloadAudio(ctx context.Context, audioURL string) ([]byte, string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", audioURL, nil)
|
||||
@@ -86,7 +56,7 @@ func (p *DashScopeASRProvider) downloadAudio(ctx context.Context, audioURL strin
|
||||
return nil, "", fmt.Errorf("create download request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
resp, err := p.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("download failed: %w", err)
|
||||
}
|
||||
@@ -103,7 +73,6 @@ func (p *DashScopeASRProvider) downloadAudio(ctx context.Context, audioURL strin
|
||||
|
||||
// inferAudioFormat determines the audio format from URL extension or Content-Type header.
|
||||
func inferAudioFormat(urlStr, contentType string) string {
|
||||
// Try URL extension first
|
||||
u, err := url.Parse(urlStr)
|
||||
if err == nil {
|
||||
path := u.Path
|
||||
@@ -115,7 +84,6 @@ func inferAudioFormat(urlStr, contentType string) string {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fallback: use Content-Type
|
||||
if strings.Contains(contentType, "audio/amr") || strings.Contains(contentType, "amr") {
|
||||
return "amr"
|
||||
}
|
||||
@@ -130,14 +98,8 @@ func inferAudioFormat(urlStr, contentType string) string {
|
||||
}
|
||||
return "amr" // default for QQ voice messages
|
||||
}
|
||||
// asrEndpoint derives the DashScope ASR REST endpoint from the provider base URL.
|
||||
func asrEndpoint(baseURL string) string {
|
||||
if u, err := url.Parse(baseURL); err == nil {
|
||||
return fmt.Sprintf("%s://%s/api/v1/services/audio/asr/asr", u.Scheme, u.Host)
|
||||
}
|
||||
return strings.TrimRight(baseURL, "/") + "/api/v1/services/audio/asr/asr"
|
||||
}
|
||||
func (p *DashScopeASRProvider) Transcribe(ctx context.Context, audioURL string) (string, error) {
|
||||
|
||||
func (p *DashScopeASRProvider) Transcribe(ctx context.Context, audioURL, language string) (string, error) {
|
||||
if !p.IsAvailable() {
|
||||
return "", fmt.Errorf("DashScope ASR API key not configured")
|
||||
}
|
||||
@@ -147,50 +109,15 @@ func (p *DashScopeASRProvider) Transcribe(ctx context.Context, audioURL string)
|
||||
return "", fmt.Errorf("download audio: %w", err)
|
||||
}
|
||||
|
||||
audioB64 := base64.StdEncoding.EncodeToString(audioData)
|
||||
|
||||
reqBody := asrRequest{
|
||||
Model: p.model,
|
||||
Input: asrInput{
|
||||
Audio: fmt.Sprintf("data:audio/%s;base64,%s", format, audioB64),
|
||||
},
|
||||
Parameters: asrParams{
|
||||
Format: format,
|
||||
Language: "zh",
|
||||
},
|
||||
}
|
||||
bodyBytes, err := json.Marshal(reqBody)
|
||||
// 转码为 16kHz mono PCM,提升识别兼容性
|
||||
pcmData, err := audio.ConvertToPCM16(audioData, format)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal ASR request: %w", err)
|
||||
return "", fmt.Errorf("audio transcode: %w", err)
|
||||
}
|
||||
|
||||
asrURL := asrEndpoint(p.baseURL)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", asrURL, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create ASR request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ASR request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read ASR response: %w", err)
|
||||
if language == "" || language == "auto" {
|
||||
language = "zh"
|
||||
}
|
||||
|
||||
var asrResp asrResponse
|
||||
if err := json.Unmarshal(respBytes, &asrResp); err != nil {
|
||||
return "", fmt.Errorf("parse ASR response: %w", err)
|
||||
}
|
||||
|
||||
if asrResp.Code != "" && asrResp.Code != "0" {
|
||||
return "", fmt.Errorf("ASR error: %s (code=%s)", asrResp.Message, asrResp.Code)
|
||||
}
|
||||
|
||||
return asrResp.Output.Text, nil
|
||||
return p.client.Transcribe(ctx, p.model, pcmData, "pcm", 16000, language)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user