AskaEth/Cyrene
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: 语音流式输入管线 + VAD前端集成 + 插件-工具合并清理

Browse Source 
- 前端: VAD语音检测(@ricky0123/vad-web) + useVoiceInput双模式(流式WS/REST)
- Gateway: VoiceStreamManager代理WS流式STT到voice-service
- Voice-service: DashScope REST → Realtime WS → Whisper三级引擎 + ffmpeg转码
- 共享模块: pkg/audio(音频转换) + pkg/dashscope(ASR REST客户端)
- 清理: 移除旧plugin-manager和pkg/plugins，完成插件→工具合并
- 文档: 完善gateway-api.md和voice-service.md语音API文档
- 工具: scripts/voice/ 语音转换脚本集

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
AskaEth
2026-06-06 11:50:40 +08:00
parent 258cf81b25
commit 6ef9e082a6
91 changed files with 4091 additions and 3929 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/ai-core/internal/background/thinker.go
+2 -2
View File
@@ -18,8 +18,8 @@ import (
"git.yeij.top/AskaEth/Cyrene/ai-core/internal/persona"
"git.yeij.top/AskaEth/Cyrene/ai-core/internal/tools"
plgManager "git.yeij.top/AskaEth/Cyrene/pkg/plugins/manager"
plgSDK "git.yeij.top/AskaEth/Cyrene/pkg/plugins/sdk"
plgManager "git.yeij.top/AskaEth/Cyrene-Plugins/manager"
plgSDK "git.yeij.top/AskaEth/Cyrene-Plugins/sdk"
)
// PendingThought 待推送的后台思考
backend/ai-core/internal/llm/asr.go
+20 -93
View File
@@ -1,31 +1,30 @@
package llm
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"git.yeij.top/AskaEth/Cyrene/pkg/audio"
"git.yeij.top/AskaEth/Cyrene/pkg/dashscope"
)
// ASRProvider handles speech-to-text transcription.
type ASRProvider interface {
Transcribe(ctx context.Context, audioURL string) (string, error)
Transcribe(ctx context.Context, audioURL, language string) (string, error)
IsAvailable() bool
ModelName() string
}
// DashScopeASRProvider uses DashScope Paraformer API for offline speech recognition.
type DashScopeASRProvider struct {
apiKey string
baseURL string
model string
client *http.Client
model string
client *dashscope.RESTClient
http *http.Client
}
// NewDashScopeASRProvider creates a DashScope ASR provider.
@@ -34,16 +33,15 @@ func NewDashScopeASRProvider(baseURL, apiKey, model string) *DashScopeASRProvide
model = "qwen3-asr-flash-2026-02-10"
}
return &DashScopeASRProvider{
apiKey: apiKey,
baseURL: baseURL,
model: model,
client: &http.Client{Timeout: 60 * time.Second},
model: model,
client: dashscope.NewRESTClient(apiKey),
http: &http.Client{Timeout: 60 * time.Second},
}
}
// IsAvailable returns true if the API key is configured.
func (p *DashScopeASRProvider) IsAvailable() bool {
return p.apiKey != ""
return p.client.IsAvailable()
}
// ModelName returns the ASR model name.
@@ -51,34 +49,6 @@ func (p *DashScopeASRProvider) ModelName() string {
return p.model
}
type asrRequest struct {
Model string `json:"model"`
Input asrInput `json:"input"`
Parameters asrParams `json:"parameters"`
}
type asrInput struct {
Audio string `json:"audio"`
}
type asrParams struct {
Format string `json:"format,omitempty"`
SampleRate int `json:"sample_rate,omitempty"`
Language string `json:"language,omitempty"`
}
type asrResponse struct {
Output struct {
Text string `json:"text"`
} `json:"output"`
Usage struct {
TotalTokens int `json:"total_tokens"`
} `json:"usage"`
RequestID string `json:"request_id"`
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
}
// downloadAudio fetches audio data from a URL and returns the bytes with inferred format.
func (p *DashScopeASRProvider) downloadAudio(ctx context.Context, audioURL string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, "GET", audioURL, nil)
@@ -86,7 +56,7 @@ func (p *DashScopeASRProvider) downloadAudio(ctx context.Context, audioURL strin
return nil, "", fmt.Errorf("create download request: %w", err)
}
resp, err := p.client.Do(req)
resp, err := p.http.Do(req)
if err != nil {
return nil, "", fmt.Errorf("download failed: %w", err)
}
@@ -103,7 +73,6 @@ func (p *DashScopeASRProvider) downloadAudio(ctx context.Context, audioURL strin
// inferAudioFormat determines the audio format from URL extension or Content-Type header.
func inferAudioFormat(urlStr, contentType string) string {
// Try URL extension first
u, err := url.Parse(urlStr)
if err == nil {
path := u.Path
@@ -115,7 +84,6 @@ func inferAudioFormat(urlStr, contentType string) string {
}
}
}
// Fallback: use Content-Type
if strings.Contains(contentType, "audio/amr") || strings.Contains(contentType, "amr") {
return "amr"
}
@@ -130,14 +98,8 @@ func inferAudioFormat(urlStr, contentType string) string {
}
return "amr" // default for QQ voice messages
}
// asrEndpoint derives the DashScope ASR REST endpoint from the provider base URL.
func asrEndpoint(baseURL string) string {
if u, err := url.Parse(baseURL); err == nil {
return fmt.Sprintf("%s://%s/api/v1/services/audio/asr/asr", u.Scheme, u.Host)
}
return strings.TrimRight(baseURL, "/") + "/api/v1/services/audio/asr/asr"
}
func (p *DashScopeASRProvider) Transcribe(ctx context.Context, audioURL string) (string, error) {
func (p *DashScopeASRProvider) Transcribe(ctx context.Context, audioURL, language string) (string, error) {
if !p.IsAvailable() {
return "", fmt.Errorf("DashScope ASR API key not configured")
}
@@ -147,50 +109,15 @@ func (p *DashScopeASRProvider) Transcribe(ctx context.Context, audioURL string)
return "", fmt.Errorf("download audio: %w", err)
}
audioB64 := base64.StdEncoding.EncodeToString(audioData)
reqBody := asrRequest{
Model: p.model,
Input: asrInput{
Audio: fmt.Sprintf("data:audio/%s;base64,%s", format, audioB64),
},
Parameters: asrParams{
Format: format,
Language: "zh",
},
}
bodyBytes, err := json.Marshal(reqBody)
// 转码为 16kHz mono PCM，提升识别兼容性
pcmData, err := audio.ConvertToPCM16(audioData, format)
if err != nil {
return "", fmt.Errorf("marshal ASR request: %w", err)
return "", fmt.Errorf("audio transcode: %w", err)
}
asrURL := asrEndpoint(p.baseURL)
req, err := http.NewRequestWithContext(ctx, "POST", asrURL, bytes.NewReader(bodyBytes))
if err != nil {
return "", fmt.Errorf("create ASR request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+p.apiKey)
resp, err := p.client.Do(req)
if err != nil {
return "", fmt.Errorf("ASR request failed: %w", err)
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("read ASR response: %w", err)
if language == "" || language == "auto" {
language = "zh"
}
var asrResp asrResponse
if err := json.Unmarshal(respBytes, &asrResp); err != nil {
return "", fmt.Errorf("parse ASR response: %w", err)
}
if asrResp.Code != "" && asrResp.Code != "0" {
return "", fmt.Errorf("ASR error: %s (code=%s)", asrResp.Message, asrResp.Code)
}
return asrResp.Output.Text, nil
return p.client.Transcribe(ctx, p.model, pcmData, "pcm", 16000, language)
}
backend/ai-core/internal/orchestrator/orchestrator.go
+2 -2
View File
@@ -19,7 +19,7 @@ import (
"git.yeij.top/AskaEth/Cyrene/ai-core/internal/bus"
"git.yeij.top/AskaEth/Cyrene/ai-core/internal/scheduler"
plgManager "git.yeij.top/AskaEth/Cyrene/pkg/plugins/manager"
plgManager "git.yeij.top/AskaEth/Cyrene-Plugins/manager"
)
// Orchestrator 对话编排器 v2.0
@@ -878,7 +878,7 @@ func (o *Orchestrator) preprocessVoice(ctx context.Context, message string, voic
var transcriptions []string
for i, url := range voiceURLs {
text, err := o.asrProvider.Transcribe(ctx, url)
text, err := o.asrProvider.Transcribe(ctx, url, "zh")
if err != nil {
logger.Printf("[orchestrator] 语音 %d 转录失败: %v", i, err)
continue
backend/ai-core/internal/orchestrator/synthesizer.go
+2 -2
View File
@@ -10,8 +10,8 @@ import (
"git.yeij.top/AskaEth/Cyrene/ai-core/internal/llm"
"git.yeij.top/AskaEth/Cyrene/ai-core/internal/model"
"git.yeij.top/AskaEth/Cyrene/pkg/logger"
plgManager "git.yeij.top/AskaEth/Cyrene/pkg/plugins/manager"
plgSDK "git.yeij.top/AskaEth/Cyrene/pkg/plugins/sdk"
plgManager "git.yeij.top/AskaEth/Cyrene-Plugins/manager"
plgSDK "git.yeij.top/AskaEth/Cyrene-Plugins/sdk"
)
// Synthesizer 主会话综合器
backend/ai-core/internal/tools/plugin_manager_client.go
-128
View File
@@ -1,128 +0,0 @@
package tools
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
)
// PluginManagerClient calls the plugin-manager service.
type PluginManagerClient struct {
baseURL string
httpClient *http.Client
}
// PMToolDefinition matches the plugin-manager tool definition format.
type PMToolDefinition struct {
ID string `json:"id"`
Name string `json:"name"`
DisplayName string `json:"displayName"`
Description string `json:"description"`
Category string `json:"category"`
Complexity string `json:"complexity"`
Parameters map[string]interface{} `json:"parameters"`
DangerLevel string `json:"danger_level,omitempty"`
}
// PMToolResult matches the plugin-manager execution result.
type PMToolResult struct {
ToolName string `json:"tool_name"`
Success bool `json:"success"`
Output string `json:"output,omitempty"`
Error string `json:"error,omitempty"`
}
// PMPluginInfo matches plugin-manager plugin info.
type PMPluginInfo struct {
Name string `json:"name"`
Version string `json:"version"`
Status string `json:"status"`
Enabled bool `json:"enabled"`
Tools []string `json:"tools"`
}
func NewPluginManagerClient(baseURL string) *PluginManagerClient {
return &PluginManagerClient{
baseURL: baseURL,
httpClient: &http.Client{Timeout: 10 * time.Second},
}
}
// GetToolDefinitions fetches all tool definitions from plugin-manager.
func (c *PluginManagerClient) GetToolDefinitions(ctx context.Context) ([]PMToolDefinition, error) {
req, _ := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/v1/tools", nil)
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("plugin-manager GetToolDefinitions: %w", err)
}
defer resp.Body.Close()
var body struct {
Tools []PMToolDefinition `json:"tools"`
}
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
return nil, fmt.Errorf("plugin-manager decode tools: %w", err)
}
return body.Tools, nil
}
// ExecuteTool calls a tool on plugin-manager by ID.
func (c *PluginManagerClient) ExecuteTool(ctx context.Context, toolID string, args map[string]interface{}) (*PMToolResult, error) {
body, _ := json.Marshal(map[string]interface{}{"arguments": args})
url := fmt.Sprintf("%s/api/v1/tools/%s/execute", c.baseURL, toolID)
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("plugin-manager ExecuteTool: %w", err)
}
defer resp.Body.Close()
var result PMToolResult
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, fmt.Errorf("plugin-manager decode result: %w", err)
}
return &result, nil
}
// ListPlugins fetches all installed plugins from plugin-manager.
func (c *PluginManagerClient) ListPlugins(ctx context.Context) ([]PMPluginInfo, error) {
req, _ := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/v1/plugins", nil)
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var body struct {
Plugins []PMPluginInfo `json:"plugins"`
}
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
return nil, err
}
return body.Plugins, nil
}
// AdaptDefinitions converts PM tool definitions to ai-core ToolDefinition format.
func (c *PluginManagerClient) AdaptDefinitions(ctx context.Context) ([]ToolDefinition, error) {
pmDefs, err := c.GetToolDefinitions(ctx)
if err != nil {
return nil, err
}
defs := make([]ToolDefinition, 0, len(pmDefs))
for _, d := range pmDefs {
defs = append(defs, ToolDefinition{
Name: d.Name,
Description: d.Description,
Parameters: d.Parameters,
})
}
return defs, nil
}