feat(multimodal): 视觉分层路由(多模态检测→主/视觉/OCR 工具)+ 语音 STT→TTS 降级链(异步)

This commit is contained in:
Aska committed 2026-09-06 05:12:36 +08:00
1 parent 02aae2bbb4
commit 0a936a3202
19 files changed
+2121

No files matched your search

+17
View File
@@ -0,0 +1,17 @@
// Package multimodal groups the runtime's non-text input/output handling:
// image understanding (vision) and speech (voice).
//
// It follows DevDocs/design/06-多模态.md:
//
// - The vision subpackage routes image messages through a layered strategy
// (main model first, then a tool-shaped vision/OCR adapter) and registers
// those adapters as sdk.Tool capabilities so the M2 tool loop can invoke
// them without putting the image in the main-model context.
// - The voice subpackage provides asynchronous STT/TTS through a degrading
// provider chain (cloud → local → edge-tts/espeak → silent).
//
// Both subpackages keep real provider calls behind interfaces with an HTTP
// default implementation, so they are testable with a stdlib httptest server
// and leave the real cloud/local providers as adaptation points for later
// milestones.
package multimodal
@@ -0,0 +1,217 @@
package vision
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"strings"
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
)
// visionMaxTokens caps the vision/OCR response length.
const visionMaxTokens = 512
// VisionAdapter understands an image and returns a description of it. It is the
// backend behind the vision tool; real providers implement it without exposing
// the image to the main-model context.
type VisionAdapter interface {
// Describe returns a description of img. prompt is an optional instruction
// overriding the default "describe this image" ask.
Describe(ctx context.Context, img ImageRef, prompt string) (string, error)
}
// OCRAdapter extracts text from an image. It is the backend behind the OCR tool.
type OCRAdapter interface {
// Extract returns the text found in img.
Extract(ctx context.Context, img ImageRef) (string, error)
}
// OpenAICompatibleAdapter implements both VisionAdapter and OCRAdapter by
// calling an OpenAI-compatible /chat/completions endpoint with the image
// embedded as a content part. It is the ready default implementation; real
// cloud vision/OCR providers expose the same protocol.
type OpenAICompatibleAdapter struct {
cfg config.LLMConfig
model string
client *http.Client
log *slog.Logger
}
// OpenAIOption customizes an OpenAICompatibleAdapter.
type OpenAIOption func(*OpenAICompatibleAdapter)
// WithModel overrides the adapter model (defaults to cfg.Model).
func WithModel(m string) OpenAIOption {
return func(a *OpenAICompatibleAdapter) {
if m != "" {
a.model = m
}
}
}
// WithHTTPClient sets the *http.Client used for the request; nil keeps default.
func WithHTTPClient(c *http.Client) OpenAIOption {
return func(a *OpenAICompatibleAdapter) {
if c != nil {
a.client = c
}
}
}
// WithLogger sets the logger used on failures; nil keeps a discard logger.
func WithLogger(l *slog.Logger) OpenAIOption {
return func(a *OpenAICompatibleAdapter) {
if l != nil {
a.log = l
}
}
}
// NewOpenAICompatible builds an adapter from cfg, using cfg.Model unless
// overridden.
func NewOpenAICompatible(cfg config.LLMConfig, opts ...OpenAIOption) *OpenAICompatibleAdapter {
a := &OpenAICompatibleAdapter{
cfg: cfg,
model: cfg.Model,
client: defaultHTTPClient(),
log: slog.New(slog.NewTextHandler(io.Discard, nil)),
}
for _, o := range opts {
o(a)
}
return a
}
// Describe returns an image description from the underlying chat endpoint.
func (a *OpenAICompatibleAdapter) Describe(ctx context.Context, img ImageRef, prompt string) (string, error) {
if prompt == "" {
prompt = "Describe this image in detail."
}
return a.ask(ctx, img, prompt)
}
// Extract returns text found in the image via the same endpoint.
func (a *OpenAICompatibleAdapter) Extract(ctx context.Context, img ImageRef) (string, error) {
return a.ask(ctx, img, "Extract all text visible in this image. Return only the text.")
}
// ask resolves the image, builds a chat request and returns the assistant text.
func (a *OpenAICompatibleAdapter) ask(ctx context.Context, img ImageRef, prompt string) (string, error) {
if strings.TrimSpace(a.cfg.BaseURL) == "" {
return "", ErrNotConfigured
}
if strings.TrimSpace(a.model) == "" {
return "", ErrNotConfigured
}
if err := ctx.Err(); err != nil {
return "", err
}
imageURL, err := a.imageURL(ctx, img)
if err != nil {
return "", err
}
payload, err := json.Marshal(a.request(imageURL, prompt))
if err != nil {
return "", fmt.Errorf("vision: %w: marshal request: %v", ErrUpstream, err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(a.cfg.BaseURL, "/")+"/chat/completions", bytes.NewReader(payload))
if err != nil {
return "", fmt.Errorf("vision: %w: build request: %v", ErrUpstream, err)
}
req.Header.Set("Content-Type", "application/json")
if a.cfg.APIKey != "" {
req.Header.Set("Authorization", "Bearer "+a.cfg.APIKey)
}
resp, err := a.client.Do(req)
if err != nil {
return "", fmt.Errorf("vision: %w: %v", ErrUpstream, err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
snippet, _ := io.ReadAll(io.LimitReader(resp.Body, maxFetchedBody))
return "", fmt.Errorf("vision: %w: status %d: %s", ErrUpstream, resp.StatusCode, strings.TrimSpace(string(snippet)))
}
return a.decode(resp.Body)
}
// imageURL converts an ImageRef to the image_url value the wire expects: an
// http(s) URL passes through, anything else becomes a base64 data: URI.
func (a *OpenAICompatibleAdapter) imageURL(ctx context.Context, img ImageRef) (string, error) {
if img.Source == SourceURL {
return img.Data, nil
}
mime, data, err := Resolve(ctx, img, a.client)
if err != nil {
return "", err
}
return DataURI(mime, data), nil
}
// openAIVisionRequest is the JSON body sent to /chat/completions.
type openAIVisionRequest struct {
Model string `json:"model"`
Messages []openAIMessage `json:"messages"`
MaxTokens int `json:"max_tokens"`
}
type openAIMessage struct {
Role string `json:"role"`
Content any `json:"content"`
}
type openAIPart struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
ImageURL *openAIImageURL `json:"image_url,omitempty"`
}
type openAIImageURL struct {
URL string `json:"url"`
}
type openAIVisionResponse struct {
Choices []openAIChoice `json:"choices"`
}
type openAIChoice struct {
Message openAIResponseMessage `json:"message"`
}
type openAIResponseMessage struct {
Content string `json:"content"`
}
// request builds the wire body for an image+prompt.
func (a *OpenAICompatibleAdapter) request(imageURL, prompt string) openAIVisionRequest {
return openAIVisionRequest{
Model: a.model,
Messages: []openAIMessage{
{Role: "user", Content: []openAIPart{
{Type: "text", Text: prompt},
{Type: "image_url", ImageURL: &openAIImageURL{URL: imageURL}},
}},
},
MaxTokens: visionMaxTokens,
}
}
// decode extracts the first choice's text from a response body.
func (a *OpenAICompatibleAdapter) decode(r io.Reader) (string, error) {
var wire openAIVisionResponse
if err := json.NewDecoder(r).Decode(&wire); err != nil {
return "", fmt.Errorf("vision: %w: decode response: %v", ErrUpstream, err)
}
if len(wire.Choices) == 0 {
return "", fmt.Errorf("vision: %w: empty choices", ErrUpstream)
}
return wire.Choices[0].Message.Content, nil
}
// interface assertion: one concrete adapter serves both directions.
var _ VisionAdapter = (*OpenAICompatibleAdapter)(nil)
var _ OCRAdapter = (*OpenAICompatibleAdapter)(nil)
@@ -0,0 +1,53 @@
package vision
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"testing"
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
)
func TestOpenAICompatibleAdapter_Describe(t *testing.T) {
t.Parallel()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/chat/completions" {
t.Errorf("path = %s, want /chat/completions", r.URL.Path)
}
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode body: %v", err)
}
msgs, _ := body["messages"].([]any)
if len(msgs) != 1 {
t.Fatalf("messages = %d, want 1", len(msgs))
}
content, _ := msgs[0].(map[string]any)["content"].([]any)
if len(content) != 2 {
t.Fatalf("content parts = %d, want 2", len(content))
}
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"a red car"}}]}`)
}))
defer srv.Close()
ad := NewOpenAICompatible(config.LLMConfig{BaseURL: srv.URL, Model: "vision-"})
text, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: "aGVsbG8="}, "")
if err != nil {
t.Fatalf("Describe error: %v", err)
}
if text != "a red car" {
t.Errorf("text = %q, want a red car", text)
}
}
func TestOpenAICompatibleAdapter_NotConfigured(t *testing.T) {
t.Parallel()
ad := NewOpenAICompatible(config.LLMConfig{})
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: "aGVsbG8="}, ""); err == nil {
t.Fatal("want error for empty base url, got nil")
}
}
@@ -0,0 +1,22 @@
package vision
import "errors"
// Sentinel errors returned by the vision package. Each is wrapped with %w on
// the way out so callers can test for a category while keeping the cause.
var (
// ErrNoImage is returned when an image source string is empty.
ErrNoImage = errors.New("vision: empty image source")
// ErrNotConfigured is returned by adapters when no endpoint/model is set.
ErrNotConfigured = errors.New("vision: adapter not configured")
// ErrUnsupportedSource is returned when an image source cannot be handled.
ErrUnsupportedSource = errors.New("vision: unsupported image source")
// ErrFetch is returned when a URL image cannot be downloaded.
ErrFetch = errors.New("vision: cannot fetch image")
// ErrUpstream is returned when the vision/OCR provider request fails.
ErrUpstream = errors.New("vision: upstream request failed")
// ErrBadImage is returned when an image cannot be read or decoded.
ErrBadImage = errors.New("vision: cannot read image")
// ErrBadArgs is returned by a tool when its arguments are malformed.
ErrBadArgs = errors.New("vision: bad tool arguments")
)
+178
View File
@@ -0,0 +1,178 @@
package vision
import (
"context"
"encoding/base64"
"fmt"
"io"
"net/http"
"os"
"strings"
)
// Size caps for reading image bytes and response bodies, so a hostile or
// malformed source cannot exhaust memory. The 50MB cap matches the file-upload
// limit in DevDocs/02-development-constraints.md §11.
const (
maxImageBytes = 50 << 20
maxFetchedBody = 50 << 20
)
// DataURI builds a base64 data: image URI from mime and raw bytes.
func DataURI(mime string, data []byte) string {
return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data)
}
// Resolve loads the bytes for an ImageRef. Paths and base64 payloads are read
// locally; URLs are fetched with an http.Client (the provided client, or a
// default one when nil). For base64 input the Data field is returned verbatim as
// the payload and the MIME type is guessed from a data: prefix.
func Resolve(ctx context.Context, ref ImageRef, client *http.Client) (mime string, data []byte, err error) {
switch ref.Source {
case SourcePath:
return readFile(ref.Data)
case SourceBase64:
return decodeBase64(ref.Data)
case SourceURL:
return fetchURL(ctx, ref.Data, client)
default:
return "", nil, fmt.Errorf("vision: %w: %q", ErrUnsupportedSource, ref.Source)
}
}
// readFile reads an image file and guesses its MIME type from the extension.
func readFile(path string) (string, []byte, error) {
f, err := os.Open(path)
if err != nil {
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
}
defer func() { _ = f.Close() }()
data, err := io.ReadAll(io.LimitReader(f, maxImageBytes))
if err != nil {
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
}
return mimeByExt(path), data, nil
}
// decodeBase64 returns the decoded bytes of a base64/data: payload and its MIME.
func decodeBase64(src string) (string, []byte, error) {
mime, encoded, isDataURI := splitDataURI(src)
if isDataURI {
decoded, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
}
return mime, decoded, nil
}
decoded, err := base64.StdEncoding.DecodeString(src)
if err != nil {
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
}
return "image/png", decoded, nil
}
// fetchURL downloads an image over http(s); only http/https schemes are allowed.
func fetchURL(ctx context.Context, rawURL string, client *http.Client) (string, []byte, error) {
if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
return "", nil, fmt.Errorf("vision: %w: %q", ErrUnsupportedSource, rawURL)
}
if client == nil {
client = defaultHTTPClient()
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err)
}
resp, err := client.Do(req)
if err != nil {
return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return "", nil, fmt.Errorf("vision: %w: status %d", ErrFetch, resp.StatusCode)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, maxFetchedBody))
if err != nil {
return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err)
}
return mimeByContentType(resp.Header.Get("Content-Type")), data, nil
}
// splitDataURI parses a "data:<mime>;base64,<payload>" prefix. The second return
// is false when src is not a data: URI.
func splitDataURI(src string) (mime, payload string, ok bool) {
if !strings.HasPrefix(src, "data:") {
return "", "", false
}
rest := strings.TrimPrefix(src, "data:")
comma := strings.IndexByte(rest, ',')
if comma < 0 {
return "", "", false
}
meta := rest[:comma]
payload = rest[comma+1:]
mime = "image/png"
if strings.HasPrefix(meta, ";base64") {
return mime, payload, true
}
if idx := strings.Index(meta, ";base64"); idx >= 0 {
if t := strings.TrimSuffix(meta[:idx], ";base64"); t != "" {
mime = t
}
return mime, payload, true
}
return mime, payload, true
}
// mimeByExt guesses a MIME type from a file path extension.
func mimeByExt(path string) string {
switch strings.ToLower(pathExtension(path)) {
case ".png":
return "image/png"
case ".jpg", ".jpeg":
return "image/jpeg"
case ".gif":
return "image/gif"
case ".webp":
return "image/webp"
default:
return "image/png"
}
}
// mimeByContentType returns a MIME type from an HTTP Content-Type header,
// defaulting to image/png when absent or non-image.
func mimeByContentType(ct string) string {
if ct == "" || !strings.HasPrefix(ct, "image/") {
return "image/png"
}
return strings.TrimSpace(strings.Split(ct, ";")[0])
}
// pathExtension returns the file extension with its dot, or empty when none.
func pathExtension(path string) string {
base := path
if idx := strings.LastIndexByte(path, '/'); idx >= 0 {
base = path[idx+1:]
} else if idx := strings.LastIndexByte(path, '\\'); idx >= 0 {
base = path[idx+1:]
}
dot := strings.LastIndexByte(base, '.')
if dot < 0 {
return ""
}
return base[dot:]
}
// fileExists reports whether path exists on disk and is a regular file.
func fileExists(path string) bool {
info, err := os.Stat(path)
return err == nil && info.Mode().IsRegular()
}
// defaultHTTPClient builds the HTTP client described in
// DevDocs/02-development-constraints.md §2.3. A fresh client is returned per
// call rather than a package global, which the constraints forbid.
func defaultHTTPClient() *http.Client {
return &http.Client{Transport: http.DefaultTransport}
}
@@ -0,0 +1,128 @@
package vision
// Assessment is the model's own confidence in reading an image.
type Assessment string
const (
// AssessmentUnknown means the model reported no self-assessment.
AssessmentUnknown Assessment = "unknown"
// AssessmentGood means the model is confident in its read.
AssessmentGood Assessment = "good"
// AssessmentPoor means the model judged its recognition as poor.
AssessmentPoor Assessment = "poor"
)
// ModelInfo describes the main model's vision capability.
type ModelInfo struct {
// Name identifies the model.
Name string
// SupportsVision reports whether the model accepts image input.
SupportsVision bool
// SelfAssessment is the model's own confidence for the current image.
// It drives the layer-2 tool route when SupportsVision is true.
SelfAssessment Assessment
}
// DecisionKind is the chosen layered route.
type DecisionKind string
const (
// KindMain routes the image to the main model with the image in context.
KindMain DecisionKind = "main"
// KindVisionTool routes to a dedicated vision tool, returning only the tool
// result so the image never occupies main-model context.
KindVisionTool DecisionKind = "vision_tool"
// KindOCRTool routes to an OCR tool, returning only the extracted text.
KindOCRTool DecisionKind = "ocr_tool"
// KindUnsupported means no route can process the image.
KindUnsupported DecisionKind = "unsupported"
)
// Decision is the outcome of a routing call.
type Decision struct {
// Kind is the chosen route.
Kind DecisionKind
// Tool names the vision or OCR tool to invoke, set when Kind is a tool route.
Tool string
// Reason is a human-readable explanation for observability and logs.
Reason string
}
// RouteRequest carries routing hints from the caller on top of the model info.
type RouteRequest struct {
// TextPrimary reports that the user indicated the image is mainly text, so
// an OCR tool is preferred over a general vision model.
TextPrimary bool
// MainSelfAssessedPoor is set when the main model has already judged its own
// read of the image as poor, requesting a vision/OCR tool route.
MainSelfAssessedPoor bool
}
// RouterOption configures a Router.
type RouterOption func(*Router)
// WithVisionTool binds the name of the registered vision-understanding tool.
func WithVisionTool(name string) RouterOption {
return func(r *Router) {
if name != "" {
r.visionTool = name
}
}
}
// WithOCRTool binds the name of the registered OCR tool.
func WithOCRTool(name string) RouterOption {
return func(r *Router) {
if name != "" {
r.ocrTool = name
}
}
}
// Router implements the layered vision routing policy. It does no I/O; the
// caller uses the returned Decision to dispatch the request.
type Router struct {
visionTool string
ocrTool string
}
// NewRouter builds a Router. Unknown (empty) tool names leave the corresponding
// route unavailable, so the router degrades to KindMain or KindUnsupported.
func NewRouter(opts ...RouterOption) *Router {
r := &Router{}
for _, o := range opts {
o(r)
}
return r
}
// Route picks a layered route for an image request given the target model info
// and caller hints. The order is: the main model directly when it can see and
// has not self-assessed poorly (layer 1); otherwise a vision or OCR tool whose
// result replaces the image in context (layers 2 and 3); otherwise
// KindUnsupported when nothing can process the image.
func (r *Router) Route(target ModelInfo, req RouteRequest) Decision {
poor := target.SelfAssessment == AssessmentPoor || req.MainSelfAssessedPoor
// Layer 1: the main model accepts images and is confident, so use it first.
if target.SupportsVision && !poor {
return Decision{
Kind: KindMain,
Reason: "main model supports vision and is confident; use it directly",
}
}
// OCR is preferred when the user says the image is mostly text.
if req.TextPrimary && r.ocrTool != "" {
return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "image is mainly text; route to OCR"}
}
if r.visionTool != "" {
return Decision{Kind: KindVisionTool, Tool: r.visionTool, Reason: "route to dedicated vision tool"}
}
if r.ocrTool != "" {
return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "route to OCR tool"}
}
// No tool is bound. A sighted main model remains the only option.
if target.SupportsVision {
return Decision{Kind: KindMain, Reason: "no vision/OCR tool bound; keep image with main model"}
}
return Decision{Kind: KindUnsupported, Reason: "main model cannot see and no vision/OCR tool is bound"}
}
@@ -0,0 +1,178 @@
package vision
import (
"context"
"testing"
"git.yeij.top/AskaEth/Cyrene_v2/internal/capability/registry"
)
func TestRouter_Route(t *testing.T) {
t.Parallel()
cases := []struct {
name string
router *Router
target ModelInfo
req RouteRequest
want DecisionKind
tool string
}{
{
name: "sighted confident main -> main",
router: NewRouter(WithVisionTool("v"), WithOCRTool("o")),
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentGood},
want: KindMain,
},
{
name: "poor self-assessment sighted main -> vision tool",
router: NewRouter(WithVisionTool("v"), WithOCRTool("o")),
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor},
req: RouteRequest{MainSelfAssessedPoor: true},
want: KindVisionTool,
tool: "v",
},
{
name: "poor self-assessment only OCR -> OCR tool",
router: NewRouter(WithOCRTool("o")),
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor},
want: KindOCRTool,
tool: "o",
},
{
name: "blind main with vision tool -> vision tool",
router: NewRouter(WithVisionTool("v")),
target: ModelInfo{Name: "m", SupportsVision: false},
want: KindVisionTool,
tool: "v",
},
{
name: "blind main text primary -> OCR tool",
router: NewRouter(WithVisionTool("v"), WithOCRTool("o")),
target: ModelInfo{Name: "m", SupportsVision: false},
req: RouteRequest{TextPrimary: true},
want: KindOCRTool,
tool: "o",
},
{
name: "blind main only OCR non-text -> OCR fallback",
router: NewRouter(WithOCRTool("o")),
target: ModelInfo{Name: "m", SupportsVision: false},
want: KindOCRTool,
tool: "o",
},
{
name: "blind main no tools -> unsupported",
router: NewRouter(),
target: ModelInfo{Name: "m", SupportsVision: false},
want: KindUnsupported,
},
{
name: "sighted main no tools -> main",
router: NewRouter(),
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentGood},
want: KindMain,
},
{
name: "sighted main poor no tools -> main fallback",
router: NewRouter(),
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor},
want: KindMain,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got := tc.router.Route(tc.target, tc.req)
if got.Kind != tc.want {
t.Fatalf("Kind = %q, want %q (reason: %s)", got.Kind, tc.want, got.Reason)
}
if tc.tool != "" && got.Tool != tc.tool {
t.Errorf("Tool = %q, want %q", got.Tool, tc.tool)
}
if got.Reason == "" {
t.Error("Reason must be non-empty")
}
})
}
}
// fakeVision implements VisionAdapter for tool tests.
type fakeVision struct{ desc string }
func (f *fakeVision) Describe(_ context.Context, _ ImageRef, _ string) (string, error) {
return f.desc, nil
}
// fakeOCR implements OCRAdapter for tool tests.
type fakeOCR struct{ text string }
func (f *fakeOCR) Extract(_ context.Context, _ ImageRef) (string, error) {
return f.text, nil
}
func TestRegisterTools(t *testing.T) {
t.Parallel()
reg := registry.New()
v := NewVisionTool("vision_understand", &fakeVision{desc: "a cat"})
o := NewOCRTool("ocr_extract", &fakeOCR{text: "HELLO"})
if err := RegisterTools(reg, v, o); err != nil {
t.Fatalf("RegisterTools error: %v", err)
}
if capv, err := reg.Get("vision_understand"); err != nil || capv.Kind != registry.KindTool {
t.Fatalf("Get(vision_understand) = %+v, %v", capv, err)
}
if capo, err := reg.Get("ocr_extract"); err != nil || capo.Kind != registry.KindTool {
t.Fatalf("Get(ocr_extract) = %+v, %v", capo, err)
}
if len(reg.List(registry.KindTool)) != 2 {
t.Errorf("List(tool) = %d, want 2", len(reg.List(registry.KindTool)))
}
}
func TestRegisterTools_NilRegistrar(t *testing.T) {
t.Parallel()
if err := RegisterTools(nil, nil, nil); err == nil {
t.Fatal("want error for nil registrar, got nil")
}
}
func TestVisionTool_Execute(t *testing.T) {
t.Parallel()
v := NewVisionTool("vision_understand", &fakeVision{desc: "a cat"})
res, err := v.Execute(context.Background(), map[string]any{
"image": "data:image/png;base64,aGVsbG8=",
"prompt": "what is this",
})
if err != nil {
t.Fatalf("Execute error: %v", err)
}
m, ok := res.(map[string]any)
if !ok || m["description"] != "a cat" {
t.Errorf("result = %v, want description=a cat", res)
}
}
func TestVisionTool_Execute_BadArgs(t *testing.T) {
t.Parallel()
v := NewVisionTool("vision_understand", &fakeVision{desc: ""})
if _, err := v.Execute(context.Background(), map[string]any{"prompt": "x"}); err == nil {
t.Fatal("want error for missing image, got nil")
}
if _, err := v.Execute(context.Background(), "not-a-map"); err == nil {
t.Fatal("want error for non-map args, got nil")
}
}
func TestOCRTool_Execute(t *testing.T) {
t.Parallel()
o := NewOCRTool("ocr_extract", &fakeOCR{text: "HELLO"})
res, err := o.Execute(context.Background(), map[string]any{"image": "aGVsbG8="})
if err != nil {
t.Fatalf("Execute error: %v", err)
}
m, ok := res.(map[string]any)
if !ok || m["text"] != "HELLO" {
t.Errorf("result = %v, want text=HELLO", res)
}
}
+148
View File
@@ -0,0 +1,148 @@
package vision
import (
"context"
"fmt"
"git.yeij.top/AskaEth/Cyrene_v2/pkg/plugins/sdk"
)
// toolParameters returns the shared input JSON Schema for the vision and OCR
// tools. It builds a fresh map each call rather than sharing a package-level
// variable.
func toolParameters() map[string]any {
return map[string]any{
"type": "object",
"properties": map[string]any{
"image": map[string]any{
"type": "string",
"description": "The image: a local path, an http(s) URL, or a base64/data: URI.",
},
"prompt": map[string]any{
"type": "string",
"description": "Optional instruction for the vision model (vision tool only).",
},
},
"required": []any{"image"},
}
}
// VisionTool wraps a VisionAdapter as an sdk.Tool so the M2 tool loop can invoke
// it and return its description as a tool result, keeping the image out of the
// main-model context. It is safe for concurrent use.
type VisionTool struct {
name string
adapter VisionAdapter
}
// NewVisionTool builds a VisionTool with the given name and adapter.
func NewVisionTool(name string, adapter VisionAdapter) *VisionTool {
return &VisionTool{name: name, adapter: adapter}
}
// Name returns the tool identifier the agent uses to invoke it.
func (t *VisionTool) Name() string { return t.name }
// Description explains what the tool does.
func (t *VisionTool) Description() string {
return "Understand an image and return a description of its contents."
}
// Parameters returns the input JSON Schema.
func (t *VisionTool) Parameters() (map[string]any, error) {
return toolParameters(), nil
}
// Execute runs the adapter on the image from args and returns the description.
func (t *VisionTool) Execute(ctx context.Context, args any) (any, error) {
image, prompt, err := toolArgs(args)
if err != nil {
return nil, err
}
ref, err := ParseImageRef(image)
if err != nil {
return nil, err
}
desc, err := t.adapter.Describe(ctx, ref, prompt)
if err != nil {
return nil, err
}
return map[string]any{"description": desc}, nil
}
// OCRTool wraps an OCRAdapter as an sdk.Tool. It returns the extracted text as a
// tool result. It is safe for concurrent use.
type OCRTool struct {
name string
adapter OCRAdapter
}
// NewOCRTool builds an OCRTool with the given name and adapter.
func NewOCRTool(name string, adapter OCRAdapter) *OCRTool {
return &OCRTool{name: name, adapter: adapter}
}
// Name returns the tool identifier.
func (t *OCRTool) Name() string { return t.name }
// Description explains what the tool does.
func (t *OCRTool) Description() string {
return "Extract text from an image (OCR) and return the recognized text."
}
// Parameters returns the input JSON Schema.
func (t *OCRTool) Parameters() (map[string]any, error) {
return toolParameters(), nil
}
// Execute runs the OCR adapter on the image from args and returns the text.
func (t *OCRTool) Execute(ctx context.Context, args any) (any, error) {
image, _, err := toolArgs(args)
if err != nil {
return nil, err
}
ref, err := ParseImageRef(image)
if err != nil {
return nil, err
}
text, err := t.adapter.Extract(ctx, ref)
if err != nil {
return nil, err
}
return map[string]any{"text": text}, nil
}
// RegisterTools registers the vision and OCR tools with r via the SDK helper.
// Each tool is only registered when its pointer is non-nil. It returns the
// first registration error, or nil when every tool registered.
func RegisterTools(r sdk.Registrar, tool *VisionTool, ocr *OCRTool) error {
if r == nil {
return sdk.ErrNilRegistrar
}
if tool != nil {
if err := sdk.RegisterTool(r, tool); err != nil {
return fmt.Errorf("vision: register tool %q: %w", tool.Name(), err)
}
}
if ocr != nil {
if err := sdk.RegisterTool(r, ocr); err != nil {
return fmt.Errorf("vision: register tool %q: %w", ocr.Name(), err)
}
}
return nil
}
// toolArgs decodes sdk.Tool.Execute args into the image string and an optional
// prompt. Args must be a map[string]any with a non-empty "image" string.
func toolArgs(args any) (image, prompt string, err error) {
m, ok := args.(map[string]any)
if !ok {
return "", "", fmt.Errorf("vision: %w: args must be an object", ErrBadArgs)
}
image, _ = m["image"].(string)
if image == "" {
return "", "", fmt.Errorf("vision: %w: missing image", ErrBadArgs)
}
prompt, _ = m["prompt"].(string)
return image, prompt, nil
}
@@ -0,0 +1,79 @@
// Package vision implements the layered image routing described in
// DevDocs/design/06-多模态.md §0.
//
// A request that carries image content is first checked by DetectMultimodal.
// The VisionRouter then picks one of the layered routes: the main model
// directly (when it accepts images), a tool-shaped vision/OCR adapter (when the
// main model self-assesses poorly or cannot see), or the OCR tool (when the
// image is mainly text). Vision/OCR adapters are registered as sdk.Tool
// capabilities through the capability registry, and their output is a tool
// result, so the image itself never occupies main-model context.
//
// Images may come from a local path, an http(s) URL, or a base64/data: URI.
package vision
import (
"strings"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
)
// DetectMultimodal reports whether msgs contains any non-text content part,
// i.e. whether multimodal handling is required. Today model only distinguishes
// text from image content, so this is true exactly when a message carries an
// image part; the check is written against "not text" so future content types
// (audio) extend it without rework.
func DetectMultimodal(msgs []model.Message) bool {
for _, m := range msgs {
for _, part := range m.Content {
if part.Type != model.ContentTypeText {
return true
}
}
}
return false
}
// ImageSource classifies where an image's bytes come from.
type ImageSource string
const (
// SourcePath is a local filesystem path.
SourcePath ImageSource = "path"
// SourceURL is an http(s):// URL.
SourceURL ImageSource = "url"
// SourceBase64 is a base64 payload, optionally wrapped in a data: URI.
SourceBase64 ImageSource = "base64"
)
// ImageRef is a single image reference plus its resolved source kind.
type ImageRef struct {
// Source is the classified source kind.
Source ImageSource
// Data is the local path, URL, or base64/data: payload.
Data string
}
// ParseImageRef classifies a raw image source string into an ImageRef. Empty
// input is rejected. Local paths that exist on disk are classified as paths;
// http(s) URLs as URLs; every other value, including a data: URI, is treated as
// a base64 payload. This keeps routing deterministic without guessing MIME.
func ParseImageRef(src string) (ImageRef, error) {
if src == "" {
return ImageRef{}, ErrNoImage
}
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
return ImageRef{Source: SourceURL, Data: src}, nil
}
if strings.HasPrefix(src, "file://") {
return ImageRef{Source: SourcePath, Data: strings.TrimPrefix(src, "file://")}, nil
}
// A data: URI is base64 already; treat as base64, not path.
if strings.HasPrefix(src, "data:") {
return ImageRef{Source: SourceBase64, Data: src}, nil
}
if fileExists(src) {
return ImageRef{Source: SourcePath, Data: src}, nil
}
return ImageRef{Source: SourceBase64, Data: src}, nil
}
@@ -0,0 +1,95 @@
package vision
import (
"context"
"strings"
"testing"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
)
func TestDetectMultimodal(t *testing.T) {
t.Parallel()
tests := []struct {
name string
msgs []model.Message
want bool
}{
{name: "text only", msgs: []model.Message{model.TextMessage(model.RoleUser, "hi")}, want: false},
{name: "empty", msgs: nil, want: false},
{name: "single image", msgs: []model.Message{model.ImageMessage(model.RoleUser, "what", "data:image/png;base64,abc")}, want: true},
{name: "image no text", msgs: []model.Message{model.ImageMessage(model.RoleUser, "", "https://x/y.png")}, want: true},
{name: "image in later turn", msgs: []model.Message{
model.TextMessage(model.RoleUser, "hi"),
model.ImageMessage(model.RoleAssistant, "", "http://x/z.png"),
}, want: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := DetectMultimodal(tt.msgs); got != tt.want {
t.Errorf("DetectMultimodal() = %v, want %v", got, tt.want)
}
})
}
}
func TestParseImageRef(t *testing.T) {
t.Parallel()
tests := []struct {
name string
src string
wantSource ImageSource
}{
{name: "http url", src: "https://a/b.png", wantSource: SourceURL},
{name: "https url", src: "https://a/b.png", wantSource: SourceURL},
{name: "file uri", src: "file:///tmp/a.png", wantSource: SourcePath},
{name: "data uri", src: "data:image/png;base64,abc", wantSource: SourceBase64},
{name: "raw base64", src: "aGVsbG8=", wantSource: SourceBase64},
{name: "empty rejected", src: "", wantSource: ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
ref, err := ParseImageRef(tt.src)
if tt.src == "" {
if err == nil {
t.Fatal("want error for empty source, got nil")
}
return
}
if err != nil {
t.Fatalf("ParseImageRef error: %v", err)
}
if ref.Source != tt.wantSource {
t.Errorf("Source = %q, want %q", ref.Source, tt.wantSource)
}
})
}
}
func TestResolve_Base64(t *testing.T) {
t.Parallel()
ref, err := ParseImageRef("data:image/png;base64,aGVsbG8=")
if err != nil {
t.Fatalf("ParseImageRef error: %v", err)
}
mime, data, err := Resolve(context.Background(), ref, nil)
if err != nil {
t.Fatalf("Resolve error: %v", err)
}
if mime != "image/png" {
t.Errorf("mime = %q, want image/png", mime)
}
if string(data) != "hello" {
t.Errorf("data = %q, want hello", data)
}
}
func TestResolve_DataURI(t *testing.T) {
t.Parallel()
got := DataURI("image/webp", []byte{1, 2, 3})
if !strings.HasPrefix(got, "data:image/webp;base64,") {
t.Errorf("DataURI = %q, want data:image/webp;base64,", got)
}
}
+112
View File
@@ -0,0 +1,112 @@
package voice
import (
"context"
"fmt"
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
)
// Chain is a degrading provider list. STT and TTS walk the steps in order,
// returning the first successful result and moving to the next step when a
// provider is unavailable (nil direction) or fails. It implements STT and TTS so
// it can be used anywhere a single provider is expected.
type Chain struct {
steps []Provider
}
// NewChain builds a degrading chain from the given providers in the order they
// should be tried. Passing a SilentProvider last guarantees the chain always
// succeeds.
func NewChain(steps ...Provider) *Chain {
return &Chain{steps: steps}
}
// STT walks the chain for a transcription.
func (c *Chain) STT(ctx context.Context, audio Audio) (string, error) {
if len(c.steps) == 0 {
return "", ErrProviderFailed
}
var lastErr error
for _, p := range c.steps {
stt := p.STT()
if stt == nil {
continue
}
text, err := stt.STT(ctx, audio)
if err == nil {
return text, nil
}
lastErr = err
}
return "", fmt.Errorf("voice: %w: %v", ErrProviderFailed, lastErr)
}
// TTS walks the chain for synthesized audio.
func (c *Chain) TTS(ctx context.Context, text string) (Audio, error) {
if len(c.steps) == 0 {
return Audio{}, ErrProviderFailed
}
var lastErr error
for _, p := range c.steps {
tts := p.TTS()
if tts == nil {
continue
}
audio, err := tts.TTS(ctx, text)
if err == nil {
return audio, nil
}
lastErr = err
}
return Audio{}, fmt.Errorf("voice: %w: %v", ErrProviderFailed, lastErr)
}
// Option customizes a chain built by New.
type Option func(*chainConfig)
// chainConfig carries the assembled steps for New.
type chainConfig struct {
steps []Provider
}
// WithProviders overrides the provider steps assembled from config. Tests use it
// to inject deterministic mock providers instead of the real adaptation-point
// rungs.
func WithProviders(steps ...Provider) Option {
return func(c *chainConfig) { c.steps = steps }
}
// New assembles a chain from cfg using the fixed degradation order
// cloud → local → edge-tts/espeak → silent.
//
// - cloud is included only when a cloud voice setting is present (today an
// API key) so a mis-configured build degrades quietly.
// - local is a pure adaptation point and is only included when supplied via
// WithProviders; by default it is skipped.
// - the edge-tts or espeak rung is included according to cfg.TTSProvider.
// - silent is always appended last so the chain never fails outright.
//
// opts may supply a fully assembled step list to override this mapping.
func New(cfg config.VoiceConfig, opts ...Option) *Chain {
conf := &chainConfig{}
for _, o := range opts {
o(conf)
}
if len(conf.steps) > 0 {
return NewChain(conf.steps...)
}
steps := make([]Provider, 0, 5)
if cfg.DashscopeAPIKey != "" {
steps = append(steps, CloudProvider(newDashboardSTT(cfg), newDashboardTTS(cfg)))
}
switch cfg.TTSProvider {
case "edge-tts", "edge":
steps = append(steps, EdgeTTSProvider(newEdgeTTS()))
case "espeak":
steps = append(steps, EspeakProvider(newEspeak()))
}
steps = append(steps, SilentProvider())
return NewChain(steps...)
}
@@ -0,0 +1,186 @@
package voice
import (
"context"
"errors"
"testing"
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
)
// stubSTT is a mock STT returning a fixed result.
type stubSTT struct {
text string
err error
}
func (s *stubSTT) STT(_ context.Context, _ Audio) (string, error) { return s.text, s.err }
// stubTTS is a mock TTS returning a fixed result.
type stubTTS struct {
audio Audio
err error
}
func (s *stubTTS) TTS(_ context.Context, _ string) (Audio, error) { return s.audio, s.err }
// stubProvider is a mock Provider carrying optional directions.
type stubProvider struct {
name string
stt STT
tts TTS
}
func (s *stubProvider) Name() string { return s.name }
func (s *stubProvider) STT() STT { return s.stt }
func (s *stubProvider) TTS() TTS { return s.tts }
func TestChain_STT_Degrades(t *testing.T) {
t.Parallel()
tests := []struct {
name string
steps []Provider
want string
wantErr bool
}{
{
name: "first provider succeeds",
steps: []Provider{&stubProvider{name: "cloud", stt: &stubSTT{text: "hello"}}},
want: "hello",
},
{
name: "first fails second succeeds",
steps: []Provider{
&stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}},
&stubProvider{name: "local", stt: &stubSTT{text: "degraded"}},
},
want: "degraded",
},
{
name: "nil direction skipped",
steps: []Provider{
&stubProvider{name: "edge", stt: nil},
&stubProvider{name: "silent", stt: &stubSTT{text: "silence"}},
},
want: "silence",
},
{
name: "all fail no silent -> error",
steps: []Provider{
&stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}},
},
wantErr: true,
},
{
name: "all fail silent last -> no error",
steps: []Provider{
&stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}},
SilentProvider(),
},
want: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got, err := NewChain(tt.steps...).STT(context.Background(), Audio{Format: "wav", Data: []byte("x")})
if tt.wantErr {
if !errors.Is(err, ErrProviderFailed) {
t.Fatalf("err = %v, want ErrProviderFailed", err)
}
return
}
if err != nil {
t.Fatalf("STT error: %v", err)
}
if got != tt.want {
t.Errorf("text = %q, want %q", got, tt.want)
}
})
}
}
func TestChain_TTS_Degrades(t *testing.T) {
t.Parallel()
tests := []struct {
name string
steps []Provider
want Audio
wantErr bool
}{
{
name: "first provider succeeds",
steps: []Provider{&stubProvider{name: "cloud", tts: &stubTTS{audio: Audio{Format: "mp3", Data: []byte("a")}}}},
want: Audio{Format: "mp3", Data: []byte("a")},
},
{
name: "first fails second succeeds",
steps: []Provider{
&stubProvider{name: "cloud", tts: &stubTTS{err: ErrNotConfigured}},
&stubProvider{name: "edge-tts", tts: &stubTTS{audio: Audio{Format: "mp3", Data: []byte("b")}}},
},
want: Audio{Format: "mp3", Data: []byte("b")},
},
{
name: "all fail no silent -> error",
steps: []Provider{
&stubProvider{name: "cloud", tts: &stubTTS{err: ErrNotConfigured}},
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got, err := NewChain(tt.steps...).TTS(context.Background(), "hi")
if tt.wantErr {
if !errors.Is(err, ErrProviderFailed) {
t.Fatalf("err = %v, want ErrProviderFailed", err)
}
return
}
if err != nil {
t.Fatalf("TTS error: %v", err)
}
if got.Format != tt.want.Format || string(got.Data) != string(tt.want.Data) {
t.Errorf("audio = %+v, want %+v", got, tt.want)
}
})
}
}
func TestChain_Empty(t *testing.T) {
t.Parallel()
if _, err := NewChain().STT(context.Background(), Audio{}); !errors.Is(err, ErrProviderFailed) {
t.Fatalf("err = %v, want ErrProviderFailed", err)
}
}
func TestNew_ConfigAssembly(t *testing.T) {
t.Parallel()
// With a cloud key and default edge-tts TTS, the chain starts at cloud and
// ends at silent.
c := New(config.VoiceConfig{DashscopeAPIKey: "k"})
names := make([]string, 0, len(c.steps))
for _, p := range c.steps {
names = append(names, p.Name())
}
if len(names) < 2 || names[len(names)-1] != "silent" {
t.Fatalf("steps = %v, want to end at silent", names)
}
if names[0] != "cloud" {
t.Errorf("first step = %q, want cloud", names[0])
}
// Without a key the cloud rung is skipped.
c2 := New(config.VoiceConfig{})
foundCloud := false
for _, p := range c2.steps {
if p.Name() == "cloud" {
foundCloud = true
}
}
if foundCloud {
t.Error("cloud step present without an API key")
}
}
+227
View File
@@ -0,0 +1,227 @@
package voice
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"strings"
"time"
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
)
// cloudMaxResponseBytes caps how much of a cloud voice response is read, so an
// oversized or malformed response cannot exhaust memory. The request timeout
// bounds a single call.
const (
cloudMaxResponseBytes = 8 << 20
cloudHTTPTimeout = 180 * time.Second
)
// HTTPSTT transcribes audio by POSTing a JSON body to a cloud endpoint and
// reading {"text": "..."} back. It is the stubbed default STT implementation;
// the real DashScope/whisper-facing adapter is an adaptation point.
type HTTPSTT struct {
endpoint string
apiKey string
client *http.Client
log *slog.Logger
}
// HTTPTTS synthesizes speech by POSTing a JSON body and reading the returned
// base64 audio. It is the stubbed default TTS implementation.
type HTTPTTS struct {
endpoint string
apiKey string
client *http.Client
log *slog.Logger
}
// HTTPOption customizes the client and logger of an HTTPSTT/HTTPTTS.
type HTTPOption func(*httpConfig)
type httpConfig struct {
client *http.Client
log *slog.Logger
}
// WithHTTPClient sets the *http.Client used for the call; nil keeps the default.
func WithHTTPClient(c *http.Client) HTTPOption {
return func(o *httpConfig) {
if c != nil {
o.client = c
}
}
}
// WithLogger sets the logger used on failures; nil keeps a discard logger.
func WithLogger(l *slog.Logger) HTTPOption {
return func(o *httpConfig) {
if l != nil {
o.log = l
}
}
}
// NewHTTPSTT builds an HTTPSTT for the given endpoint and API key.
func NewHTTPSTT(endpoint, apiKey string, opts ...HTTPOption) *HTTPSTT {
h := &httpConfig{client: defaultHTTPClient(), log: discardLogger()}
for _, o := range opts {
o(h)
}
return &HTTPSTT{endpoint: strings.TrimRight(endpoint, "/"), apiKey: apiKey, client: h.client, log: h.log}
}
// NewHTTPTTS builds an HTTPTTS for the given endpoint and API key.
func NewHTTPTTS(endpoint, apiKey string, opts ...HTTPOption) *HTTPTTS {
h := &httpConfig{client: defaultHTTPClient(), log: discardLogger()}
for _, o := range opts {
o(h)
}
return &HTTPTTS{endpoint: strings.TrimRight(endpoint, "/"), apiKey: apiKey, client: h.client, log: h.log}
}
// sttRequest is the JSON body sent to the STT endpoint.
type sttRequest struct {
Audio string `json:"audio"`
Format string `json:"format"`
}
// sttResponse is the transcription returned by the STT endpoint.
type sttResponse struct {
Text string `json:"text"`
}
// STT posts the audio and returns the recognized text. A blank endpoint returns
// ErrNotConfigured so the chain degrades.
func (s *HTTPSTT) STT(ctx context.Context, audio Audio) (string, error) {
if s.endpoint == "" {
return "", ErrNotConfigured
}
if err := ctx.Err(); err != nil {
return "", err
}
payload, err := json.Marshal(sttRequest{Audio: base64.StdEncoding.EncodeToString(audio.Data), Format: audio.Format})
if err != nil {
return "", fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err)
}
resp, err := postJSON(ctx, s.client, s.apiKey, s.endpoint, payload)
if err != nil {
return "", err
}
defer func() { _ = resp.Body.Close() }()
var decoded sttResponse
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
return "", fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err)
}
return decoded.Text, nil
}
// ttsRequest is the JSON body sent to the TTS endpoint.
type ttsRequest struct {
Text string `json:"text"`
}
// ttsResponse is the synthesized audio returned by the TTS endpoint.
type ttsResponse struct {
Audio string `json:"audio"`
Format string `json:"format"`
}
// TTS posts the text and returns the decoded audio. A blank endpoint returns
// ErrNotConfigured so the chain degrades.
func (s *HTTPTTS) TTS(ctx context.Context, text string) (Audio, error) {
if s.endpoint == "" {
return Audio{}, ErrNotConfigured
}
if strings.TrimSpace(text) == "" {
return Audio{}, ErrEmptyText
}
if err := ctx.Err(); err != nil {
return Audio{}, err
}
payload, err := json.Marshal(ttsRequest{Text: text})
if err != nil {
return Audio{}, fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err)
}
resp, err := postJSON(ctx, s.client, s.apiKey, s.endpoint, payload)
if err != nil {
return Audio{}, err
}
defer func() { _ = resp.Body.Close() }()
var decoded ttsResponse
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
return Audio{}, fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err)
}
data, err := base64.StdEncoding.DecodeString(decoded.Audio)
if err != nil {
return Audio{}, fmt.Errorf("voice: %w: decode audio: %v", ErrProviderFailed, err)
}
format := decoded.Format
if format == "" {
format = "mp3"
}
return Audio{Format: format, Data: data}, nil
}
// postJSON POSTs a JSON payload, wires auth, and checks the status code.
func postJSON(ctx context.Context, client *http.Client, apiKey, url string, payload []byte) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
if err != nil {
return nil, fmt.Errorf("voice: %w: build: %v", ErrProviderFailed, err)
}
req.Header.Set("Content-Type", "application/json")
if apiKey != "" {
req.Header.Set("Authorization", "Bearer "+apiKey)
}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("voice: %w: request: %v", ErrProviderFailed, err)
}
if resp.StatusCode != http.StatusOK {
_ = resp.Body.Close()
return nil, fmt.Errorf("voice: %w: status %d", ErrProviderFailed, resp.StatusCode)
}
return resp, nil
}
// defaultHTTPClient builds a client with a request timeout. A fresh client is
// returned per call rather than a package global, which the constraints forbid.
func defaultHTTPClient() *http.Client {
return &http.Client{Transport: http.DefaultTransport, Timeout: cloudHTTPTimeout}
}
// discardLogger returns a logger that drops all output.
func discardLogger() *slog.Logger {
return slog.New(slog.NewTextHandler(io.Discard, nil))
}
// newDashboardSTT returns the cloud STT for a config. The endpoint is an
// adaptation point and is empty until config gains a voice URL; a call then
// returns ErrNotConfigured and the chain degrades.
func newDashboardSTT(cfg config.VoiceConfig) STT {
return NewHTTPSTT("", cfg.DashscopeAPIKey)
}
// newDashboardTTS returns the cloud TTS for a config.
func newDashboardTTS(cfg config.VoiceConfig) TTS {
return NewHTTPTTS("", cfg.DashscopeAPIKey)
}
// newEdgeTTS returns the edge-tts adapter point (not yet implemented).
func newEdgeTTS() TTS { return notImplementedTTS{} }
// newEspeak returns the espeak adapter point (not yet implemented).
func newEspeak() TTS { return notImplementedTTS{} }
// notImplementedTTS is an adaptation point that always degrades.
type notImplementedTTS struct{}
func (notImplementedTTS) TTS(_ context.Context, _ string) (Audio, error) {
return Audio{}, ErrNotImplemented
}
@@ -0,0 +1,81 @@
package voice
import (
"context"
"encoding/base64"
"encoding/hex"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
)
func TestHTTPSTT(t *testing.T) {
t.Parallel()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode: %v", err)
}
if r.Header.Get("Authorization") != "Bearer sk" {
t.Errorf("auth = %q, want Bearer sk", r.Header.Get("Authorization"))
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"text":"hello world"}`))
}))
defer srv.Close()
stt := NewHTTPSTT(srv.URL, "sk")
text, err := stt.STT(context.Background(), Audio{Format: "wav", Data: []byte("raw")})
if err != nil {
t.Fatalf("STT error: %v", err)
}
if text != "hello world" {
t.Errorf("text = %q, want hello world", text)
}
}
func TestHTTPSTT_NotConfigured(t *testing.T) {
t.Parallel()
stt := NewHTTPSTT("", "")
if _, err := stt.STT(context.Background(), Audio{}); err != ErrNotConfigured {
t.Fatalf("err = %v, want ErrNotConfigured", err)
}
}
func TestHTTPTTS(t *testing.T) {
t.Parallel()
audioBytes := []byte{1, 2, 3}
b64 := base64.StdEncoding.EncodeToString(audioBytes)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode: %v", err)
}
if body["text"] != "hi" {
t.Errorf("text = %v, want hi", body["text"])
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"audio":"` + b64 + `","format":"wav"}`))
}))
defer srv.Close()
tts := NewHTTPTTS(srv.URL, "")
audio, err := tts.TTS(context.Background(), "hi")
if err != nil {
t.Fatalf("TTS error: %v", err)
}
if audio.Format != "wav" || hex.EncodeToString(audio.Data) != hex.EncodeToString(audioBytes) {
t.Errorf("audio = %+v, want wav/010203", audio)
}
}
func TestHTTPTTS_EmptyText(t *testing.T) {
t.Parallel()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
defer srv.Close()
tts := NewHTTPTTS(srv.URL, "")
if _, err := tts.TTS(context.Background(), " "); err != ErrEmptyText {
t.Fatalf("err = %v, want ErrEmptyText", err)
}
}
@@ -0,0 +1,18 @@
package voice
import "errors"
// Sentinel errors returned by the voice package. Each is wrapped with %w so
// callers can test for a category while keeping the cause.
var (
// ErrProviderFailed is returned when every provider in a chain fails.
ErrProviderFailed = errors.New("voice: all providers failed")
// ErrNotConfigured is returned when a provider has no endpoint/setting.
ErrNotConfigured = errors.New("voice: provider not configured")
// ErrNotImplemented is returned by an adaptation-point provider.
ErrNotImplemented = errors.New("voice: provider not implemented")
// ErrEmptyText is returned when TTS receives empty text.
ErrEmptyText = errors.New("voice: empty text")
// ErrClosed is returned when a service call arrives after Close.
ErrClosed = errors.New("voice: service closed")
)
@@ -0,0 +1,79 @@
package voice
import "context"
// Provider is one rung in the voice degradation chain. It may implement zero,
// one, or both directions; a nil return means that direction is not available
// on this rung and the chain moves to the next.
type Provider interface {
// Name returns a stable provider identifier.
Name() string
// STT returns the speech-to-text implementation, or nil when the provider
// cannot transcribe.
STT() STT
// TTS returns the text-to-speech implementation, or nil when the provider
// cannot synthesize.
TTS() TTS
}
// StaticProvider is a Provider with a fixed name whose STT/TTS are set at
// construction. It is used by the concrete providers below.
type StaticProvider struct {
name string
stt STT
tts TTS
}
// Name returns the provider identifier.
func (p *StaticProvider) Name() string { return p.name }
// STT returns the configured STT, or nil.
func (p *StaticProvider) STT() STT { return p.stt }
// TTS returns the configured TTS, or nil.
func (p *StaticProvider) TTS() TTS { return p.tts }
// NewProvider builds a Provider with the given name and optional directions it
// supports.
func NewProvider(name string, stt STT, tts TTS) *StaticProvider {
return &StaticProvider{name: name, stt: stt, tts: tts}
}
// CloudProvider is the cloud rung of the chain. It is an adaptation point: the
// real STT/TTS implementations are wired in via NewProvider with cloud adapters.
func CloudProvider(stt STT, tts TTS) *StaticProvider {
return NewProvider("cloud", stt, tts)
}
// LocalProvider is the local-model rung of the chain.
func LocalProvider(stt STT, tts TTS) *StaticProvider {
return NewProvider("local", stt, tts)
}
// EdgeTTSProvider is the edge-tts rung (text-to-speech only).
func EdgeTTSProvider(tts TTS) *StaticProvider {
return NewProvider("edge-tts", nil, tts)
}
// EspeakProvider is the espeak rung (text-to-speech only).
func EspeakProvider(tts TTS) *StaticProvider {
return NewProvider("espeak", nil, tts)
}
// SilentProvider is the terminal rung. It always succeeds, returning an empty
// transcription and empty audio, so a chain that includes it never errors.
func SilentProvider() *StaticProvider {
return NewProvider("silent", silenceSTT{}, silenceTTS{})
}
// silenceSTT is the silent STT rung; it returns an empty transcription.
type silenceSTT struct{}
func (silenceSTT) STT(_ context.Context, _ Audio) (string, error) { return "", nil }
// silenceTTS is the silent TTS rung; it returns empty audio.
type silenceTTS struct{}
func (silenceTTS) TTS(_ context.Context, _ string) (Audio, error) {
return Audio{}, nil
}
@@ -0,0 +1,160 @@
package voice
import (
"context"
"fmt"
"sync"
)
// Service runs STT/TTS requests on a background worker pool so caller
// goroutines (the agent loop) are never blocked by provider latency. Each
// request is enqueued on a jobs channel and answered on a per-job result
// channel; a canceled caller context or a closed service unblocks the caller
// immediately and no worker goroutine is leaked. It is safe for concurrent use.
type Service struct {
chain *Chain
jobs chan job
bgCtx context.Context
cancel context.CancelFunc
nWorkers int
wg sync.WaitGroup
mu sync.RWMutex
closed bool
}
// ServiceOption customizes a Service.
type ServiceOption func(*Service)
// WithWorkers sets the worker-pool size. Values <= 0 are ignored and the
// default single worker is kept.
func WithWorkers(n int) ServiceOption {
return func(s *Service) {
if n > 0 {
s.nWorkers = n
}
}
}
// jobKind selects the operation a worker performs.
type jobKind int
const (
jobSTT jobKind = iota
jobTTS
)
// job is one request enqueued by the caller.
type job struct {
kind jobKind
audio Audio
text string
res chan jobResult
}
// jobResult carries the outcome written back to the caller.
type jobResult struct {
text string
audio Audio
err error
}
// NewService builds a Service backed by chain and starts the worker pool.
func NewService(chain *Chain, opts ...ServiceOption) *Service {
bg, cancel := context.WithCancel(context.Background())
s := &Service{
chain: chain,
jobs: make(chan job),
bgCtx: bg,
cancel: cancel,
nWorkers: 1,
}
for _, o := range opts {
o(s)
}
for i := 0; i < s.nWorkers; i++ {
s.wg.Add(1)
go s.worker()
}
return s
}
// worker consumes jobs until the service closes.
func (s *Service) worker() {
defer s.wg.Done()
for {
select {
case <-s.bgCtx.Done():
return
case j := <-s.jobs:
s.run(j)
}
}
}
// run executes a job and delivers the result. Providers run under the service's
// background context so Close cancels in-flight work and no worker goroutine is
// leaked; the result is written to a buffered channel so a caller that already
// returned never blocks the worker.
func (s *Service) run(j job) {
res := jobResult{}
switch j.kind {
case jobSTT:
res.text, res.err = s.chain.STT(s.bgCtx, j.audio)
case jobTTS:
res.audio, res.err = s.chain.TTS(s.bgCtx, j.text)
}
j.res <- res
}
// STT enqueues a transcription request and waits for the result. ctx cancels
// the wait and the underlying provider call.
func (s *Service) STT(ctx context.Context, audio Audio) (string, error) {
res, err := s.submit(ctx, job{kind: jobSTT, audio: audio, res: make(chan jobResult, 1)})
if err != nil {
return "", err
}
return res.text, res.err
}
// TTS enqueues a synthesis request and waits for the result. ctx cancels the
// wait and the underlying provider call.
func (s *Service) TTS(ctx context.Context, text string) (Audio, error) {
res, err := s.submit(ctx, job{kind: jobTTS, text: text, res: make(chan jobResult, 1)})
if err != nil {
return Audio{}, err
}
return res.audio, res.err
}
// submit enqueues j and blocks until the result or a cancellation.
func (s *Service) submit(ctx context.Context, j job) (jobResult, error) {
select {
case s.jobs <- j:
case <-ctx.Done():
return jobResult{}, fmt.Errorf("voice: %w", ctx.Err())
case <-s.bgCtx.Done():
return jobResult{}, ErrClosed
}
select {
case r := <-j.res:
return r, nil
case <-ctx.Done():
return jobResult{}, fmt.Errorf("voice: %w", ctx.Err())
case <-s.bgCtx.Done():
return jobResult{}, ErrClosed
}
}
// Close cancels the worker pool and waits for in-flight jobs to finish. It is
// safe to call once; later calls are no-ops. In-flight provider calls are
// bounded by each provider's own context timeout, so Close returns promptly.
func (s *Service) Close() {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return
}
s.closed = true
s.cancel()
s.wg.Wait()
}
@@ -0,0 +1,108 @@
package voice
import (
"context"
"errors"
"sync"
"testing"
"time"
)
func TestService_STTAndTTS(t *testing.T) {
t.Parallel()
chain := NewChain(
NewProvider("cloud", &stubSTT{text: "transcribed"}, &stubTTS{audio: Audio{Format: "mp3", Data: []byte("a")}}),
)
svc := NewService(chain)
defer svc.Close()
text, err := svc.STT(context.Background(), Audio{Format: "wav", Data: []byte("x")})
if err != nil {
t.Fatalf("STT error: %v", err)
}
if text != "transcribed" {
t.Errorf("text = %q, want transcribed", text)
}
audio, err := svc.TTS(context.Background(), "hello")
if err != nil {
t.Fatalf("TTS error: %v", err)
}
if string(audio.Data) != "a" || audio.Format != "mp3" {
t.Errorf("audio = %+v, want mp3/a", audio)
}
}
func TestService_CloseStopsNewCalls(t *testing.T) {
t.Parallel()
svc := NewService(NewChain(SilentProvider()))
svc.Close()
_, err := svc.STT(context.Background(), Audio{})
if !errors.Is(err, ErrClosed) {
t.Fatalf("err = %v, want ErrClosed", err)
}
}
// blockingSTT blocks until its context is canceled, marking that it started.
type blockingSTT struct {
started chan struct{}
once sync.Once
}
func (b *blockingSTT) STT(ctx context.Context, _ Audio) (string, error) {
b.once.Do(func() { close(b.started) })
<-ctx.Done()
return "", ctx.Err()
}
func TestService_CloseCancelsInFlightWorker(t *testing.T) {
t.Parallel()
blk := &blockingSTT{started: make(chan struct{})}
chain := NewChain(NewProvider("cloud", blk, nil))
svc := NewService(chain)
done := make(chan struct{})
go func() {
defer close(done)
_, _ = svc.STT(context.Background(), Audio{Format: "wav", Data: []byte("x")})
}()
select {
case <-blk.started:
case <-time.After(2 * time.Second):
t.Fatal("provider never started")
}
svc.Close()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("STT call did not return after Close; worker likely leaked")
}
}
func TestService_CallerCancelUnblocks(t *testing.T) {
t.Parallel()
blk := &blockingSTT{started: make(chan struct{})}
svc := NewService(NewChain(NewProvider("cloud", blk, nil)))
defer svc.Close()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan struct{})
go func() {
defer close(done)
_, _ = svc.STT(ctx, Audio{Format: "wav", Data: []byte("x")})
}()
<-blk.started
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("caller cancellation did not unblock STT")
}
}
@@ -0,0 +1,35 @@
// Package voice implements asynchronous speech-to-text and text-to-speech for
// the Cyrene runtime, following DevDocs/design/06-多模态.md §1.
//
// Voice input (STT) is transcribed to text and fed into the same agent pipeline
// as a chat message; the reply is synthesized (TTS) back to audio. Both
// directions degrade through a provider chain — cloud → local → edge-tts/espeak
// → silent — so an unavailable provider falls through to the next rung instead
// of failing the whole turn. The stub default implementations are HTTP-based and
// testable with a stdlib httptest server; real cloud/local providers remain
// adaptation points for later milestones.
package voice
import "context"
// Audio is a piece of audio data plus its encoding.
type Audio struct {
// Format is the encoding, e.g. "wav", "mp3", "ogg".
Format string
// Data is the raw audio bytes.
Data []byte
}
// STT converts audio to text.
type STT interface {
// STT transcribes audio and returns the spoken text. ctx controls
// cancellation and must be honored.
STT(ctx context.Context, audio Audio) (string, error)
}
// TTS synthesizes speech audio from text.
type TTS interface {
// TTS synthesizes audio from text. ctx controls cancellation and must be
// honored.
TTS(ctx context.Context, text string) (Audio, error)
}