feat(multimodal): 视觉分层路由(多模态检测→主/视觉/OCR 工具)+ 语音 STT→TTS 降级链(异步)
This commit is contained in:
1 parent
02aae2bbb4
commit
0a936a3202
19 files changed
+2121
No files matched your search
@@ -0,0 +1,17 @@
|
||||
// Package multimodal groups the runtime's non-text input/output handling:
|
||||
// image understanding (vision) and speech (voice).
|
||||
//
|
||||
// It follows DevDocs/design/06-多模态.md:
|
||||
//
|
||||
// - The vision subpackage routes image messages through a layered strategy
|
||||
// (main model first, then a tool-shaped vision/OCR adapter) and registers
|
||||
// those adapters as sdk.Tool capabilities so the M2 tool loop can invoke
|
||||
// them without putting the image in the main-model context.
|
||||
// - The voice subpackage provides asynchronous STT/TTS through a degrading
|
||||
// provider chain (cloud → local → edge-tts/espeak → silent).
|
||||
//
|
||||
// Both subpackages keep real provider calls behind interfaces with an HTTP
|
||||
// default implementation, so they are testable with a stdlib httptest server
|
||||
// and leave the real cloud/local providers as adaptation points for later
|
||||
// milestones.
|
||||
package multimodal
|
||||
@@ -0,0 +1,217 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||
)
|
||||
|
||||
// visionMaxTokens caps the vision/OCR response length.
|
||||
const visionMaxTokens = 512
|
||||
|
||||
// VisionAdapter understands an image and returns a description of it. It is the
|
||||
// backend behind the vision tool; real providers implement it without exposing
|
||||
// the image to the main-model context.
|
||||
type VisionAdapter interface {
|
||||
// Describe returns a description of img. prompt is an optional instruction
|
||||
// overriding the default "describe this image" ask.
|
||||
Describe(ctx context.Context, img ImageRef, prompt string) (string, error)
|
||||
}
|
||||
|
||||
// OCRAdapter extracts text from an image. It is the backend behind the OCR tool.
|
||||
type OCRAdapter interface {
|
||||
// Extract returns the text found in img.
|
||||
Extract(ctx context.Context, img ImageRef) (string, error)
|
||||
}
|
||||
|
||||
// OpenAICompatibleAdapter implements both VisionAdapter and OCRAdapter by
|
||||
// calling an OpenAI-compatible /chat/completions endpoint with the image
|
||||
// embedded as a content part. It is the ready default implementation; real
|
||||
// cloud vision/OCR providers expose the same protocol.
|
||||
type OpenAICompatibleAdapter struct {
|
||||
cfg config.LLMConfig
|
||||
model string
|
||||
client *http.Client
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// OpenAIOption customizes an OpenAICompatibleAdapter.
|
||||
type OpenAIOption func(*OpenAICompatibleAdapter)
|
||||
|
||||
// WithModel overrides the adapter model (defaults to cfg.Model).
|
||||
func WithModel(m string) OpenAIOption {
|
||||
return func(a *OpenAICompatibleAdapter) {
|
||||
if m != "" {
|
||||
a.model = m
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithHTTPClient sets the *http.Client used for the request; nil keeps default.
|
||||
func WithHTTPClient(c *http.Client) OpenAIOption {
|
||||
return func(a *OpenAICompatibleAdapter) {
|
||||
if c != nil {
|
||||
a.client = c
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithLogger sets the logger used on failures; nil keeps a discard logger.
|
||||
func WithLogger(l *slog.Logger) OpenAIOption {
|
||||
return func(a *OpenAICompatibleAdapter) {
|
||||
if l != nil {
|
||||
a.log = l
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewOpenAICompatible builds an adapter from cfg, using cfg.Model unless
|
||||
// overridden.
|
||||
func NewOpenAICompatible(cfg config.LLMConfig, opts ...OpenAIOption) *OpenAICompatibleAdapter {
|
||||
a := &OpenAICompatibleAdapter{
|
||||
cfg: cfg,
|
||||
model: cfg.Model,
|
||||
client: defaultHTTPClient(),
|
||||
log: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
}
|
||||
for _, o := range opts {
|
||||
o(a)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// Describe returns an image description from the underlying chat endpoint.
|
||||
func (a *OpenAICompatibleAdapter) Describe(ctx context.Context, img ImageRef, prompt string) (string, error) {
|
||||
if prompt == "" {
|
||||
prompt = "Describe this image in detail."
|
||||
}
|
||||
return a.ask(ctx, img, prompt)
|
||||
}
|
||||
|
||||
// Extract returns text found in the image via the same endpoint.
|
||||
func (a *OpenAICompatibleAdapter) Extract(ctx context.Context, img ImageRef) (string, error) {
|
||||
return a.ask(ctx, img, "Extract all text visible in this image. Return only the text.")
|
||||
}
|
||||
|
||||
// ask resolves the image, builds a chat request and returns the assistant text.
|
||||
func (a *OpenAICompatibleAdapter) ask(ctx context.Context, img ImageRef, prompt string) (string, error) {
|
||||
if strings.TrimSpace(a.cfg.BaseURL) == "" {
|
||||
return "", ErrNotConfigured
|
||||
}
|
||||
if strings.TrimSpace(a.model) == "" {
|
||||
return "", ErrNotConfigured
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
imageURL, err := a.imageURL(ctx, img)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
payload, err := json.Marshal(a.request(imageURL, prompt))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("vision: %w: marshal request: %v", ErrUpstream, err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(a.cfg.BaseURL, "/")+"/chat/completions", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("vision: %w: build request: %v", ErrUpstream, err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if a.cfg.APIKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+a.cfg.APIKey)
|
||||
}
|
||||
resp, err := a.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("vision: %w: %v", ErrUpstream, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
snippet, _ := io.ReadAll(io.LimitReader(resp.Body, maxFetchedBody))
|
||||
return "", fmt.Errorf("vision: %w: status %d: %s", ErrUpstream, resp.StatusCode, strings.TrimSpace(string(snippet)))
|
||||
}
|
||||
return a.decode(resp.Body)
|
||||
}
|
||||
|
||||
// imageURL converts an ImageRef to the image_url value the wire expects: an
|
||||
// http(s) URL passes through, anything else becomes a base64 data: URI.
|
||||
func (a *OpenAICompatibleAdapter) imageURL(ctx context.Context, img ImageRef) (string, error) {
|
||||
if img.Source == SourceURL {
|
||||
return img.Data, nil
|
||||
}
|
||||
mime, data, err := Resolve(ctx, img, a.client)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return DataURI(mime, data), nil
|
||||
}
|
||||
|
||||
// openAIVisionRequest is the JSON body sent to /chat/completions.
|
||||
type openAIVisionRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []openAIMessage `json:"messages"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
}
|
||||
|
||||
type openAIMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content any `json:"content"`
|
||||
}
|
||||
|
||||
type openAIPart struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text,omitempty"`
|
||||
ImageURL *openAIImageURL `json:"image_url,omitempty"`
|
||||
}
|
||||
|
||||
type openAIImageURL struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type openAIVisionResponse struct {
|
||||
Choices []openAIChoice `json:"choices"`
|
||||
}
|
||||
|
||||
type openAIChoice struct {
|
||||
Message openAIResponseMessage `json:"message"`
|
||||
}
|
||||
|
||||
type openAIResponseMessage struct {
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
// request builds the wire body for an image+prompt.
|
||||
func (a *OpenAICompatibleAdapter) request(imageURL, prompt string) openAIVisionRequest {
|
||||
return openAIVisionRequest{
|
||||
Model: a.model,
|
||||
Messages: []openAIMessage{
|
||||
{Role: "user", Content: []openAIPart{
|
||||
{Type: "text", Text: prompt},
|
||||
{Type: "image_url", ImageURL: &openAIImageURL{URL: imageURL}},
|
||||
}},
|
||||
},
|
||||
MaxTokens: visionMaxTokens,
|
||||
}
|
||||
}
|
||||
|
||||
// decode extracts the first choice's text from a response body.
|
||||
func (a *OpenAICompatibleAdapter) decode(r io.Reader) (string, error) {
|
||||
var wire openAIVisionResponse
|
||||
if err := json.NewDecoder(r).Decode(&wire); err != nil {
|
||||
return "", fmt.Errorf("vision: %w: decode response: %v", ErrUpstream, err)
|
||||
}
|
||||
if len(wire.Choices) == 0 {
|
||||
return "", fmt.Errorf("vision: %w: empty choices", ErrUpstream)
|
||||
}
|
||||
return wire.Choices[0].Message.Content, nil
|
||||
}
|
||||
|
||||
// interface assertion: one concrete adapter serves both directions.
|
||||
var _ VisionAdapter = (*OpenAICompatibleAdapter)(nil)
|
||||
var _ OCRAdapter = (*OpenAICompatibleAdapter)(nil)
|
||||
@@ -0,0 +1,53 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||
)
|
||||
|
||||
func TestOpenAICompatibleAdapter_Describe(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/chat/completions" {
|
||||
t.Errorf("path = %s, want /chat/completions", r.URL.Path)
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
msgs, _ := body["messages"].([]any)
|
||||
if len(msgs) != 1 {
|
||||
t.Fatalf("messages = %d, want 1", len(msgs))
|
||||
}
|
||||
content, _ := msgs[0].(map[string]any)["content"].([]any)
|
||||
if len(content) != 2 {
|
||||
t.Fatalf("content parts = %d, want 2", len(content))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"a red car"}}]}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ad := NewOpenAICompatible(config.LLMConfig{BaseURL: srv.URL, Model: "vision-"})
|
||||
text, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: "aGVsbG8="}, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Describe error: %v", err)
|
||||
}
|
||||
if text != "a red car" {
|
||||
t.Errorf("text = %q, want a red car", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleAdapter_NotConfigured(t *testing.T) {
|
||||
t.Parallel()
|
||||
ad := NewOpenAICompatible(config.LLMConfig{})
|
||||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: "aGVsbG8="}, ""); err == nil {
|
||||
t.Fatal("want error for empty base url, got nil")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package vision
|
||||
|
||||
import "errors"
|
||||
|
||||
// Sentinel errors returned by the vision package. Each is wrapped with %w on
|
||||
// the way out so callers can test for a category while keeping the cause.
|
||||
var (
|
||||
// ErrNoImage is returned when an image source string is empty.
|
||||
ErrNoImage = errors.New("vision: empty image source")
|
||||
// ErrNotConfigured is returned by adapters when no endpoint/model is set.
|
||||
ErrNotConfigured = errors.New("vision: adapter not configured")
|
||||
// ErrUnsupportedSource is returned when an image source cannot be handled.
|
||||
ErrUnsupportedSource = errors.New("vision: unsupported image source")
|
||||
// ErrFetch is returned when a URL image cannot be downloaded.
|
||||
ErrFetch = errors.New("vision: cannot fetch image")
|
||||
// ErrUpstream is returned when the vision/OCR provider request fails.
|
||||
ErrUpstream = errors.New("vision: upstream request failed")
|
||||
// ErrBadImage is returned when an image cannot be read or decoded.
|
||||
ErrBadImage = errors.New("vision: cannot read image")
|
||||
// ErrBadArgs is returned by a tool when its arguments are malformed.
|
||||
ErrBadArgs = errors.New("vision: bad tool arguments")
|
||||
)
|
||||
@@ -0,0 +1,178 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Size caps for reading image bytes and response bodies, so a hostile or
|
||||
// malformed source cannot exhaust memory. The 50MB cap matches the file-upload
|
||||
// limit in DevDocs/02-development-constraints.md §11.
|
||||
const (
|
||||
maxImageBytes = 50 << 20
|
||||
maxFetchedBody = 50 << 20
|
||||
)
|
||||
|
||||
// DataURI builds a base64 data: image URI from mime and raw bytes.
|
||||
func DataURI(mime string, data []byte) string {
|
||||
return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data)
|
||||
}
|
||||
|
||||
// Resolve loads the bytes for an ImageRef. Paths and base64 payloads are read
|
||||
// locally; URLs are fetched with an http.Client (the provided client, or a
|
||||
// default one when nil). For base64 input the Data field is returned verbatim as
|
||||
// the payload and the MIME type is guessed from a data: prefix.
|
||||
func Resolve(ctx context.Context, ref ImageRef, client *http.Client) (mime string, data []byte, err error) {
|
||||
switch ref.Source {
|
||||
case SourcePath:
|
||||
return readFile(ref.Data)
|
||||
case SourceBase64:
|
||||
return decodeBase64(ref.Data)
|
||||
case SourceURL:
|
||||
return fetchURL(ctx, ref.Data, client)
|
||||
default:
|
||||
return "", nil, fmt.Errorf("vision: %w: %q", ErrUnsupportedSource, ref.Source)
|
||||
}
|
||||
}
|
||||
|
||||
// readFile reads an image file and guesses its MIME type from the extension.
|
||||
func readFile(path string) (string, []byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
data, err := io.ReadAll(io.LimitReader(f, maxImageBytes))
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
|
||||
}
|
||||
return mimeByExt(path), data, nil
|
||||
}
|
||||
|
||||
// decodeBase64 returns the decoded bytes of a base64/data: payload and its MIME.
|
||||
func decodeBase64(src string) (string, []byte, error) {
|
||||
mime, encoded, isDataURI := splitDataURI(src)
|
||||
if isDataURI {
|
||||
decoded, err := base64.StdEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
|
||||
}
|
||||
return mime, decoded, nil
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(src)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
|
||||
}
|
||||
return "image/png", decoded, nil
|
||||
}
|
||||
|
||||
// fetchURL downloads an image over http(s); only http/https schemes are allowed.
|
||||
func fetchURL(ctx context.Context, rawURL string, client *http.Client) (string, []byte, error) {
|
||||
if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
|
||||
return "", nil, fmt.Errorf("vision: %w: %q", ErrUnsupportedSource, rawURL)
|
||||
}
|
||||
if client == nil {
|
||||
client = defaultHTTPClient()
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", nil, fmt.Errorf("vision: %w: status %d", ErrFetch, resp.StatusCode)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, maxFetchedBody))
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err)
|
||||
}
|
||||
return mimeByContentType(resp.Header.Get("Content-Type")), data, nil
|
||||
}
|
||||
|
||||
// splitDataURI parses a "data:<mime>;base64,<payload>" prefix. The second return
|
||||
// is false when src is not a data: URI.
|
||||
func splitDataURI(src string) (mime, payload string, ok bool) {
|
||||
if !strings.HasPrefix(src, "data:") {
|
||||
return "", "", false
|
||||
}
|
||||
rest := strings.TrimPrefix(src, "data:")
|
||||
comma := strings.IndexByte(rest, ',')
|
||||
if comma < 0 {
|
||||
return "", "", false
|
||||
}
|
||||
meta := rest[:comma]
|
||||
payload = rest[comma+1:]
|
||||
mime = "image/png"
|
||||
if strings.HasPrefix(meta, ";base64") {
|
||||
return mime, payload, true
|
||||
}
|
||||
if idx := strings.Index(meta, ";base64"); idx >= 0 {
|
||||
if t := strings.TrimSuffix(meta[:idx], ";base64"); t != "" {
|
||||
mime = t
|
||||
}
|
||||
return mime, payload, true
|
||||
}
|
||||
return mime, payload, true
|
||||
}
|
||||
|
||||
// mimeByExt guesses a MIME type from a file path extension.
|
||||
func mimeByExt(path string) string {
|
||||
switch strings.ToLower(pathExtension(path)) {
|
||||
case ".png":
|
||||
return "image/png"
|
||||
case ".jpg", ".jpeg":
|
||||
return "image/jpeg"
|
||||
case ".gif":
|
||||
return "image/gif"
|
||||
case ".webp":
|
||||
return "image/webp"
|
||||
default:
|
||||
return "image/png"
|
||||
}
|
||||
}
|
||||
|
||||
// mimeByContentType returns a MIME type from an HTTP Content-Type header,
|
||||
// defaulting to image/png when absent or non-image.
|
||||
func mimeByContentType(ct string) string {
|
||||
if ct == "" || !strings.HasPrefix(ct, "image/") {
|
||||
return "image/png"
|
||||
}
|
||||
return strings.TrimSpace(strings.Split(ct, ";")[0])
|
||||
}
|
||||
|
||||
// pathExtension returns the file extension with its dot, or empty when none.
|
||||
func pathExtension(path string) string {
|
||||
base := path
|
||||
if idx := strings.LastIndexByte(path, '/'); idx >= 0 {
|
||||
base = path[idx+1:]
|
||||
} else if idx := strings.LastIndexByte(path, '\\'); idx >= 0 {
|
||||
base = path[idx+1:]
|
||||
}
|
||||
dot := strings.LastIndexByte(base, '.')
|
||||
if dot < 0 {
|
||||
return ""
|
||||
}
|
||||
return base[dot:]
|
||||
}
|
||||
|
||||
// fileExists reports whether path exists on disk and is a regular file.
|
||||
func fileExists(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && info.Mode().IsRegular()
|
||||
}
|
||||
|
||||
// defaultHTTPClient builds the HTTP client described in
|
||||
// DevDocs/02-development-constraints.md §2.3. A fresh client is returned per
|
||||
// call rather than a package global, which the constraints forbid.
|
||||
func defaultHTTPClient() *http.Client {
|
||||
return &http.Client{Transport: http.DefaultTransport}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package vision
|
||||
|
||||
// Assessment is the model's own confidence in reading an image.
|
||||
type Assessment string
|
||||
|
||||
const (
|
||||
// AssessmentUnknown means the model reported no self-assessment.
|
||||
AssessmentUnknown Assessment = "unknown"
|
||||
// AssessmentGood means the model is confident in its read.
|
||||
AssessmentGood Assessment = "good"
|
||||
// AssessmentPoor means the model judged its recognition as poor.
|
||||
AssessmentPoor Assessment = "poor"
|
||||
)
|
||||
|
||||
// ModelInfo describes the main model's vision capability.
|
||||
type ModelInfo struct {
|
||||
// Name identifies the model.
|
||||
Name string
|
||||
// SupportsVision reports whether the model accepts image input.
|
||||
SupportsVision bool
|
||||
// SelfAssessment is the model's own confidence for the current image.
|
||||
// It drives the layer-2 tool route when SupportsVision is true.
|
||||
SelfAssessment Assessment
|
||||
}
|
||||
|
||||
// DecisionKind is the chosen layered route.
|
||||
type DecisionKind string
|
||||
|
||||
const (
|
||||
// KindMain routes the image to the main model with the image in context.
|
||||
KindMain DecisionKind = "main"
|
||||
// KindVisionTool routes to a dedicated vision tool, returning only the tool
|
||||
// result so the image never occupies main-model context.
|
||||
KindVisionTool DecisionKind = "vision_tool"
|
||||
// KindOCRTool routes to an OCR tool, returning only the extracted text.
|
||||
KindOCRTool DecisionKind = "ocr_tool"
|
||||
// KindUnsupported means no route can process the image.
|
||||
KindUnsupported DecisionKind = "unsupported"
|
||||
)
|
||||
|
||||
// Decision is the outcome of a routing call.
|
||||
type Decision struct {
|
||||
// Kind is the chosen route.
|
||||
Kind DecisionKind
|
||||
// Tool names the vision or OCR tool to invoke, set when Kind is a tool route.
|
||||
Tool string
|
||||
// Reason is a human-readable explanation for observability and logs.
|
||||
Reason string
|
||||
}
|
||||
|
||||
// RouteRequest carries routing hints from the caller on top of the model info.
|
||||
type RouteRequest struct {
|
||||
// TextPrimary reports that the user indicated the image is mainly text, so
|
||||
// an OCR tool is preferred over a general vision model.
|
||||
TextPrimary bool
|
||||
// MainSelfAssessedPoor is set when the main model has already judged its own
|
||||
// read of the image as poor, requesting a vision/OCR tool route.
|
||||
MainSelfAssessedPoor bool
|
||||
}
|
||||
|
||||
// RouterOption configures a Router.
|
||||
type RouterOption func(*Router)
|
||||
|
||||
// WithVisionTool binds the name of the registered vision-understanding tool.
|
||||
func WithVisionTool(name string) RouterOption {
|
||||
return func(r *Router) {
|
||||
if name != "" {
|
||||
r.visionTool = name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithOCRTool binds the name of the registered OCR tool.
|
||||
func WithOCRTool(name string) RouterOption {
|
||||
return func(r *Router) {
|
||||
if name != "" {
|
||||
r.ocrTool = name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Router implements the layered vision routing policy. It does no I/O; the
|
||||
// caller uses the returned Decision to dispatch the request.
|
||||
type Router struct {
|
||||
visionTool string
|
||||
ocrTool string
|
||||
}
|
||||
|
||||
// NewRouter builds a Router. Unknown (empty) tool names leave the corresponding
|
||||
// route unavailable, so the router degrades to KindMain or KindUnsupported.
|
||||
func NewRouter(opts ...RouterOption) *Router {
|
||||
r := &Router{}
|
||||
for _, o := range opts {
|
||||
o(r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Route picks a layered route for an image request given the target model info
|
||||
// and caller hints. The order is: the main model directly when it can see and
|
||||
// has not self-assessed poorly (layer 1); otherwise a vision or OCR tool whose
|
||||
// result replaces the image in context (layers 2 and 3); otherwise
|
||||
// KindUnsupported when nothing can process the image.
|
||||
func (r *Router) Route(target ModelInfo, req RouteRequest) Decision {
|
||||
poor := target.SelfAssessment == AssessmentPoor || req.MainSelfAssessedPoor
|
||||
// Layer 1: the main model accepts images and is confident, so use it first.
|
||||
if target.SupportsVision && !poor {
|
||||
return Decision{
|
||||
Kind: KindMain,
|
||||
Reason: "main model supports vision and is confident; use it directly",
|
||||
}
|
||||
}
|
||||
// OCR is preferred when the user says the image is mostly text.
|
||||
if req.TextPrimary && r.ocrTool != "" {
|
||||
return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "image is mainly text; route to OCR"}
|
||||
}
|
||||
if r.visionTool != "" {
|
||||
return Decision{Kind: KindVisionTool, Tool: r.visionTool, Reason: "route to dedicated vision tool"}
|
||||
}
|
||||
if r.ocrTool != "" {
|
||||
return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "route to OCR tool"}
|
||||
}
|
||||
// No tool is bound. A sighted main model remains the only option.
|
||||
if target.SupportsVision {
|
||||
return Decision{Kind: KindMain, Reason: "no vision/OCR tool bound; keep image with main model"}
|
||||
}
|
||||
return Decision{Kind: KindUnsupported, Reason: "main model cannot see and no vision/OCR tool is bound"}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/capability/registry"
|
||||
)
|
||||
|
||||
func TestRouter_Route(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
router *Router
|
||||
target ModelInfo
|
||||
req RouteRequest
|
||||
want DecisionKind
|
||||
tool string
|
||||
}{
|
||||
{
|
||||
name: "sighted confident main -> main",
|
||||
router: NewRouter(WithVisionTool("v"), WithOCRTool("o")),
|
||||
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentGood},
|
||||
want: KindMain,
|
||||
},
|
||||
{
|
||||
name: "poor self-assessment sighted main -> vision tool",
|
||||
router: NewRouter(WithVisionTool("v"), WithOCRTool("o")),
|
||||
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor},
|
||||
req: RouteRequest{MainSelfAssessedPoor: true},
|
||||
want: KindVisionTool,
|
||||
tool: "v",
|
||||
},
|
||||
{
|
||||
name: "poor self-assessment only OCR -> OCR tool",
|
||||
router: NewRouter(WithOCRTool("o")),
|
||||
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor},
|
||||
want: KindOCRTool,
|
||||
tool: "o",
|
||||
},
|
||||
{
|
||||
name: "blind main with vision tool -> vision tool",
|
||||
router: NewRouter(WithVisionTool("v")),
|
||||
target: ModelInfo{Name: "m", SupportsVision: false},
|
||||
want: KindVisionTool,
|
||||
tool: "v",
|
||||
},
|
||||
{
|
||||
name: "blind main text primary -> OCR tool",
|
||||
router: NewRouter(WithVisionTool("v"), WithOCRTool("o")),
|
||||
target: ModelInfo{Name: "m", SupportsVision: false},
|
||||
req: RouteRequest{TextPrimary: true},
|
||||
want: KindOCRTool,
|
||||
tool: "o",
|
||||
},
|
||||
{
|
||||
name: "blind main only OCR non-text -> OCR fallback",
|
||||
router: NewRouter(WithOCRTool("o")),
|
||||
target: ModelInfo{Name: "m", SupportsVision: false},
|
||||
want: KindOCRTool,
|
||||
tool: "o",
|
||||
},
|
||||
{
|
||||
name: "blind main no tools -> unsupported",
|
||||
router: NewRouter(),
|
||||
target: ModelInfo{Name: "m", SupportsVision: false},
|
||||
want: KindUnsupported,
|
||||
},
|
||||
{
|
||||
name: "sighted main no tools -> main",
|
||||
router: NewRouter(),
|
||||
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentGood},
|
||||
want: KindMain,
|
||||
},
|
||||
{
|
||||
name: "sighted main poor no tools -> main fallback",
|
||||
router: NewRouter(),
|
||||
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor},
|
||||
want: KindMain,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := tc.router.Route(tc.target, tc.req)
|
||||
if got.Kind != tc.want {
|
||||
t.Fatalf("Kind = %q, want %q (reason: %s)", got.Kind, tc.want, got.Reason)
|
||||
}
|
||||
if tc.tool != "" && got.Tool != tc.tool {
|
||||
t.Errorf("Tool = %q, want %q", got.Tool, tc.tool)
|
||||
}
|
||||
if got.Reason == "" {
|
||||
t.Error("Reason must be non-empty")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// fakeVision implements VisionAdapter for tool tests.
|
||||
type fakeVision struct{ desc string }
|
||||
|
||||
func (f *fakeVision) Describe(_ context.Context, _ ImageRef, _ string) (string, error) {
|
||||
return f.desc, nil
|
||||
}
|
||||
|
||||
// fakeOCR implements OCRAdapter for tool tests.
|
||||
type fakeOCR struct{ text string }
|
||||
|
||||
func (f *fakeOCR) Extract(_ context.Context, _ ImageRef) (string, error) {
|
||||
return f.text, nil
|
||||
}
|
||||
|
||||
func TestRegisterTools(t *testing.T) {
|
||||
t.Parallel()
|
||||
reg := registry.New()
|
||||
v := NewVisionTool("vision_understand", &fakeVision{desc: "a cat"})
|
||||
o := NewOCRTool("ocr_extract", &fakeOCR{text: "HELLO"})
|
||||
|
||||
if err := RegisterTools(reg, v, o); err != nil {
|
||||
t.Fatalf("RegisterTools error: %v", err)
|
||||
}
|
||||
if capv, err := reg.Get("vision_understand"); err != nil || capv.Kind != registry.KindTool {
|
||||
t.Fatalf("Get(vision_understand) = %+v, %v", capv, err)
|
||||
}
|
||||
if capo, err := reg.Get("ocr_extract"); err != nil || capo.Kind != registry.KindTool {
|
||||
t.Fatalf("Get(ocr_extract) = %+v, %v", capo, err)
|
||||
}
|
||||
if len(reg.List(registry.KindTool)) != 2 {
|
||||
t.Errorf("List(tool) = %d, want 2", len(reg.List(registry.KindTool)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterTools_NilRegistrar(t *testing.T) {
|
||||
t.Parallel()
|
||||
if err := RegisterTools(nil, nil, nil); err == nil {
|
||||
t.Fatal("want error for nil registrar, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionTool_Execute(t *testing.T) {
|
||||
t.Parallel()
|
||||
v := NewVisionTool("vision_understand", &fakeVision{desc: "a cat"})
|
||||
res, err := v.Execute(context.Background(), map[string]any{
|
||||
"image": "data:image/png;base64,aGVsbG8=",
|
||||
"prompt": "what is this",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Execute error: %v", err)
|
||||
}
|
||||
m, ok := res.(map[string]any)
|
||||
if !ok || m["description"] != "a cat" {
|
||||
t.Errorf("result = %v, want description=a cat", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionTool_Execute_BadArgs(t *testing.T) {
|
||||
t.Parallel()
|
||||
v := NewVisionTool("vision_understand", &fakeVision{desc: ""})
|
||||
if _, err := v.Execute(context.Background(), map[string]any{"prompt": "x"}); err == nil {
|
||||
t.Fatal("want error for missing image, got nil")
|
||||
}
|
||||
if _, err := v.Execute(context.Background(), "not-a-map"); err == nil {
|
||||
t.Fatal("want error for non-map args, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOCRTool_Execute(t *testing.T) {
|
||||
t.Parallel()
|
||||
o := NewOCRTool("ocr_extract", &fakeOCR{text: "HELLO"})
|
||||
res, err := o.Execute(context.Background(), map[string]any{"image": "aGVsbG8="})
|
||||
if err != nil {
|
||||
t.Fatalf("Execute error: %v", err)
|
||||
}
|
||||
m, ok := res.(map[string]any)
|
||||
if !ok || m["text"] != "HELLO" {
|
||||
t.Errorf("result = %v, want text=HELLO", res)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/pkg/plugins/sdk"
|
||||
)
|
||||
|
||||
// toolParameters returns the shared input JSON Schema for the vision and OCR
|
||||
// tools. It builds a fresh map each call rather than sharing a package-level
|
||||
// variable.
|
||||
func toolParameters() map[string]any {
|
||||
return map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"image": map[string]any{
|
||||
"type": "string",
|
||||
"description": "The image: a local path, an http(s) URL, or a base64/data: URI.",
|
||||
},
|
||||
"prompt": map[string]any{
|
||||
"type": "string",
|
||||
"description": "Optional instruction for the vision model (vision tool only).",
|
||||
},
|
||||
},
|
||||
"required": []any{"image"},
|
||||
}
|
||||
}
|
||||
|
||||
// VisionTool wraps a VisionAdapter as an sdk.Tool so the M2 tool loop can invoke
|
||||
// it and return its description as a tool result, keeping the image out of the
|
||||
// main-model context. It is safe for concurrent use.
|
||||
type VisionTool struct {
|
||||
name string
|
||||
adapter VisionAdapter
|
||||
}
|
||||
|
||||
// NewVisionTool builds a VisionTool with the given name and adapter.
|
||||
func NewVisionTool(name string, adapter VisionAdapter) *VisionTool {
|
||||
return &VisionTool{name: name, adapter: adapter}
|
||||
}
|
||||
|
||||
// Name returns the tool identifier the agent uses to invoke it.
|
||||
func (t *VisionTool) Name() string { return t.name }
|
||||
|
||||
// Description explains what the tool does.
|
||||
func (t *VisionTool) Description() string {
|
||||
return "Understand an image and return a description of its contents."
|
||||
}
|
||||
|
||||
// Parameters returns the input JSON Schema.
|
||||
func (t *VisionTool) Parameters() (map[string]any, error) {
|
||||
return toolParameters(), nil
|
||||
}
|
||||
|
||||
// Execute runs the adapter on the image from args and returns the description.
|
||||
func (t *VisionTool) Execute(ctx context.Context, args any) (any, error) {
|
||||
image, prompt, err := toolArgs(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ref, err := ParseImageRef(image)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
desc, err := t.adapter.Describe(ctx, ref, prompt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"description": desc}, nil
|
||||
}
|
||||
|
||||
// OCRTool wraps an OCRAdapter as an sdk.Tool. It returns the extracted text as a
|
||||
// tool result. It is safe for concurrent use.
|
||||
type OCRTool struct {
|
||||
name string
|
||||
adapter OCRAdapter
|
||||
}
|
||||
|
||||
// NewOCRTool builds an OCRTool with the given name and adapter.
|
||||
func NewOCRTool(name string, adapter OCRAdapter) *OCRTool {
|
||||
return &OCRTool{name: name, adapter: adapter}
|
||||
}
|
||||
|
||||
// Name returns the tool identifier.
|
||||
func (t *OCRTool) Name() string { return t.name }
|
||||
|
||||
// Description explains what the tool does.
|
||||
func (t *OCRTool) Description() string {
|
||||
return "Extract text from an image (OCR) and return the recognized text."
|
||||
}
|
||||
|
||||
// Parameters returns the input JSON Schema.
|
||||
func (t *OCRTool) Parameters() (map[string]any, error) {
|
||||
return toolParameters(), nil
|
||||
}
|
||||
|
||||
// Execute runs the OCR adapter on the image from args and returns the text.
|
||||
func (t *OCRTool) Execute(ctx context.Context, args any) (any, error) {
|
||||
image, _, err := toolArgs(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ref, err := ParseImageRef(image)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
text, err := t.adapter.Extract(ctx, ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"text": text}, nil
|
||||
}
|
||||
|
||||
// RegisterTools registers the vision and OCR tools with r via the SDK helper.
|
||||
// Each tool is only registered when its pointer is non-nil. It returns the
|
||||
// first registration error, or nil when every tool registered.
|
||||
func RegisterTools(r sdk.Registrar, tool *VisionTool, ocr *OCRTool) error {
|
||||
if r == nil {
|
||||
return sdk.ErrNilRegistrar
|
||||
}
|
||||
if tool != nil {
|
||||
if err := sdk.RegisterTool(r, tool); err != nil {
|
||||
return fmt.Errorf("vision: register tool %q: %w", tool.Name(), err)
|
||||
}
|
||||
}
|
||||
if ocr != nil {
|
||||
if err := sdk.RegisterTool(r, ocr); err != nil {
|
||||
return fmt.Errorf("vision: register tool %q: %w", ocr.Name(), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// toolArgs decodes sdk.Tool.Execute args into the image string and an optional
|
||||
// prompt. Args must be a map[string]any with a non-empty "image" string.
|
||||
func toolArgs(args any) (image, prompt string, err error) {
|
||||
m, ok := args.(map[string]any)
|
||||
if !ok {
|
||||
return "", "", fmt.Errorf("vision: %w: args must be an object", ErrBadArgs)
|
||||
}
|
||||
image, _ = m["image"].(string)
|
||||
if image == "" {
|
||||
return "", "", fmt.Errorf("vision: %w: missing image", ErrBadArgs)
|
||||
}
|
||||
prompt, _ = m["prompt"].(string)
|
||||
return image, prompt, nil
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// Package vision implements the layered image routing described in
|
||||
// DevDocs/design/06-多模态.md §0.
|
||||
//
|
||||
// A request that carries image content is first checked by DetectMultimodal.
|
||||
// The VisionRouter then picks one of the layered routes: the main model
|
||||
// directly (when it accepts images), a tool-shaped vision/OCR adapter (when the
|
||||
// main model self-assesses poorly or cannot see), or the OCR tool (when the
|
||||
// image is mainly text). Vision/OCR adapters are registered as sdk.Tool
|
||||
// capabilities through the capability registry, and their output is a tool
|
||||
// result, so the image itself never occupies main-model context.
|
||||
//
|
||||
// Images may come from a local path, an http(s) URL, or a base64/data: URI.
|
||||
package vision
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||
)
|
||||
|
||||
// DetectMultimodal reports whether msgs contains any non-text content part,
|
||||
// i.e. whether multimodal handling is required. Today model only distinguishes
|
||||
// text from image content, so this is true exactly when a message carries an
|
||||
// image part; the check is written against "not text" so future content types
|
||||
// (audio) extend it without rework.
|
||||
func DetectMultimodal(msgs []model.Message) bool {
|
||||
for _, m := range msgs {
|
||||
for _, part := range m.Content {
|
||||
if part.Type != model.ContentTypeText {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ImageSource classifies where an image's bytes come from.
|
||||
type ImageSource string
|
||||
|
||||
const (
|
||||
// SourcePath is a local filesystem path.
|
||||
SourcePath ImageSource = "path"
|
||||
// SourceURL is an http(s):// URL.
|
||||
SourceURL ImageSource = "url"
|
||||
// SourceBase64 is a base64 payload, optionally wrapped in a data: URI.
|
||||
SourceBase64 ImageSource = "base64"
|
||||
)
|
||||
|
||||
// ImageRef is a single image reference plus its resolved source kind.
|
||||
type ImageRef struct {
|
||||
// Source is the classified source kind.
|
||||
Source ImageSource
|
||||
// Data is the local path, URL, or base64/data: payload.
|
||||
Data string
|
||||
}
|
||||
|
||||
// ParseImageRef classifies a raw image source string into an ImageRef. Empty
|
||||
// input is rejected. Local paths that exist on disk are classified as paths;
|
||||
// http(s) URLs as URLs; every other value, including a data: URI, is treated as
|
||||
// a base64 payload. This keeps routing deterministic without guessing MIME.
|
||||
func ParseImageRef(src string) (ImageRef, error) {
|
||||
if src == "" {
|
||||
return ImageRef{}, ErrNoImage
|
||||
}
|
||||
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
|
||||
return ImageRef{Source: SourceURL, Data: src}, nil
|
||||
}
|
||||
if strings.HasPrefix(src, "file://") {
|
||||
return ImageRef{Source: SourcePath, Data: strings.TrimPrefix(src, "file://")}, nil
|
||||
}
|
||||
// A data: URI is base64 already; treat as base64, not path.
|
||||
if strings.HasPrefix(src, "data:") {
|
||||
return ImageRef{Source: SourceBase64, Data: src}, nil
|
||||
}
|
||||
if fileExists(src) {
|
||||
return ImageRef{Source: SourcePath, Data: src}, nil
|
||||
}
|
||||
return ImageRef{Source: SourceBase64, Data: src}, nil
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||
)
|
||||
|
||||
func TestDetectMultimodal(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
msgs []model.Message
|
||||
want bool
|
||||
}{
|
||||
{name: "text only", msgs: []model.Message{model.TextMessage(model.RoleUser, "hi")}, want: false},
|
||||
{name: "empty", msgs: nil, want: false},
|
||||
{name: "single image", msgs: []model.Message{model.ImageMessage(model.RoleUser, "what", "data:image/png;base64,abc")}, want: true},
|
||||
{name: "image no text", msgs: []model.Message{model.ImageMessage(model.RoleUser, "", "https://x/y.png")}, want: true},
|
||||
{name: "image in later turn", msgs: []model.Message{
|
||||
model.TextMessage(model.RoleUser, "hi"),
|
||||
model.ImageMessage(model.RoleAssistant, "", "http://x/z.png"),
|
||||
}, want: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := DetectMultimodal(tt.msgs); got != tt.want {
|
||||
t.Errorf("DetectMultimodal() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseImageRef(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
src string
|
||||
wantSource ImageSource
|
||||
}{
|
||||
{name: "http url", src: "https://a/b.png", wantSource: SourceURL},
|
||||
{name: "https url", src: "https://a/b.png", wantSource: SourceURL},
|
||||
{name: "file uri", src: "file:///tmp/a.png", wantSource: SourcePath},
|
||||
{name: "data uri", src: "data:image/png;base64,abc", wantSource: SourceBase64},
|
||||
{name: "raw base64", src: "aGVsbG8=", wantSource: SourceBase64},
|
||||
{name: "empty rejected", src: "", wantSource: ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
ref, err := ParseImageRef(tt.src)
|
||||
if tt.src == "" {
|
||||
if err == nil {
|
||||
t.Fatal("want error for empty source, got nil")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("ParseImageRef error: %v", err)
|
||||
}
|
||||
if ref.Source != tt.wantSource {
|
||||
t.Errorf("Source = %q, want %q", ref.Source, tt.wantSource)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve_Base64(t *testing.T) {
|
||||
t.Parallel()
|
||||
ref, err := ParseImageRef("data:image/png;base64,aGVsbG8=")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseImageRef error: %v", err)
|
||||
}
|
||||
mime, data, err := Resolve(context.Background(), ref, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
if mime != "image/png" {
|
||||
t.Errorf("mime = %q, want image/png", mime)
|
||||
}
|
||||
if string(data) != "hello" {
|
||||
t.Errorf("data = %q, want hello", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve_DataURI(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := DataURI("image/webp", []byte{1, 2, 3})
|
||||
if !strings.HasPrefix(got, "data:image/webp;base64,") {
|
||||
t.Errorf("DataURI = %q, want data:image/webp;base64,", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package voice
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||
)
|
||||
|
||||
// Chain is a degrading provider list. STT and TTS walk the steps in order,
|
||||
// returning the first successful result and moving to the next step when a
|
||||
// provider is unavailable (nil direction) or fails. It implements STT and TTS so
|
||||
// it can be used anywhere a single provider is expected.
|
||||
type Chain struct {
|
||||
steps []Provider
|
||||
}
|
||||
|
||||
// NewChain builds a degrading chain from the given providers in the order they
|
||||
// should be tried. Passing a SilentProvider last guarantees the chain always
|
||||
// succeeds.
|
||||
func NewChain(steps ...Provider) *Chain {
|
||||
return &Chain{steps: steps}
|
||||
}
|
||||
|
||||
// STT walks the chain for a transcription.
|
||||
func (c *Chain) STT(ctx context.Context, audio Audio) (string, error) {
|
||||
if len(c.steps) == 0 {
|
||||
return "", ErrProviderFailed
|
||||
}
|
||||
var lastErr error
|
||||
for _, p := range c.steps {
|
||||
stt := p.STT()
|
||||
if stt == nil {
|
||||
continue
|
||||
}
|
||||
text, err := stt.STT(ctx, audio)
|
||||
if err == nil {
|
||||
return text, nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return "", fmt.Errorf("voice: %w: %v", ErrProviderFailed, lastErr)
|
||||
}
|
||||
|
||||
// TTS walks the chain for synthesized audio.
|
||||
func (c *Chain) TTS(ctx context.Context, text string) (Audio, error) {
|
||||
if len(c.steps) == 0 {
|
||||
return Audio{}, ErrProviderFailed
|
||||
}
|
||||
var lastErr error
|
||||
for _, p := range c.steps {
|
||||
tts := p.TTS()
|
||||
if tts == nil {
|
||||
continue
|
||||
}
|
||||
audio, err := tts.TTS(ctx, text)
|
||||
if err == nil {
|
||||
return audio, nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return Audio{}, fmt.Errorf("voice: %w: %v", ErrProviderFailed, lastErr)
|
||||
}
|
||||
|
||||
// Option customizes a chain built by New.
|
||||
type Option func(*chainConfig)
|
||||
|
||||
// chainConfig carries the assembled steps for New.
|
||||
type chainConfig struct {
|
||||
steps []Provider
|
||||
}
|
||||
|
||||
// WithProviders overrides the provider steps assembled from config. Tests use it
|
||||
// to inject deterministic mock providers instead of the real adaptation-point
|
||||
// rungs.
|
||||
func WithProviders(steps ...Provider) Option {
|
||||
return func(c *chainConfig) { c.steps = steps }
|
||||
}
|
||||
|
||||
// New assembles a chain from cfg using the fixed degradation order
|
||||
// cloud → local → edge-tts/espeak → silent.
|
||||
//
|
||||
// - cloud is included only when a cloud voice setting is present (today an
|
||||
// API key) so a mis-configured build degrades quietly.
|
||||
// - local is a pure adaptation point and is only included when supplied via
|
||||
// WithProviders; by default it is skipped.
|
||||
// - the edge-tts or espeak rung is included according to cfg.TTSProvider.
|
||||
// - silent is always appended last so the chain never fails outright.
|
||||
//
|
||||
// opts may supply a fully assembled step list to override this mapping.
|
||||
func New(cfg config.VoiceConfig, opts ...Option) *Chain {
|
||||
conf := &chainConfig{}
|
||||
for _, o := range opts {
|
||||
o(conf)
|
||||
}
|
||||
if len(conf.steps) > 0 {
|
||||
return NewChain(conf.steps...)
|
||||
}
|
||||
|
||||
steps := make([]Provider, 0, 5)
|
||||
if cfg.DashscopeAPIKey != "" {
|
||||
steps = append(steps, CloudProvider(newDashboardSTT(cfg), newDashboardTTS(cfg)))
|
||||
}
|
||||
switch cfg.TTSProvider {
|
||||
case "edge-tts", "edge":
|
||||
steps = append(steps, EdgeTTSProvider(newEdgeTTS()))
|
||||
case "espeak":
|
||||
steps = append(steps, EspeakProvider(newEspeak()))
|
||||
}
|
||||
steps = append(steps, SilentProvider())
|
||||
return NewChain(steps...)
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package voice
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||
)
|
||||
|
||||
// stubSTT is a mock STT returning a fixed result.
|
||||
type stubSTT struct {
|
||||
text string
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *stubSTT) STT(_ context.Context, _ Audio) (string, error) { return s.text, s.err }
|
||||
|
||||
// stubTTS is a mock TTS returning a fixed result.
|
||||
type stubTTS struct {
|
||||
audio Audio
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *stubTTS) TTS(_ context.Context, _ string) (Audio, error) { return s.audio, s.err }
|
||||
|
||||
// stubProvider is a mock Provider carrying optional directions.
|
||||
type stubProvider struct {
|
||||
name string
|
||||
stt STT
|
||||
tts TTS
|
||||
}
|
||||
|
||||
func (s *stubProvider) Name() string { return s.name }
|
||||
func (s *stubProvider) STT() STT { return s.stt }
|
||||
func (s *stubProvider) TTS() TTS { return s.tts }
|
||||
|
||||
func TestChain_STT_Degrades(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
steps []Provider
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "first provider succeeds",
|
||||
steps: []Provider{&stubProvider{name: "cloud", stt: &stubSTT{text: "hello"}}},
|
||||
want: "hello",
|
||||
},
|
||||
{
|
||||
name: "first fails second succeeds",
|
||||
steps: []Provider{
|
||||
&stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}},
|
||||
&stubProvider{name: "local", stt: &stubSTT{text: "degraded"}},
|
||||
},
|
||||
want: "degraded",
|
||||
},
|
||||
{
|
||||
name: "nil direction skipped",
|
||||
steps: []Provider{
|
||||
&stubProvider{name: "edge", stt: nil},
|
||||
&stubProvider{name: "silent", stt: &stubSTT{text: "silence"}},
|
||||
},
|
||||
want: "silence",
|
||||
},
|
||||
{
|
||||
name: "all fail no silent -> error",
|
||||
steps: []Provider{
|
||||
&stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "all fail silent last -> no error",
|
||||
steps: []Provider{
|
||||
&stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}},
|
||||
SilentProvider(),
|
||||
},
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := NewChain(tt.steps...).STT(context.Background(), Audio{Format: "wav", Data: []byte("x")})
|
||||
if tt.wantErr {
|
||||
if !errors.Is(err, ErrProviderFailed) {
|
||||
t.Fatalf("err = %v, want ErrProviderFailed", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("STT error: %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("text = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChain_TTS_Degrades(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
steps []Provider
|
||||
want Audio
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "first provider succeeds",
|
||||
steps: []Provider{&stubProvider{name: "cloud", tts: &stubTTS{audio: Audio{Format: "mp3", Data: []byte("a")}}}},
|
||||
want: Audio{Format: "mp3", Data: []byte("a")},
|
||||
},
|
||||
{
|
||||
name: "first fails second succeeds",
|
||||
steps: []Provider{
|
||||
&stubProvider{name: "cloud", tts: &stubTTS{err: ErrNotConfigured}},
|
||||
&stubProvider{name: "edge-tts", tts: &stubTTS{audio: Audio{Format: "mp3", Data: []byte("b")}}},
|
||||
},
|
||||
want: Audio{Format: "mp3", Data: []byte("b")},
|
||||
},
|
||||
{
|
||||
name: "all fail no silent -> error",
|
||||
steps: []Provider{
|
||||
&stubProvider{name: "cloud", tts: &stubTTS{err: ErrNotConfigured}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := NewChain(tt.steps...).TTS(context.Background(), "hi")
|
||||
if tt.wantErr {
|
||||
if !errors.Is(err, ErrProviderFailed) {
|
||||
t.Fatalf("err = %v, want ErrProviderFailed", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("TTS error: %v", err)
|
||||
}
|
||||
if got.Format != tt.want.Format || string(got.Data) != string(tt.want.Data) {
|
||||
t.Errorf("audio = %+v, want %+v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChain_Empty(t *testing.T) {
|
||||
t.Parallel()
|
||||
if _, err := NewChain().STT(context.Background(), Audio{}); !errors.Is(err, ErrProviderFailed) {
|
||||
t.Fatalf("err = %v, want ErrProviderFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew_ConfigAssembly(t *testing.T) {
|
||||
t.Parallel()
|
||||
// With a cloud key and default edge-tts TTS, the chain starts at cloud and
|
||||
// ends at silent.
|
||||
c := New(config.VoiceConfig{DashscopeAPIKey: "k"})
|
||||
names := make([]string, 0, len(c.steps))
|
||||
for _, p := range c.steps {
|
||||
names = append(names, p.Name())
|
||||
}
|
||||
if len(names) < 2 || names[len(names)-1] != "silent" {
|
||||
t.Fatalf("steps = %v, want to end at silent", names)
|
||||
}
|
||||
if names[0] != "cloud" {
|
||||
t.Errorf("first step = %q, want cloud", names[0])
|
||||
}
|
||||
|
||||
// Without a key the cloud rung is skipped.
|
||||
c2 := New(config.VoiceConfig{})
|
||||
foundCloud := false
|
||||
for _, p := range c2.steps {
|
||||
if p.Name() == "cloud" {
|
||||
foundCloud = true
|
||||
}
|
||||
}
|
||||
if foundCloud {
|
||||
t.Error("cloud step present without an API key")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
package voice
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||
)
|
||||
|
||||
// cloudMaxResponseBytes caps how much of a cloud voice response is read, so an
|
||||
// oversized or malformed response cannot exhaust memory. The request timeout
|
||||
// bounds a single call.
|
||||
const (
|
||||
cloudMaxResponseBytes = 8 << 20
|
||||
cloudHTTPTimeout = 180 * time.Second
|
||||
)
|
||||
|
||||
// HTTPSTT transcribes audio by POSTing a JSON body to a cloud endpoint and
|
||||
// reading {"text": "..."} back. It is the stubbed default STT implementation;
|
||||
// the real DashScope/whisper-facing adapter is an adaptation point.
|
||||
type HTTPSTT struct {
|
||||
endpoint string
|
||||
apiKey string
|
||||
client *http.Client
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// HTTPTTS synthesizes speech by POSTing a JSON body and reading the returned
|
||||
// base64 audio. It is the stubbed default TTS implementation.
|
||||
type HTTPTTS struct {
|
||||
endpoint string
|
||||
apiKey string
|
||||
client *http.Client
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// HTTPOption customizes the client and logger of an HTTPSTT/HTTPTTS.
|
||||
type HTTPOption func(*httpConfig)
|
||||
|
||||
type httpConfig struct {
|
||||
client *http.Client
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// WithHTTPClient sets the *http.Client used for the call; nil keeps the default.
|
||||
func WithHTTPClient(c *http.Client) HTTPOption {
|
||||
return func(o *httpConfig) {
|
||||
if c != nil {
|
||||
o.client = c
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithLogger sets the logger used on failures; nil keeps a discard logger.
|
||||
func WithLogger(l *slog.Logger) HTTPOption {
|
||||
return func(o *httpConfig) {
|
||||
if l != nil {
|
||||
o.log = l
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewHTTPSTT builds an HTTPSTT for the given endpoint and API key.
|
||||
func NewHTTPSTT(endpoint, apiKey string, opts ...HTTPOption) *HTTPSTT {
|
||||
h := &httpConfig{client: defaultHTTPClient(), log: discardLogger()}
|
||||
for _, o := range opts {
|
||||
o(h)
|
||||
}
|
||||
return &HTTPSTT{endpoint: strings.TrimRight(endpoint, "/"), apiKey: apiKey, client: h.client, log: h.log}
|
||||
}
|
||||
|
||||
// NewHTTPTTS builds an HTTPTTS for the given endpoint and API key.
|
||||
func NewHTTPTTS(endpoint, apiKey string, opts ...HTTPOption) *HTTPTTS {
|
||||
h := &httpConfig{client: defaultHTTPClient(), log: discardLogger()}
|
||||
for _, o := range opts {
|
||||
o(h)
|
||||
}
|
||||
return &HTTPTTS{endpoint: strings.TrimRight(endpoint, "/"), apiKey: apiKey, client: h.client, log: h.log}
|
||||
}
|
||||
|
||||
// sttRequest is the JSON body sent to the STT endpoint.
|
||||
type sttRequest struct {
|
||||
Audio string `json:"audio"`
|
||||
Format string `json:"format"`
|
||||
}
|
||||
|
||||
// sttResponse is the transcription returned by the STT endpoint.
|
||||
type sttResponse struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// STT posts the audio and returns the recognized text. A blank endpoint returns
|
||||
// ErrNotConfigured so the chain degrades.
|
||||
func (s *HTTPSTT) STT(ctx context.Context, audio Audio) (string, error) {
|
||||
if s.endpoint == "" {
|
||||
return "", ErrNotConfigured
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
payload, err := json.Marshal(sttRequest{Audio: base64.StdEncoding.EncodeToString(audio.Data), Format: audio.Format})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err)
|
||||
}
|
||||
resp, err := postJSON(ctx, s.client, s.apiKey, s.endpoint, payload)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
var decoded sttResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
|
||||
return "", fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err)
|
||||
}
|
||||
return decoded.Text, nil
|
||||
}
|
||||
|
||||
// ttsRequest is the JSON body sent to the TTS endpoint.
|
||||
type ttsRequest struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// ttsResponse is the synthesized audio returned by the TTS endpoint.
|
||||
type ttsResponse struct {
|
||||
Audio string `json:"audio"`
|
||||
Format string `json:"format"`
|
||||
}
|
||||
|
||||
// TTS posts the text and returns the decoded audio. A blank endpoint returns
|
||||
// ErrNotConfigured so the chain degrades.
|
||||
func (s *HTTPTTS) TTS(ctx context.Context, text string) (Audio, error) {
|
||||
if s.endpoint == "" {
|
||||
return Audio{}, ErrNotConfigured
|
||||
}
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return Audio{}, ErrEmptyText
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Audio{}, err
|
||||
}
|
||||
payload, err := json.Marshal(ttsRequest{Text: text})
|
||||
if err != nil {
|
||||
return Audio{}, fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err)
|
||||
}
|
||||
resp, err := postJSON(ctx, s.client, s.apiKey, s.endpoint, payload)
|
||||
if err != nil {
|
||||
return Audio{}, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
var decoded ttsResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
|
||||
return Audio{}, fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err)
|
||||
}
|
||||
data, err := base64.StdEncoding.DecodeString(decoded.Audio)
|
||||
if err != nil {
|
||||
return Audio{}, fmt.Errorf("voice: %w: decode audio: %v", ErrProviderFailed, err)
|
||||
}
|
||||
format := decoded.Format
|
||||
if format == "" {
|
||||
format = "mp3"
|
||||
}
|
||||
return Audio{Format: format, Data: data}, nil
|
||||
}
|
||||
|
||||
// postJSON POSTs a JSON payload, wires auth, and checks the status code.
|
||||
func postJSON(ctx context.Context, client *http.Client, apiKey, url string, payload []byte) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("voice: %w: build: %v", ErrProviderFailed, err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("voice: %w: request: %v", ErrProviderFailed, err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_ = resp.Body.Close()
|
||||
return nil, fmt.Errorf("voice: %w: status %d", ErrProviderFailed, resp.StatusCode)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// defaultHTTPClient builds a client with a request timeout. A fresh client is
|
||||
// returned per call rather than a package global, which the constraints forbid.
|
||||
func defaultHTTPClient() *http.Client {
|
||||
return &http.Client{Transport: http.DefaultTransport, Timeout: cloudHTTPTimeout}
|
||||
}
|
||||
|
||||
// discardLogger returns a logger that drops all output.
|
||||
func discardLogger() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
}
|
||||
|
||||
// newDashboardSTT returns the cloud STT for a config. The endpoint is an
|
||||
// adaptation point and is empty until config gains a voice URL; a call then
|
||||
// returns ErrNotConfigured and the chain degrades.
|
||||
func newDashboardSTT(cfg config.VoiceConfig) STT {
|
||||
return NewHTTPSTT("", cfg.DashscopeAPIKey)
|
||||
}
|
||||
|
||||
// newDashboardTTS returns the cloud TTS for a config.
|
||||
func newDashboardTTS(cfg config.VoiceConfig) TTS {
|
||||
return NewHTTPTTS("", cfg.DashscopeAPIKey)
|
||||
}
|
||||
|
||||
// newEdgeTTS returns the edge-tts adapter point (not yet implemented).
|
||||
func newEdgeTTS() TTS { return notImplementedTTS{} }
|
||||
|
||||
// newEspeak returns the espeak adapter point (not yet implemented).
|
||||
func newEspeak() TTS { return notImplementedTTS{} }
|
||||
|
||||
// notImplementedTTS is an adaptation point that always degrades.
|
||||
type notImplementedTTS struct{}
|
||||
|
||||
func (notImplementedTTS) TTS(_ context.Context, _ string) (Audio, error) {
|
||||
return Audio{}, ErrNotImplemented
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package voice
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHTTPSTT(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if r.Header.Get("Authorization") != "Bearer sk" {
|
||||
t.Errorf("auth = %q, want Bearer sk", r.Header.Get("Authorization"))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"text":"hello world"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
stt := NewHTTPSTT(srv.URL, "sk")
|
||||
text, err := stt.STT(context.Background(), Audio{Format: "wav", Data: []byte("raw")})
|
||||
if err != nil {
|
||||
t.Fatalf("STT error: %v", err)
|
||||
}
|
||||
if text != "hello world" {
|
||||
t.Errorf("text = %q, want hello world", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPSTT_NotConfigured(t *testing.T) {
|
||||
t.Parallel()
|
||||
stt := NewHTTPSTT("", "")
|
||||
if _, err := stt.STT(context.Background(), Audio{}); err != ErrNotConfigured {
|
||||
t.Fatalf("err = %v, want ErrNotConfigured", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPTTS(t *testing.T) {
|
||||
t.Parallel()
|
||||
audioBytes := []byte{1, 2, 3}
|
||||
b64 := base64.StdEncoding.EncodeToString(audioBytes)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if body["text"] != "hi" {
|
||||
t.Errorf("text = %v, want hi", body["text"])
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"audio":"` + b64 + `","format":"wav"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tts := NewHTTPTTS(srv.URL, "")
|
||||
audio, err := tts.TTS(context.Background(), "hi")
|
||||
if err != nil {
|
||||
t.Fatalf("TTS error: %v", err)
|
||||
}
|
||||
if audio.Format != "wav" || hex.EncodeToString(audio.Data) != hex.EncodeToString(audioBytes) {
|
||||
t.Errorf("audio = %+v, want wav/010203", audio)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPTTS_EmptyText(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
defer srv.Close()
|
||||
tts := NewHTTPTTS(srv.URL, "")
|
||||
if _, err := tts.TTS(context.Background(), " "); err != ErrEmptyText {
|
||||
t.Fatalf("err = %v, want ErrEmptyText", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package voice
|
||||
|
||||
import "errors"
|
||||
|
||||
// Sentinel errors returned by the voice package. Each is wrapped with %w so
|
||||
// callers can test for a category while keeping the cause.
|
||||
var (
|
||||
// ErrProviderFailed is returned when every provider in a chain fails.
|
||||
ErrProviderFailed = errors.New("voice: all providers failed")
|
||||
// ErrNotConfigured is returned when a provider has no endpoint/setting.
|
||||
ErrNotConfigured = errors.New("voice: provider not configured")
|
||||
// ErrNotImplemented is returned by an adaptation-point provider.
|
||||
ErrNotImplemented = errors.New("voice: provider not implemented")
|
||||
// ErrEmptyText is returned when TTS receives empty text.
|
||||
ErrEmptyText = errors.New("voice: empty text")
|
||||
// ErrClosed is returned when a service call arrives after Close.
|
||||
ErrClosed = errors.New("voice: service closed")
|
||||
)
|
||||
@@ -0,0 +1,79 @@
|
||||
package voice
|
||||
|
||||
import "context"
|
||||
|
||||
// Provider is one rung in the voice degradation chain. It may implement zero,
|
||||
// one, or both directions; a nil return means that direction is not available
|
||||
// on this rung and the chain moves to the next.
|
||||
type Provider interface {
|
||||
// Name returns a stable provider identifier.
|
||||
Name() string
|
||||
// STT returns the speech-to-text implementation, or nil when the provider
|
||||
// cannot transcribe.
|
||||
STT() STT
|
||||
// TTS returns the text-to-speech implementation, or nil when the provider
|
||||
// cannot synthesize.
|
||||
TTS() TTS
|
||||
}
|
||||
|
||||
// StaticProvider is a Provider with a fixed name whose STT/TTS are set at
|
||||
// construction. It is used by the concrete providers below.
|
||||
type StaticProvider struct {
|
||||
name string
|
||||
stt STT
|
||||
tts TTS
|
||||
}
|
||||
|
||||
// Name returns the provider identifier.
|
||||
func (p *StaticProvider) Name() string { return p.name }
|
||||
|
||||
// STT returns the configured STT, or nil.
|
||||
func (p *StaticProvider) STT() STT { return p.stt }
|
||||
|
||||
// TTS returns the configured TTS, or nil.
|
||||
func (p *StaticProvider) TTS() TTS { return p.tts }
|
||||
|
||||
// NewProvider builds a Provider with the given name and optional directions it
|
||||
// supports.
|
||||
func NewProvider(name string, stt STT, tts TTS) *StaticProvider {
|
||||
return &StaticProvider{name: name, stt: stt, tts: tts}
|
||||
}
|
||||
|
||||
// CloudProvider is the cloud rung of the chain. It is an adaptation point: the
|
||||
// real STT/TTS implementations are wired in via NewProvider with cloud adapters.
|
||||
func CloudProvider(stt STT, tts TTS) *StaticProvider {
|
||||
return NewProvider("cloud", stt, tts)
|
||||
}
|
||||
|
||||
// LocalProvider is the local-model rung of the chain.
|
||||
func LocalProvider(stt STT, tts TTS) *StaticProvider {
|
||||
return NewProvider("local", stt, tts)
|
||||
}
|
||||
|
||||
// EdgeTTSProvider is the edge-tts rung (text-to-speech only).
|
||||
func EdgeTTSProvider(tts TTS) *StaticProvider {
|
||||
return NewProvider("edge-tts", nil, tts)
|
||||
}
|
||||
|
||||
// EspeakProvider is the espeak rung (text-to-speech only).
|
||||
func EspeakProvider(tts TTS) *StaticProvider {
|
||||
return NewProvider("espeak", nil, tts)
|
||||
}
|
||||
|
||||
// SilentProvider is the terminal rung. It always succeeds, returning an empty
|
||||
// transcription and empty audio, so a chain that includes it never errors.
|
||||
func SilentProvider() *StaticProvider {
|
||||
return NewProvider("silent", silenceSTT{}, silenceTTS{})
|
||||
}
|
||||
|
||||
// silenceSTT is the silent STT rung; it returns an empty transcription.
|
||||
type silenceSTT struct{}
|
||||
|
||||
func (silenceSTT) STT(_ context.Context, _ Audio) (string, error) { return "", nil }
|
||||
|
||||
// silenceTTS is the silent TTS rung; it returns empty audio.
|
||||
type silenceTTS struct{}
|
||||
|
||||
func (silenceTTS) TTS(_ context.Context, _ string) (Audio, error) {
|
||||
return Audio{}, nil
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package voice
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Service runs STT/TTS requests on a background worker pool so caller
|
||||
// goroutines (the agent loop) are never blocked by provider latency. Each
|
||||
// request is enqueued on a jobs channel and answered on a per-job result
|
||||
// channel; a canceled caller context or a closed service unblocks the caller
|
||||
// immediately and no worker goroutine is leaked. It is safe for concurrent use.
|
||||
type Service struct {
|
||||
chain *Chain
|
||||
jobs chan job
|
||||
bgCtx context.Context
|
||||
cancel context.CancelFunc
|
||||
nWorkers int
|
||||
wg sync.WaitGroup
|
||||
mu sync.RWMutex
|
||||
closed bool
|
||||
}
|
||||
|
||||
// ServiceOption customizes a Service.
|
||||
type ServiceOption func(*Service)
|
||||
|
||||
// WithWorkers sets the worker-pool size. Values <= 0 are ignored and the
|
||||
// default single worker is kept.
|
||||
func WithWorkers(n int) ServiceOption {
|
||||
return func(s *Service) {
|
||||
if n > 0 {
|
||||
s.nWorkers = n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// jobKind selects the operation a worker performs.
|
||||
type jobKind int
|
||||
|
||||
const (
|
||||
jobSTT jobKind = iota
|
||||
jobTTS
|
||||
)
|
||||
|
||||
// job is one request enqueued by the caller.
|
||||
type job struct {
|
||||
kind jobKind
|
||||
audio Audio
|
||||
text string
|
||||
res chan jobResult
|
||||
}
|
||||
|
||||
// jobResult carries the outcome written back to the caller.
|
||||
type jobResult struct {
|
||||
text string
|
||||
audio Audio
|
||||
err error
|
||||
}
|
||||
|
||||
// NewService builds a Service backed by chain and starts the worker pool.
|
||||
func NewService(chain *Chain, opts ...ServiceOption) *Service {
|
||||
bg, cancel := context.WithCancel(context.Background())
|
||||
s := &Service{
|
||||
chain: chain,
|
||||
jobs: make(chan job),
|
||||
bgCtx: bg,
|
||||
cancel: cancel,
|
||||
nWorkers: 1,
|
||||
}
|
||||
for _, o := range opts {
|
||||
o(s)
|
||||
}
|
||||
for i := 0; i < s.nWorkers; i++ {
|
||||
s.wg.Add(1)
|
||||
go s.worker()
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// worker consumes jobs until the service closes.
|
||||
func (s *Service) worker() {
|
||||
defer s.wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-s.bgCtx.Done():
|
||||
return
|
||||
case j := <-s.jobs:
|
||||
s.run(j)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// run executes a job and delivers the result. Providers run under the service's
|
||||
// background context so Close cancels in-flight work and no worker goroutine is
|
||||
// leaked; the result is written to a buffered channel so a caller that already
|
||||
// returned never blocks the worker.
|
||||
func (s *Service) run(j job) {
|
||||
res := jobResult{}
|
||||
switch j.kind {
|
||||
case jobSTT:
|
||||
res.text, res.err = s.chain.STT(s.bgCtx, j.audio)
|
||||
case jobTTS:
|
||||
res.audio, res.err = s.chain.TTS(s.bgCtx, j.text)
|
||||
}
|
||||
j.res <- res
|
||||
}
|
||||
|
||||
// STT enqueues a transcription request and waits for the result. ctx cancels
|
||||
// the wait and the underlying provider call.
|
||||
func (s *Service) STT(ctx context.Context, audio Audio) (string, error) {
|
||||
res, err := s.submit(ctx, job{kind: jobSTT, audio: audio, res: make(chan jobResult, 1)})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return res.text, res.err
|
||||
}
|
||||
|
||||
// TTS enqueues a synthesis request and waits for the result. ctx cancels the
|
||||
// wait and the underlying provider call.
|
||||
func (s *Service) TTS(ctx context.Context, text string) (Audio, error) {
|
||||
res, err := s.submit(ctx, job{kind: jobTTS, text: text, res: make(chan jobResult, 1)})
|
||||
if err != nil {
|
||||
return Audio{}, err
|
||||
}
|
||||
return res.audio, res.err
|
||||
}
|
||||
|
||||
// submit enqueues j and blocks until the result or a cancellation.
|
||||
func (s *Service) submit(ctx context.Context, j job) (jobResult, error) {
|
||||
select {
|
||||
case s.jobs <- j:
|
||||
case <-ctx.Done():
|
||||
return jobResult{}, fmt.Errorf("voice: %w", ctx.Err())
|
||||
case <-s.bgCtx.Done():
|
||||
return jobResult{}, ErrClosed
|
||||
}
|
||||
select {
|
||||
case r := <-j.res:
|
||||
return r, nil
|
||||
case <-ctx.Done():
|
||||
return jobResult{}, fmt.Errorf("voice: %w", ctx.Err())
|
||||
case <-s.bgCtx.Done():
|
||||
return jobResult{}, ErrClosed
|
||||
}
|
||||
}
|
||||
|
||||
// Close cancels the worker pool and waits for in-flight jobs to finish. It is
|
||||
// safe to call once; later calls are no-ops. In-flight provider calls are
|
||||
// bounded by each provider's own context timeout, so Close returns promptly.
|
||||
func (s *Service) Close() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return
|
||||
}
|
||||
s.closed = true
|
||||
s.cancel()
|
||||
s.wg.Wait()
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package voice
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestService_STTAndTTS(t *testing.T) {
|
||||
t.Parallel()
|
||||
chain := NewChain(
|
||||
NewProvider("cloud", &stubSTT{text: "transcribed"}, &stubTTS{audio: Audio{Format: "mp3", Data: []byte("a")}}),
|
||||
)
|
||||
svc := NewService(chain)
|
||||
defer svc.Close()
|
||||
|
||||
text, err := svc.STT(context.Background(), Audio{Format: "wav", Data: []byte("x")})
|
||||
if err != nil {
|
||||
t.Fatalf("STT error: %v", err)
|
||||
}
|
||||
if text != "transcribed" {
|
||||
t.Errorf("text = %q, want transcribed", text)
|
||||
}
|
||||
|
||||
audio, err := svc.TTS(context.Background(), "hello")
|
||||
if err != nil {
|
||||
t.Fatalf("TTS error: %v", err)
|
||||
}
|
||||
if string(audio.Data) != "a" || audio.Format != "mp3" {
|
||||
t.Errorf("audio = %+v, want mp3/a", audio)
|
||||
}
|
||||
}
|
||||
|
||||
func TestService_CloseStopsNewCalls(t *testing.T) {
|
||||
t.Parallel()
|
||||
svc := NewService(NewChain(SilentProvider()))
|
||||
svc.Close()
|
||||
_, err := svc.STT(context.Background(), Audio{})
|
||||
if !errors.Is(err, ErrClosed) {
|
||||
t.Fatalf("err = %v, want ErrClosed", err)
|
||||
}
|
||||
}
|
||||
|
||||
// blockingSTT blocks until its context is canceled, marking that it started.
|
||||
type blockingSTT struct {
|
||||
started chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (b *blockingSTT) STT(ctx context.Context, _ Audio) (string, error) {
|
||||
b.once.Do(func() { close(b.started) })
|
||||
<-ctx.Done()
|
||||
return "", ctx.Err()
|
||||
}
|
||||
|
||||
func TestService_CloseCancelsInFlightWorker(t *testing.T) {
|
||||
t.Parallel()
|
||||
blk := &blockingSTT{started: make(chan struct{})}
|
||||
chain := NewChain(NewProvider("cloud", blk, nil))
|
||||
svc := NewService(chain)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
_, _ = svc.STT(context.Background(), Audio{Format: "wav", Data: []byte("x")})
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-blk.started:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("provider never started")
|
||||
}
|
||||
|
||||
svc.Close()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("STT call did not return after Close; worker likely leaked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestService_CallerCancelUnblocks(t *testing.T) {
|
||||
t.Parallel()
|
||||
blk := &blockingSTT{started: make(chan struct{})}
|
||||
svc := NewService(NewChain(NewProvider("cloud", blk, nil)))
|
||||
defer svc.Close()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
_, _ = svc.STT(ctx, Audio{Format: "wav", Data: []byte("x")})
|
||||
}()
|
||||
|
||||
<-blk.started
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("caller cancellation did not unblock STT")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Package voice implements asynchronous speech-to-text and text-to-speech for
|
||||
// the Cyrene runtime, following DevDocs/design/06-多模态.md §1.
|
||||
//
|
||||
// Voice input (STT) is transcribed to text and fed into the same agent pipeline
|
||||
// as a chat message; the reply is synthesized (TTS) back to audio. Both
|
||||
// directions degrade through a provider chain — cloud → local → edge-tts/espeak
|
||||
// → silent — so an unavailable provider falls through to the next rung instead
|
||||
// of failing the whole turn. The stub default implementations are HTTP-based and
|
||||
// testable with a stdlib httptest server; real cloud/local providers remain
|
||||
// adaptation points for later milestones.
|
||||
package voice
|
||||
|
||||
import "context"
|
||||
|
||||
// Audio is a piece of audio data plus its encoding.
|
||||
type Audio struct {
|
||||
// Format is the encoding, e.g. "wav", "mp3", "ogg".
|
||||
Format string
|
||||
// Data is the raw audio bytes.
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// STT converts audio to text.
|
||||
type STT interface {
|
||||
// STT transcribes audio and returns the spoken text. ctx controls
|
||||
// cancellation and must be honored.
|
||||
STT(ctx context.Context, audio Audio) (string, error)
|
||||
}
|
||||
|
||||
// TTS synthesizes speech audio from text.
|
||||
type TTS interface {
|
||||
// TTS synthesizes audio from text. ctx controls cancellation and must be
|
||||
// honored.
|
||||
TTS(ctx context.Context, text string) (Audio, error)
|
||||
}
|
||||
Reference in new issue
Block a user