feat(multimodal): 视觉分层路由(多模态检测→主/视觉/OCR 工具)+ 语音 STT→TTS 降级链(异步)
This commit is contained in:
1 parent
02aae2bbb4
commit
0a936a3202
19 files changed
+2121
No files matched your search
@@ -0,0 +1,17 @@
|
|||||||
|
// Package multimodal groups the runtime's non-text input/output handling:
|
||||||
|
// image understanding (vision) and speech (voice).
|
||||||
|
//
|
||||||
|
// It follows DevDocs/design/06-多模态.md:
|
||||||
|
//
|
||||||
|
// - The vision subpackage routes image messages through a layered strategy
|
||||||
|
// (main model first, then a tool-shaped vision/OCR adapter) and registers
|
||||||
|
// those adapters as sdk.Tool capabilities so the M2 tool loop can invoke
|
||||||
|
// them without putting the image in the main-model context.
|
||||||
|
// - The voice subpackage provides asynchronous STT/TTS through a degrading
|
||||||
|
// provider chain (cloud → local → edge-tts/espeak → silent).
|
||||||
|
//
|
||||||
|
// Both subpackages keep real provider calls behind interfaces with an HTTP
|
||||||
|
// default implementation, so they are testable with a stdlib httptest server
|
||||||
|
// and leave the real cloud/local providers as adaptation points for later
|
||||||
|
// milestones.
|
||||||
|
package multimodal
|
||||||
@@ -0,0 +1,217 @@
|
|||||||
|
package vision
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// visionMaxTokens caps the vision/OCR response length.
|
||||||
|
const visionMaxTokens = 512
|
||||||
|
|
||||||
|
// VisionAdapter understands an image and returns a description of it. It is the
|
||||||
|
// backend behind the vision tool; real providers implement it without exposing
|
||||||
|
// the image to the main-model context.
|
||||||
|
type VisionAdapter interface {
|
||||||
|
// Describe returns a description of img. prompt is an optional instruction
|
||||||
|
// overriding the default "describe this image" ask.
|
||||||
|
Describe(ctx context.Context, img ImageRef, prompt string) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// OCRAdapter extracts text from an image. It is the backend behind the OCR tool.
|
||||||
|
type OCRAdapter interface {
|
||||||
|
// Extract returns the text found in img.
|
||||||
|
Extract(ctx context.Context, img ImageRef) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenAICompatibleAdapter implements both VisionAdapter and OCRAdapter by
|
||||||
|
// calling an OpenAI-compatible /chat/completions endpoint with the image
|
||||||
|
// embedded as a content part. It is the ready default implementation; real
|
||||||
|
// cloud vision/OCR providers expose the same protocol.
|
||||||
|
type OpenAICompatibleAdapter struct {
|
||||||
|
cfg config.LLMConfig
|
||||||
|
model string
|
||||||
|
client *http.Client
|
||||||
|
log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenAIOption customizes an OpenAICompatibleAdapter.
|
||||||
|
type OpenAIOption func(*OpenAICompatibleAdapter)
|
||||||
|
|
||||||
|
// WithModel overrides the adapter model (defaults to cfg.Model).
|
||||||
|
func WithModel(m string) OpenAIOption {
|
||||||
|
return func(a *OpenAICompatibleAdapter) {
|
||||||
|
if m != "" {
|
||||||
|
a.model = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithHTTPClient sets the *http.Client used for the request; nil keeps default.
|
||||||
|
func WithHTTPClient(c *http.Client) OpenAIOption {
|
||||||
|
return func(a *OpenAICompatibleAdapter) {
|
||||||
|
if c != nil {
|
||||||
|
a.client = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithLogger sets the logger used on failures; nil keeps a discard logger.
|
||||||
|
func WithLogger(l *slog.Logger) OpenAIOption {
|
||||||
|
return func(a *OpenAICompatibleAdapter) {
|
||||||
|
if l != nil {
|
||||||
|
a.log = l
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewOpenAICompatible builds an adapter from cfg, using cfg.Model unless
|
||||||
|
// overridden.
|
||||||
|
func NewOpenAICompatible(cfg config.LLMConfig, opts ...OpenAIOption) *OpenAICompatibleAdapter {
|
||||||
|
a := &OpenAICompatibleAdapter{
|
||||||
|
cfg: cfg,
|
||||||
|
model: cfg.Model,
|
||||||
|
client: defaultHTTPClient(),
|
||||||
|
log: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||||
|
}
|
||||||
|
for _, o := range opts {
|
||||||
|
o(a)
|
||||||
|
}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
// Describe returns an image description from the underlying chat endpoint.
|
||||||
|
func (a *OpenAICompatibleAdapter) Describe(ctx context.Context, img ImageRef, prompt string) (string, error) {
|
||||||
|
if prompt == "" {
|
||||||
|
prompt = "Describe this image in detail."
|
||||||
|
}
|
||||||
|
return a.ask(ctx, img, prompt)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract returns text found in the image via the same endpoint.
|
||||||
|
func (a *OpenAICompatibleAdapter) Extract(ctx context.Context, img ImageRef) (string, error) {
|
||||||
|
return a.ask(ctx, img, "Extract all text visible in this image. Return only the text.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ask resolves the image, builds a chat request and returns the assistant text.
|
||||||
|
func (a *OpenAICompatibleAdapter) ask(ctx context.Context, img ImageRef, prompt string) (string, error) {
|
||||||
|
if strings.TrimSpace(a.cfg.BaseURL) == "" {
|
||||||
|
return "", ErrNotConfigured
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(a.model) == "" {
|
||||||
|
return "", ErrNotConfigured
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
imageURL, err := a.imageURL(ctx, img)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(a.request(imageURL, prompt))
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("vision: %w: marshal request: %v", ErrUpstream, err)
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(a.cfg.BaseURL, "/")+"/chat/completions", bytes.NewReader(payload))
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("vision: %w: build request: %v", ErrUpstream, err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
if a.cfg.APIKey != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+a.cfg.APIKey)
|
||||||
|
}
|
||||||
|
resp, err := a.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("vision: %w: %v", ErrUpstream, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
snippet, _ := io.ReadAll(io.LimitReader(resp.Body, maxFetchedBody))
|
||||||
|
return "", fmt.Errorf("vision: %w: status %d: %s", ErrUpstream, resp.StatusCode, strings.TrimSpace(string(snippet)))
|
||||||
|
}
|
||||||
|
return a.decode(resp.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// imageURL converts an ImageRef to the image_url value the wire expects: an
|
||||||
|
// http(s) URL passes through, anything else becomes a base64 data: URI.
|
||||||
|
func (a *OpenAICompatibleAdapter) imageURL(ctx context.Context, img ImageRef) (string, error) {
|
||||||
|
if img.Source == SourceURL {
|
||||||
|
return img.Data, nil
|
||||||
|
}
|
||||||
|
mime, data, err := Resolve(ctx, img, a.client)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return DataURI(mime, data), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// openAIVisionRequest is the JSON body sent to /chat/completions.
|
||||||
|
type openAIVisionRequest struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Messages []openAIMessage `json:"messages"`
|
||||||
|
MaxTokens int `json:"max_tokens"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIMessage struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Content any `json:"content"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIPart struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Text string `json:"text,omitempty"`
|
||||||
|
ImageURL *openAIImageURL `json:"image_url,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIImageURL struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIVisionResponse struct {
|
||||||
|
Choices []openAIChoice `json:"choices"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIChoice struct {
|
||||||
|
Message openAIResponseMessage `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openAIResponseMessage struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// request builds the wire body for an image+prompt.
|
||||||
|
func (a *OpenAICompatibleAdapter) request(imageURL, prompt string) openAIVisionRequest {
|
||||||
|
return openAIVisionRequest{
|
||||||
|
Model: a.model,
|
||||||
|
Messages: []openAIMessage{
|
||||||
|
{Role: "user", Content: []openAIPart{
|
||||||
|
{Type: "text", Text: prompt},
|
||||||
|
{Type: "image_url", ImageURL: &openAIImageURL{URL: imageURL}},
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
MaxTokens: visionMaxTokens,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// decode extracts the first choice's text from a response body.
|
||||||
|
func (a *OpenAICompatibleAdapter) decode(r io.Reader) (string, error) {
|
||||||
|
var wire openAIVisionResponse
|
||||||
|
if err := json.NewDecoder(r).Decode(&wire); err != nil {
|
||||||
|
return "", fmt.Errorf("vision: %w: decode response: %v", ErrUpstream, err)
|
||||||
|
}
|
||||||
|
if len(wire.Choices) == 0 {
|
||||||
|
return "", fmt.Errorf("vision: %w: empty choices", ErrUpstream)
|
||||||
|
}
|
||||||
|
return wire.Choices[0].Message.Content, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// interface assertion: one concrete adapter serves both directions.
|
||||||
|
var _ VisionAdapter = (*OpenAICompatibleAdapter)(nil)
|
||||||
|
var _ OCRAdapter = (*OpenAICompatibleAdapter)(nil)
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package vision
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOpenAICompatibleAdapter_Describe(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/chat/completions" {
|
||||||
|
t.Errorf("path = %s, want /chat/completions", r.URL.Path)
|
||||||
|
}
|
||||||
|
var body map[string]any
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||||
|
t.Fatalf("decode body: %v", err)
|
||||||
|
}
|
||||||
|
msgs, _ := body["messages"].([]any)
|
||||||
|
if len(msgs) != 1 {
|
||||||
|
t.Fatalf("messages = %d, want 1", len(msgs))
|
||||||
|
}
|
||||||
|
content, _ := msgs[0].(map[string]any)["content"].([]any)
|
||||||
|
if len(content) != 2 {
|
||||||
|
t.Fatalf("content parts = %d, want 2", len(content))
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"a red car"}}]}`)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
ad := NewOpenAICompatible(config.LLMConfig{BaseURL: srv.URL, Model: "vision-"})
|
||||||
|
text, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: "aGVsbG8="}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Describe error: %v", err)
|
||||||
|
}
|
||||||
|
if text != "a red car" {
|
||||||
|
t.Errorf("text = %q, want a red car", text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleAdapter_NotConfigured(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
ad := NewOpenAICompatible(config.LLMConfig{})
|
||||||
|
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: "aGVsbG8="}, ""); err == nil {
|
||||||
|
t.Fatal("want error for empty base url, got nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package vision
|
||||||
|
|
||||||
|
import "errors"
|
||||||
|
|
||||||
|
// Sentinel errors returned by the vision package. Each is wrapped with %w on
|
||||||
|
// the way out so callers can test for a category while keeping the cause.
|
||||||
|
var (
|
||||||
|
// ErrNoImage is returned when an image source string is empty.
|
||||||
|
ErrNoImage = errors.New("vision: empty image source")
|
||||||
|
// ErrNotConfigured is returned by adapters when no endpoint/model is set.
|
||||||
|
ErrNotConfigured = errors.New("vision: adapter not configured")
|
||||||
|
// ErrUnsupportedSource is returned when an image source cannot be handled.
|
||||||
|
ErrUnsupportedSource = errors.New("vision: unsupported image source")
|
||||||
|
// ErrFetch is returned when a URL image cannot be downloaded.
|
||||||
|
ErrFetch = errors.New("vision: cannot fetch image")
|
||||||
|
// ErrUpstream is returned when the vision/OCR provider request fails.
|
||||||
|
ErrUpstream = errors.New("vision: upstream request failed")
|
||||||
|
// ErrBadImage is returned when an image cannot be read or decoded.
|
||||||
|
ErrBadImage = errors.New("vision: cannot read image")
|
||||||
|
// ErrBadArgs is returned by a tool when its arguments are malformed.
|
||||||
|
ErrBadArgs = errors.New("vision: bad tool arguments")
|
||||||
|
)
|
||||||
@@ -0,0 +1,178 @@
|
|||||||
|
package vision
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Size caps for reading image bytes and response bodies, so a hostile or
|
||||||
|
// malformed source cannot exhaust memory. The 50MB cap matches the file-upload
|
||||||
|
// limit in DevDocs/02-development-constraints.md §11.
|
||||||
|
const (
|
||||||
|
maxImageBytes = 50 << 20
|
||||||
|
maxFetchedBody = 50 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
// DataURI builds a base64 data: image URI from mime and raw bytes.
|
||||||
|
func DataURI(mime string, data []byte) string {
|
||||||
|
return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve loads the bytes for an ImageRef. Paths and base64 payloads are read
|
||||||
|
// locally; URLs are fetched with an http.Client (the provided client, or a
|
||||||
|
// default one when nil). For base64 input the Data field is returned verbatim as
|
||||||
|
// the payload and the MIME type is guessed from a data: prefix.
|
||||||
|
func Resolve(ctx context.Context, ref ImageRef, client *http.Client) (mime string, data []byte, err error) {
|
||||||
|
switch ref.Source {
|
||||||
|
case SourcePath:
|
||||||
|
return readFile(ref.Data)
|
||||||
|
case SourceBase64:
|
||||||
|
return decodeBase64(ref.Data)
|
||||||
|
case SourceURL:
|
||||||
|
return fetchURL(ctx, ref.Data, client)
|
||||||
|
default:
|
||||||
|
return "", nil, fmt.Errorf("vision: %w: %q", ErrUnsupportedSource, ref.Source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readFile reads an image file and guesses its MIME type from the extension.
|
||||||
|
func readFile(path string) (string, []byte, error) {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = f.Close() }()
|
||||||
|
data, err := io.ReadAll(io.LimitReader(f, maxImageBytes))
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
|
||||||
|
}
|
||||||
|
return mimeByExt(path), data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeBase64 returns the decoded bytes of a base64/data: payload and its MIME.
|
||||||
|
func decodeBase64(src string) (string, []byte, error) {
|
||||||
|
mime, encoded, isDataURI := splitDataURI(src)
|
||||||
|
if isDataURI {
|
||||||
|
decoded, err := base64.StdEncoding.DecodeString(encoded)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
|
||||||
|
}
|
||||||
|
return mime, decoded, nil
|
||||||
|
}
|
||||||
|
decoded, err := base64.StdEncoding.DecodeString(src)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
|
||||||
|
}
|
||||||
|
return "image/png", decoded, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchURL downloads an image over http(s); only http/https schemes are allowed.
|
||||||
|
func fetchURL(ctx context.Context, rawURL string, client *http.Client) (string, []byte, error) {
|
||||||
|
if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
|
||||||
|
return "", nil, fmt.Errorf("vision: %w: %q", ErrUnsupportedSource, rawURL)
|
||||||
|
}
|
||||||
|
if client == nil {
|
||||||
|
client = defaultHTTPClient()
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err)
|
||||||
|
}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return "", nil, fmt.Errorf("vision: %w: status %d", ErrFetch, resp.StatusCode)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(io.LimitReader(resp.Body, maxFetchedBody))
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err)
|
||||||
|
}
|
||||||
|
return mimeByContentType(resp.Header.Get("Content-Type")), data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitDataURI parses a "data:<mime>;base64,<payload>" prefix. The second return
|
||||||
|
// is false when src is not a data: URI.
|
||||||
|
func splitDataURI(src string) (mime, payload string, ok bool) {
|
||||||
|
if !strings.HasPrefix(src, "data:") {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
rest := strings.TrimPrefix(src, "data:")
|
||||||
|
comma := strings.IndexByte(rest, ',')
|
||||||
|
if comma < 0 {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
meta := rest[:comma]
|
||||||
|
payload = rest[comma+1:]
|
||||||
|
mime = "image/png"
|
||||||
|
if strings.HasPrefix(meta, ";base64") {
|
||||||
|
return mime, payload, true
|
||||||
|
}
|
||||||
|
if idx := strings.Index(meta, ";base64"); idx >= 0 {
|
||||||
|
if t := strings.TrimSuffix(meta[:idx], ";base64"); t != "" {
|
||||||
|
mime = t
|
||||||
|
}
|
||||||
|
return mime, payload, true
|
||||||
|
}
|
||||||
|
return mime, payload, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// mimeByExt guesses a MIME type from a file path extension.
|
||||||
|
func mimeByExt(path string) string {
|
||||||
|
switch strings.ToLower(pathExtension(path)) {
|
||||||
|
case ".png":
|
||||||
|
return "image/png"
|
||||||
|
case ".jpg", ".jpeg":
|
||||||
|
return "image/jpeg"
|
||||||
|
case ".gif":
|
||||||
|
return "image/gif"
|
||||||
|
case ".webp":
|
||||||
|
return "image/webp"
|
||||||
|
default:
|
||||||
|
return "image/png"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mimeByContentType returns a MIME type from an HTTP Content-Type header,
|
||||||
|
// defaulting to image/png when absent or non-image.
|
||||||
|
func mimeByContentType(ct string) string {
|
||||||
|
if ct == "" || !strings.HasPrefix(ct, "image/") {
|
||||||
|
return "image/png"
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(strings.Split(ct, ";")[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
// pathExtension returns the file extension with its dot, or empty when none.
|
||||||
|
func pathExtension(path string) string {
|
||||||
|
base := path
|
||||||
|
if idx := strings.LastIndexByte(path, '/'); idx >= 0 {
|
||||||
|
base = path[idx+1:]
|
||||||
|
} else if idx := strings.LastIndexByte(path, '\\'); idx >= 0 {
|
||||||
|
base = path[idx+1:]
|
||||||
|
}
|
||||||
|
dot := strings.LastIndexByte(base, '.')
|
||||||
|
if dot < 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return base[dot:]
|
||||||
|
}
|
||||||
|
|
||||||
|
// fileExists reports whether path exists on disk and is a regular file.
|
||||||
|
func fileExists(path string) bool {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
return err == nil && info.Mode().IsRegular()
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultHTTPClient builds the HTTP client described in
|
||||||
|
// DevDocs/02-development-constraints.md §2.3. A fresh client is returned per
|
||||||
|
// call rather than a package global, which the constraints forbid.
|
||||||
|
func defaultHTTPClient() *http.Client {
|
||||||
|
return &http.Client{Transport: http.DefaultTransport}
|
||||||
|
}
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
package vision
|
||||||
|
|
||||||
|
// Assessment is the model's own confidence in reading an image.
|
||||||
|
type Assessment string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// AssessmentUnknown means the model reported no self-assessment.
|
||||||
|
AssessmentUnknown Assessment = "unknown"
|
||||||
|
// AssessmentGood means the model is confident in its read.
|
||||||
|
AssessmentGood Assessment = "good"
|
||||||
|
// AssessmentPoor means the model judged its recognition as poor.
|
||||||
|
AssessmentPoor Assessment = "poor"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ModelInfo describes the main model's vision capability.
|
||||||
|
type ModelInfo struct {
|
||||||
|
// Name identifies the model.
|
||||||
|
Name string
|
||||||
|
// SupportsVision reports whether the model accepts image input.
|
||||||
|
SupportsVision bool
|
||||||
|
// SelfAssessment is the model's own confidence for the current image.
|
||||||
|
// It drives the layer-2 tool route when SupportsVision is true.
|
||||||
|
SelfAssessment Assessment
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecisionKind is the chosen layered route.
|
||||||
|
type DecisionKind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// KindMain routes the image to the main model with the image in context.
|
||||||
|
KindMain DecisionKind = "main"
|
||||||
|
// KindVisionTool routes to a dedicated vision tool, returning only the tool
|
||||||
|
// result so the image never occupies main-model context.
|
||||||
|
KindVisionTool DecisionKind = "vision_tool"
|
||||||
|
// KindOCRTool routes to an OCR tool, returning only the extracted text.
|
||||||
|
KindOCRTool DecisionKind = "ocr_tool"
|
||||||
|
// KindUnsupported means no route can process the image.
|
||||||
|
KindUnsupported DecisionKind = "unsupported"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Decision is the outcome of a routing call.
|
||||||
|
type Decision struct {
|
||||||
|
// Kind is the chosen route.
|
||||||
|
Kind DecisionKind
|
||||||
|
// Tool names the vision or OCR tool to invoke, set when Kind is a tool route.
|
||||||
|
Tool string
|
||||||
|
// Reason is a human-readable explanation for observability and logs.
|
||||||
|
Reason string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RouteRequest carries routing hints from the caller on top of the model info.
|
||||||
|
type RouteRequest struct {
|
||||||
|
// TextPrimary reports that the user indicated the image is mainly text, so
|
||||||
|
// an OCR tool is preferred over a general vision model.
|
||||||
|
TextPrimary bool
|
||||||
|
// MainSelfAssessedPoor is set when the main model has already judged its own
|
||||||
|
// read of the image as poor, requesting a vision/OCR tool route.
|
||||||
|
MainSelfAssessedPoor bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// RouterOption configures a Router.
|
||||||
|
type RouterOption func(*Router)
|
||||||
|
|
||||||
|
// WithVisionTool binds the name of the registered vision-understanding tool.
|
||||||
|
func WithVisionTool(name string) RouterOption {
|
||||||
|
return func(r *Router) {
|
||||||
|
if name != "" {
|
||||||
|
r.visionTool = name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithOCRTool binds the name of the registered OCR tool.
|
||||||
|
func WithOCRTool(name string) RouterOption {
|
||||||
|
return func(r *Router) {
|
||||||
|
if name != "" {
|
||||||
|
r.ocrTool = name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Router implements the layered vision routing policy. It does no I/O; the
|
||||||
|
// caller uses the returned Decision to dispatch the request.
|
||||||
|
type Router struct {
|
||||||
|
visionTool string
|
||||||
|
ocrTool string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRouter builds a Router. Unknown (empty) tool names leave the corresponding
|
||||||
|
// route unavailable, so the router degrades to KindMain or KindUnsupported.
|
||||||
|
func NewRouter(opts ...RouterOption) *Router {
|
||||||
|
r := &Router{}
|
||||||
|
for _, o := range opts {
|
||||||
|
o(r)
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// Route picks a layered route for an image request given the target model info
|
||||||
|
// and caller hints. The order is: the main model directly when it can see and
|
||||||
|
// has not self-assessed poorly (layer 1); otherwise a vision or OCR tool whose
|
||||||
|
// result replaces the image in context (layers 2 and 3); otherwise
|
||||||
|
// KindUnsupported when nothing can process the image.
|
||||||
|
func (r *Router) Route(target ModelInfo, req RouteRequest) Decision {
|
||||||
|
poor := target.SelfAssessment == AssessmentPoor || req.MainSelfAssessedPoor
|
||||||
|
// Layer 1: the main model accepts images and is confident, so use it first.
|
||||||
|
if target.SupportsVision && !poor {
|
||||||
|
return Decision{
|
||||||
|
Kind: KindMain,
|
||||||
|
Reason: "main model supports vision and is confident; use it directly",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// OCR is preferred when the user says the image is mostly text.
|
||||||
|
if req.TextPrimary && r.ocrTool != "" {
|
||||||
|
return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "image is mainly text; route to OCR"}
|
||||||
|
}
|
||||||
|
if r.visionTool != "" {
|
||||||
|
return Decision{Kind: KindVisionTool, Tool: r.visionTool, Reason: "route to dedicated vision tool"}
|
||||||
|
}
|
||||||
|
if r.ocrTool != "" {
|
||||||
|
return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "route to OCR tool"}
|
||||||
|
}
|
||||||
|
// No tool is bound. A sighted main model remains the only option.
|
||||||
|
if target.SupportsVision {
|
||||||
|
return Decision{Kind: KindMain, Reason: "no vision/OCR tool bound; keep image with main model"}
|
||||||
|
}
|
||||||
|
return Decision{Kind: KindUnsupported, Reason: "main model cannot see and no vision/OCR tool is bound"}
|
||||||
|
}
|
||||||
@@ -0,0 +1,178 @@
|
|||||||
|
package vision
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/capability/registry"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRouter_Route(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
router *Router
|
||||||
|
target ModelInfo
|
||||||
|
req RouteRequest
|
||||||
|
want DecisionKind
|
||||||
|
tool string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "sighted confident main -> main",
|
||||||
|
router: NewRouter(WithVisionTool("v"), WithOCRTool("o")),
|
||||||
|
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentGood},
|
||||||
|
want: KindMain,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "poor self-assessment sighted main -> vision tool",
|
||||||
|
router: NewRouter(WithVisionTool("v"), WithOCRTool("o")),
|
||||||
|
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor},
|
||||||
|
req: RouteRequest{MainSelfAssessedPoor: true},
|
||||||
|
want: KindVisionTool,
|
||||||
|
tool: "v",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "poor self-assessment only OCR -> OCR tool",
|
||||||
|
router: NewRouter(WithOCRTool("o")),
|
||||||
|
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor},
|
||||||
|
want: KindOCRTool,
|
||||||
|
tool: "o",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blind main with vision tool -> vision tool",
|
||||||
|
router: NewRouter(WithVisionTool("v")),
|
||||||
|
target: ModelInfo{Name: "m", SupportsVision: false},
|
||||||
|
want: KindVisionTool,
|
||||||
|
tool: "v",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blind main text primary -> OCR tool",
|
||||||
|
router: NewRouter(WithVisionTool("v"), WithOCRTool("o")),
|
||||||
|
target: ModelInfo{Name: "m", SupportsVision: false},
|
||||||
|
req: RouteRequest{TextPrimary: true},
|
||||||
|
want: KindOCRTool,
|
||||||
|
tool: "o",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blind main only OCR non-text -> OCR fallback",
|
||||||
|
router: NewRouter(WithOCRTool("o")),
|
||||||
|
target: ModelInfo{Name: "m", SupportsVision: false},
|
||||||
|
want: KindOCRTool,
|
||||||
|
tool: "o",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blind main no tools -> unsupported",
|
||||||
|
router: NewRouter(),
|
||||||
|
target: ModelInfo{Name: "m", SupportsVision: false},
|
||||||
|
want: KindUnsupported,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "sighted main no tools -> main",
|
||||||
|
router: NewRouter(),
|
||||||
|
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentGood},
|
||||||
|
want: KindMain,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "sighted main poor no tools -> main fallback",
|
||||||
|
router: NewRouter(),
|
||||||
|
target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor},
|
||||||
|
want: KindMain,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
got := tc.router.Route(tc.target, tc.req)
|
||||||
|
if got.Kind != tc.want {
|
||||||
|
t.Fatalf("Kind = %q, want %q (reason: %s)", got.Kind, tc.want, got.Reason)
|
||||||
|
}
|
||||||
|
if tc.tool != "" && got.Tool != tc.tool {
|
||||||
|
t.Errorf("Tool = %q, want %q", got.Tool, tc.tool)
|
||||||
|
}
|
||||||
|
if got.Reason == "" {
|
||||||
|
t.Error("Reason must be non-empty")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeVision implements VisionAdapter for tool tests.
|
||||||
|
type fakeVision struct{ desc string }
|
||||||
|
|
||||||
|
func (f *fakeVision) Describe(_ context.Context, _ ImageRef, _ string) (string, error) {
|
||||||
|
return f.desc, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeOCR implements OCRAdapter for tool tests.
|
||||||
|
type fakeOCR struct{ text string }
|
||||||
|
|
||||||
|
func (f *fakeOCR) Extract(_ context.Context, _ ImageRef) (string, error) {
|
||||||
|
return f.text, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegisterTools(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
reg := registry.New()
|
||||||
|
v := NewVisionTool("vision_understand", &fakeVision{desc: "a cat"})
|
||||||
|
o := NewOCRTool("ocr_extract", &fakeOCR{text: "HELLO"})
|
||||||
|
|
||||||
|
if err := RegisterTools(reg, v, o); err != nil {
|
||||||
|
t.Fatalf("RegisterTools error: %v", err)
|
||||||
|
}
|
||||||
|
if capv, err := reg.Get("vision_understand"); err != nil || capv.Kind != registry.KindTool {
|
||||||
|
t.Fatalf("Get(vision_understand) = %+v, %v", capv, err)
|
||||||
|
}
|
||||||
|
if capo, err := reg.Get("ocr_extract"); err != nil || capo.Kind != registry.KindTool {
|
||||||
|
t.Fatalf("Get(ocr_extract) = %+v, %v", capo, err)
|
||||||
|
}
|
||||||
|
if len(reg.List(registry.KindTool)) != 2 {
|
||||||
|
t.Errorf("List(tool) = %d, want 2", len(reg.List(registry.KindTool)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegisterTools_NilRegistrar(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if err := RegisterTools(nil, nil, nil); err == nil {
|
||||||
|
t.Fatal("want error for nil registrar, got nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVisionTool_Execute(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
v := NewVisionTool("vision_understand", &fakeVision{desc: "a cat"})
|
||||||
|
res, err := v.Execute(context.Background(), map[string]any{
|
||||||
|
"image": "data:image/png;base64,aGVsbG8=",
|
||||||
|
"prompt": "what is this",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Execute error: %v", err)
|
||||||
|
}
|
||||||
|
m, ok := res.(map[string]any)
|
||||||
|
if !ok || m["description"] != "a cat" {
|
||||||
|
t.Errorf("result = %v, want description=a cat", res)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVisionTool_Execute_BadArgs(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
v := NewVisionTool("vision_understand", &fakeVision{desc: ""})
|
||||||
|
if _, err := v.Execute(context.Background(), map[string]any{"prompt": "x"}); err == nil {
|
||||||
|
t.Fatal("want error for missing image, got nil")
|
||||||
|
}
|
||||||
|
if _, err := v.Execute(context.Background(), "not-a-map"); err == nil {
|
||||||
|
t.Fatal("want error for non-map args, got nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOCRTool_Execute(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
o := NewOCRTool("ocr_extract", &fakeOCR{text: "HELLO"})
|
||||||
|
res, err := o.Execute(context.Background(), map[string]any{"image": "aGVsbG8="})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Execute error: %v", err)
|
||||||
|
}
|
||||||
|
m, ok := res.(map[string]any)
|
||||||
|
if !ok || m["text"] != "HELLO" {
|
||||||
|
t.Errorf("result = %v, want text=HELLO", res)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
package vision
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.yeij.top/AskaEth/Cyrene_v2/pkg/plugins/sdk"
|
||||||
|
)
|
||||||
|
|
||||||
|
// toolParameters returns the shared input JSON Schema for the vision and OCR
|
||||||
|
// tools. It builds a fresh map each call rather than sharing a package-level
|
||||||
|
// variable.
|
||||||
|
func toolParameters() map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]any{
|
||||||
|
"image": map[string]any{
|
||||||
|
"type": "string",
|
||||||
|
"description": "The image: a local path, an http(s) URL, or a base64/data: URI.",
|
||||||
|
},
|
||||||
|
"prompt": map[string]any{
|
||||||
|
"type": "string",
|
||||||
|
"description": "Optional instruction for the vision model (vision tool only).",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"required": []any{"image"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// VisionTool wraps a VisionAdapter as an sdk.Tool so the M2 tool loop can invoke
|
||||||
|
// it and return its description as a tool result, keeping the image out of the
|
||||||
|
// main-model context. It is safe for concurrent use.
|
||||||
|
type VisionTool struct {
|
||||||
|
name string
|
||||||
|
adapter VisionAdapter
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewVisionTool builds a VisionTool with the given name and adapter.
|
||||||
|
func NewVisionTool(name string, adapter VisionAdapter) *VisionTool {
|
||||||
|
return &VisionTool{name: name, adapter: adapter}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the tool identifier the agent uses to invoke it.
|
||||||
|
func (t *VisionTool) Name() string { return t.name }
|
||||||
|
|
||||||
|
// Description explains what the tool does.
|
||||||
|
func (t *VisionTool) Description() string {
|
||||||
|
return "Understand an image and return a description of its contents."
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parameters returns the input JSON Schema.
|
||||||
|
func (t *VisionTool) Parameters() (map[string]any, error) {
|
||||||
|
return toolParameters(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute runs the adapter on the image from args and returns the description.
|
||||||
|
func (t *VisionTool) Execute(ctx context.Context, args any) (any, error) {
|
||||||
|
image, prompt, err := toolArgs(args)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
ref, err := ParseImageRef(image)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
desc, err := t.adapter.Describe(ctx, ref, prompt)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]any{"description": desc}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// OCRTool wraps an OCRAdapter as an sdk.Tool. It returns the extracted text as a
|
||||||
|
// tool result. It is safe for concurrent use.
|
||||||
|
type OCRTool struct {
|
||||||
|
name string
|
||||||
|
adapter OCRAdapter
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewOCRTool builds an OCRTool with the given name and adapter.
|
||||||
|
func NewOCRTool(name string, adapter OCRAdapter) *OCRTool {
|
||||||
|
return &OCRTool{name: name, adapter: adapter}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the tool identifier.
|
||||||
|
func (t *OCRTool) Name() string { return t.name }
|
||||||
|
|
||||||
|
// Description explains what the tool does.
|
||||||
|
func (t *OCRTool) Description() string {
|
||||||
|
return "Extract text from an image (OCR) and return the recognized text."
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parameters returns the input JSON Schema.
|
||||||
|
func (t *OCRTool) Parameters() (map[string]any, error) {
|
||||||
|
return toolParameters(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute runs the OCR adapter on the image from args and returns the text.
|
||||||
|
func (t *OCRTool) Execute(ctx context.Context, args any) (any, error) {
|
||||||
|
image, _, err := toolArgs(args)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
ref, err := ParseImageRef(image)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
text, err := t.adapter.Extract(ctx, ref)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]any{"text": text}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RegisterTools registers the vision and OCR tools with r via the SDK helper.
|
||||||
|
// Each tool is only registered when its pointer is non-nil. It returns the
|
||||||
|
// first registration error, or nil when every tool registered.
|
||||||
|
func RegisterTools(r sdk.Registrar, tool *VisionTool, ocr *OCRTool) error {
|
||||||
|
if r == nil {
|
||||||
|
return sdk.ErrNilRegistrar
|
||||||
|
}
|
||||||
|
if tool != nil {
|
||||||
|
if err := sdk.RegisterTool(r, tool); err != nil {
|
||||||
|
return fmt.Errorf("vision: register tool %q: %w", tool.Name(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ocr != nil {
|
||||||
|
if err := sdk.RegisterTool(r, ocr); err != nil {
|
||||||
|
return fmt.Errorf("vision: register tool %q: %w", ocr.Name(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// toolArgs decodes sdk.Tool.Execute args into the image string and an optional
|
||||||
|
// prompt. Args must be a map[string]any with a non-empty "image" string.
|
||||||
|
func toolArgs(args any) (image, prompt string, err error) {
|
||||||
|
m, ok := args.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return "", "", fmt.Errorf("vision: %w: args must be an object", ErrBadArgs)
|
||||||
|
}
|
||||||
|
image, _ = m["image"].(string)
|
||||||
|
if image == "" {
|
||||||
|
return "", "", fmt.Errorf("vision: %w: missing image", ErrBadArgs)
|
||||||
|
}
|
||||||
|
prompt, _ = m["prompt"].(string)
|
||||||
|
return image, prompt, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
// Package vision implements the layered image routing described in
|
||||||
|
// DevDocs/design/06-多模态.md §0.
|
||||||
|
//
|
||||||
|
// A request that carries image content is first checked by DetectMultimodal.
|
||||||
|
// The VisionRouter then picks one of the layered routes: the main model
|
||||||
|
// directly (when it accepts images), a tool-shaped vision/OCR adapter (when the
|
||||||
|
// main model self-assesses poorly or cannot see), or the OCR tool (when the
|
||||||
|
// image is mainly text). Vision/OCR adapters are registered as sdk.Tool
|
||||||
|
// capabilities through the capability registry, and their output is a tool
|
||||||
|
// result, so the image itself never occupies main-model context.
|
||||||
|
//
|
||||||
|
// Images may come from a local path, an http(s) URL, or a base64/data: URI.
|
||||||
|
package vision
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DetectMultimodal reports whether msgs contains any non-text content part,
|
||||||
|
// i.e. whether multimodal handling is required. Today model only distinguishes
|
||||||
|
// text from image content, so this is true exactly when a message carries an
|
||||||
|
// image part; the check is written against "not text" so future content types
|
||||||
|
// (audio) extend it without rework.
|
||||||
|
func DetectMultimodal(msgs []model.Message) bool {
|
||||||
|
for _, m := range msgs {
|
||||||
|
for _, part := range m.Content {
|
||||||
|
if part.Type != model.ContentTypeText {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ImageSource classifies where an image's bytes come from.
|
||||||
|
type ImageSource string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// SourcePath is a local filesystem path.
|
||||||
|
SourcePath ImageSource = "path"
|
||||||
|
// SourceURL is an http(s):// URL.
|
||||||
|
SourceURL ImageSource = "url"
|
||||||
|
// SourceBase64 is a base64 payload, optionally wrapped in a data: URI.
|
||||||
|
SourceBase64 ImageSource = "base64"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ImageRef is a single image reference plus its resolved source kind.
|
||||||
|
type ImageRef struct {
|
||||||
|
// Source is the classified source kind.
|
||||||
|
Source ImageSource
|
||||||
|
// Data is the local path, URL, or base64/data: payload.
|
||||||
|
Data string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseImageRef classifies a raw image source string into an ImageRef. Empty
|
||||||
|
// input is rejected. Local paths that exist on disk are classified as paths;
|
||||||
|
// http(s) URLs as URLs; every other value, including a data: URI, is treated as
|
||||||
|
// a base64 payload. This keeps routing deterministic without guessing MIME.
|
||||||
|
func ParseImageRef(src string) (ImageRef, error) {
|
||||||
|
if src == "" {
|
||||||
|
return ImageRef{}, ErrNoImage
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
|
||||||
|
return ImageRef{Source: SourceURL, Data: src}, nil
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(src, "file://") {
|
||||||
|
return ImageRef{Source: SourcePath, Data: strings.TrimPrefix(src, "file://")}, nil
|
||||||
|
}
|
||||||
|
// A data: URI is base64 already; treat as base64, not path.
|
||||||
|
if strings.HasPrefix(src, "data:") {
|
||||||
|
return ImageRef{Source: SourceBase64, Data: src}, nil
|
||||||
|
}
|
||||||
|
if fileExists(src) {
|
||||||
|
return ImageRef{Source: SourcePath, Data: src}, nil
|
||||||
|
}
|
||||||
|
return ImageRef{Source: SourceBase64, Data: src}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
package vision
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDetectMultimodal(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
msgs []model.Message
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{name: "text only", msgs: []model.Message{model.TextMessage(model.RoleUser, "hi")}, want: false},
|
||||||
|
{name: "empty", msgs: nil, want: false},
|
||||||
|
{name: "single image", msgs: []model.Message{model.ImageMessage(model.RoleUser, "what", "data:image/png;base64,abc")}, want: true},
|
||||||
|
{name: "image no text", msgs: []model.Message{model.ImageMessage(model.RoleUser, "", "https://x/y.png")}, want: true},
|
||||||
|
{name: "image in later turn", msgs: []model.Message{
|
||||||
|
model.TextMessage(model.RoleUser, "hi"),
|
||||||
|
model.ImageMessage(model.RoleAssistant, "", "http://x/z.png"),
|
||||||
|
}, want: true},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if got := DetectMultimodal(tt.msgs); got != tt.want {
|
||||||
|
t.Errorf("DetectMultimodal() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseImageRef(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
src string
|
||||||
|
wantSource ImageSource
|
||||||
|
}{
|
||||||
|
{name: "http url", src: "https://a/b.png", wantSource: SourceURL},
|
||||||
|
{name: "https url", src: "https://a/b.png", wantSource: SourceURL},
|
||||||
|
{name: "file uri", src: "file:///tmp/a.png", wantSource: SourcePath},
|
||||||
|
{name: "data uri", src: "data:image/png;base64,abc", wantSource: SourceBase64},
|
||||||
|
{name: "raw base64", src: "aGVsbG8=", wantSource: SourceBase64},
|
||||||
|
{name: "empty rejected", src: "", wantSource: ""},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
ref, err := ParseImageRef(tt.src)
|
||||||
|
if tt.src == "" {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("want error for empty source, got nil")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseImageRef error: %v", err)
|
||||||
|
}
|
||||||
|
if ref.Source != tt.wantSource {
|
||||||
|
t.Errorf("Source = %q, want %q", ref.Source, tt.wantSource)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolve_Base64(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
ref, err := ParseImageRef("data:image/png;base64,aGVsbG8=")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseImageRef error: %v", err)
|
||||||
|
}
|
||||||
|
mime, data, err := Resolve(context.Background(), ref, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Resolve error: %v", err)
|
||||||
|
}
|
||||||
|
if mime != "image/png" {
|
||||||
|
t.Errorf("mime = %q, want image/png", mime)
|
||||||
|
}
|
||||||
|
if string(data) != "hello" {
|
||||||
|
t.Errorf("data = %q, want hello", data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolve_DataURI(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
got := DataURI("image/webp", []byte{1, 2, 3})
|
||||||
|
if !strings.HasPrefix(got, "data:image/webp;base64,") {
|
||||||
|
t.Errorf("DataURI = %q, want data:image/webp;base64,", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
package voice
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Chain is a degrading provider list. STT and TTS walk the steps in order,
|
||||||
|
// returning the first successful result and moving to the next step when a
|
||||||
|
// provider is unavailable (nil direction) or fails. It implements STT and TTS so
|
||||||
|
// it can be used anywhere a single provider is expected.
|
||||||
|
type Chain struct {
|
||||||
|
steps []Provider
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewChain builds a degrading chain from the given providers in the order they
|
||||||
|
// should be tried. Passing a SilentProvider last guarantees the chain always
|
||||||
|
// succeeds.
|
||||||
|
func NewChain(steps ...Provider) *Chain {
|
||||||
|
return &Chain{steps: steps}
|
||||||
|
}
|
||||||
|
|
||||||
|
// STT walks the chain for a transcription.
|
||||||
|
func (c *Chain) STT(ctx context.Context, audio Audio) (string, error) {
|
||||||
|
if len(c.steps) == 0 {
|
||||||
|
return "", ErrProviderFailed
|
||||||
|
}
|
||||||
|
var lastErr error
|
||||||
|
for _, p := range c.steps {
|
||||||
|
stt := p.STT()
|
||||||
|
if stt == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
text, err := stt.STT(ctx, audio)
|
||||||
|
if err == nil {
|
||||||
|
return text, nil
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("voice: %w: %v", ErrProviderFailed, lastErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TTS walks the chain for synthesized audio.
|
||||||
|
func (c *Chain) TTS(ctx context.Context, text string) (Audio, error) {
|
||||||
|
if len(c.steps) == 0 {
|
||||||
|
return Audio{}, ErrProviderFailed
|
||||||
|
}
|
||||||
|
var lastErr error
|
||||||
|
for _, p := range c.steps {
|
||||||
|
tts := p.TTS()
|
||||||
|
if tts == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
audio, err := tts.TTS(ctx, text)
|
||||||
|
if err == nil {
|
||||||
|
return audio, nil
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
}
|
||||||
|
return Audio{}, fmt.Errorf("voice: %w: %v", ErrProviderFailed, lastErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Option customizes a chain built by New.
|
||||||
|
type Option func(*chainConfig)
|
||||||
|
|
||||||
|
// chainConfig carries the assembled steps for New.
|
||||||
|
type chainConfig struct {
|
||||||
|
steps []Provider
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithProviders overrides the provider steps assembled from config. Tests use it
|
||||||
|
// to inject deterministic mock providers instead of the real adaptation-point
|
||||||
|
// rungs.
|
||||||
|
func WithProviders(steps ...Provider) Option {
|
||||||
|
return func(c *chainConfig) { c.steps = steps }
|
||||||
|
}
|
||||||
|
|
||||||
|
// New assembles a chain from cfg using the fixed degradation order
|
||||||
|
// cloud → local → edge-tts/espeak → silent.
|
||||||
|
//
|
||||||
|
// - cloud is included only when a cloud voice setting is present (today an
|
||||||
|
// API key) so a mis-configured build degrades quietly.
|
||||||
|
// - local is a pure adaptation point and is only included when supplied via
|
||||||
|
// WithProviders; by default it is skipped.
|
||||||
|
// - the edge-tts or espeak rung is included according to cfg.TTSProvider.
|
||||||
|
// - silent is always appended last so the chain never fails outright.
|
||||||
|
//
|
||||||
|
// opts may supply a fully assembled step list to override this mapping.
|
||||||
|
func New(cfg config.VoiceConfig, opts ...Option) *Chain {
|
||||||
|
conf := &chainConfig{}
|
||||||
|
for _, o := range opts {
|
||||||
|
o(conf)
|
||||||
|
}
|
||||||
|
if len(conf.steps) > 0 {
|
||||||
|
return NewChain(conf.steps...)
|
||||||
|
}
|
||||||
|
|
||||||
|
steps := make([]Provider, 0, 5)
|
||||||
|
if cfg.DashscopeAPIKey != "" {
|
||||||
|
steps = append(steps, CloudProvider(newDashboardSTT(cfg), newDashboardTTS(cfg)))
|
||||||
|
}
|
||||||
|
switch cfg.TTSProvider {
|
||||||
|
case "edge-tts", "edge":
|
||||||
|
steps = append(steps, EdgeTTSProvider(newEdgeTTS()))
|
||||||
|
case "espeak":
|
||||||
|
steps = append(steps, EspeakProvider(newEspeak()))
|
||||||
|
}
|
||||||
|
steps = append(steps, SilentProvider())
|
||||||
|
return NewChain(steps...)
|
||||||
|
}
|
||||||
@@ -0,0 +1,186 @@
|
|||||||
|
package voice
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// stubSTT is a mock STT returning a fixed result.
|
||||||
|
type stubSTT struct {
|
||||||
|
text string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubSTT) STT(_ context.Context, _ Audio) (string, error) { return s.text, s.err }
|
||||||
|
|
||||||
|
// stubTTS is a mock TTS returning a fixed result.
|
||||||
|
type stubTTS struct {
|
||||||
|
audio Audio
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubTTS) TTS(_ context.Context, _ string) (Audio, error) { return s.audio, s.err }
|
||||||
|
|
||||||
|
// stubProvider is a mock Provider carrying optional directions.
|
||||||
|
type stubProvider struct {
|
||||||
|
name string
|
||||||
|
stt STT
|
||||||
|
tts TTS
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubProvider) Name() string { return s.name }
|
||||||
|
func (s *stubProvider) STT() STT { return s.stt }
|
||||||
|
func (s *stubProvider) TTS() TTS { return s.tts }
|
||||||
|
|
||||||
|
func TestChain_STT_Degrades(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
steps []Provider
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "first provider succeeds",
|
||||||
|
steps: []Provider{&stubProvider{name: "cloud", stt: &stubSTT{text: "hello"}}},
|
||||||
|
want: "hello",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "first fails second succeeds",
|
||||||
|
steps: []Provider{
|
||||||
|
&stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}},
|
||||||
|
&stubProvider{name: "local", stt: &stubSTT{text: "degraded"}},
|
||||||
|
},
|
||||||
|
want: "degraded",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nil direction skipped",
|
||||||
|
steps: []Provider{
|
||||||
|
&stubProvider{name: "edge", stt: nil},
|
||||||
|
&stubProvider{name: "silent", stt: &stubSTT{text: "silence"}},
|
||||||
|
},
|
||||||
|
want: "silence",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "all fail no silent -> error",
|
||||||
|
steps: []Provider{
|
||||||
|
&stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}},
|
||||||
|
},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "all fail silent last -> no error",
|
||||||
|
steps: []Provider{
|
||||||
|
&stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}},
|
||||||
|
SilentProvider(),
|
||||||
|
},
|
||||||
|
want: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
got, err := NewChain(tt.steps...).STT(context.Background(), Audio{Format: "wav", Data: []byte("x")})
|
||||||
|
if tt.wantErr {
|
||||||
|
if !errors.Is(err, ErrProviderFailed) {
|
||||||
|
t.Fatalf("err = %v, want ErrProviderFailed", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("STT error: %v", err)
|
||||||
|
}
|
||||||
|
if got != tt.want {
|
||||||
|
t.Errorf("text = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_TTS_Degrades(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
steps []Provider
|
||||||
|
want Audio
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "first provider succeeds",
|
||||||
|
steps: []Provider{&stubProvider{name: "cloud", tts: &stubTTS{audio: Audio{Format: "mp3", Data: []byte("a")}}}},
|
||||||
|
want: Audio{Format: "mp3", Data: []byte("a")},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "first fails second succeeds",
|
||||||
|
steps: []Provider{
|
||||||
|
&stubProvider{name: "cloud", tts: &stubTTS{err: ErrNotConfigured}},
|
||||||
|
&stubProvider{name: "edge-tts", tts: &stubTTS{audio: Audio{Format: "mp3", Data: []byte("b")}}},
|
||||||
|
},
|
||||||
|
want: Audio{Format: "mp3", Data: []byte("b")},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "all fail no silent -> error",
|
||||||
|
steps: []Provider{
|
||||||
|
&stubProvider{name: "cloud", tts: &stubTTS{err: ErrNotConfigured}},
|
||||||
|
},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
got, err := NewChain(tt.steps...).TTS(context.Background(), "hi")
|
||||||
|
if tt.wantErr {
|
||||||
|
if !errors.Is(err, ErrProviderFailed) {
|
||||||
|
t.Fatalf("err = %v, want ErrProviderFailed", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("TTS error: %v", err)
|
||||||
|
}
|
||||||
|
if got.Format != tt.want.Format || string(got.Data) != string(tt.want.Data) {
|
||||||
|
t.Errorf("audio = %+v, want %+v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_Empty(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if _, err := NewChain().STT(context.Background(), Audio{}); !errors.Is(err, ErrProviderFailed) {
|
||||||
|
t.Fatalf("err = %v, want ErrProviderFailed", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNew_ConfigAssembly(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
// With a cloud key and default edge-tts TTS, the chain starts at cloud and
|
||||||
|
// ends at silent.
|
||||||
|
c := New(config.VoiceConfig{DashscopeAPIKey: "k"})
|
||||||
|
names := make([]string, 0, len(c.steps))
|
||||||
|
for _, p := range c.steps {
|
||||||
|
names = append(names, p.Name())
|
||||||
|
}
|
||||||
|
if len(names) < 2 || names[len(names)-1] != "silent" {
|
||||||
|
t.Fatalf("steps = %v, want to end at silent", names)
|
||||||
|
}
|
||||||
|
if names[0] != "cloud" {
|
||||||
|
t.Errorf("first step = %q, want cloud", names[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Without a key the cloud rung is skipped.
|
||||||
|
c2 := New(config.VoiceConfig{})
|
||||||
|
foundCloud := false
|
||||||
|
for _, p := range c2.steps {
|
||||||
|
if p.Name() == "cloud" {
|
||||||
|
foundCloud = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if foundCloud {
|
||||||
|
t.Error("cloud step present without an API key")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
package voice
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// cloudMaxResponseBytes caps how much of a cloud voice response is read, so an
|
||||||
|
// oversized or malformed response cannot exhaust memory. The request timeout
|
||||||
|
// bounds a single call.
|
||||||
|
const (
|
||||||
|
cloudMaxResponseBytes = 8 << 20
|
||||||
|
cloudHTTPTimeout = 180 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// HTTPSTT transcribes audio by POSTing a JSON body to a cloud endpoint and
|
||||||
|
// reading {"text": "..."} back. It is the stubbed default STT implementation;
|
||||||
|
// the real DashScope/whisper-facing adapter is an adaptation point.
|
||||||
|
type HTTPSTT struct {
|
||||||
|
endpoint string
|
||||||
|
apiKey string
|
||||||
|
client *http.Client
|
||||||
|
log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTPTTS synthesizes speech by POSTing a JSON body and reading the returned
|
||||||
|
// base64 audio. It is the stubbed default TTS implementation.
|
||||||
|
type HTTPTTS struct {
|
||||||
|
endpoint string
|
||||||
|
apiKey string
|
||||||
|
client *http.Client
|
||||||
|
log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTPOption customizes the client and logger of an HTTPSTT/HTTPTTS.
|
||||||
|
type HTTPOption func(*httpConfig)
|
||||||
|
|
||||||
|
type httpConfig struct {
|
||||||
|
client *http.Client
|
||||||
|
log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithHTTPClient sets the *http.Client used for the call; nil keeps the default.
|
||||||
|
func WithHTTPClient(c *http.Client) HTTPOption {
|
||||||
|
return func(o *httpConfig) {
|
||||||
|
if c != nil {
|
||||||
|
o.client = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithLogger sets the logger used on failures; nil keeps a discard logger.
|
||||||
|
func WithLogger(l *slog.Logger) HTTPOption {
|
||||||
|
return func(o *httpConfig) {
|
||||||
|
if l != nil {
|
||||||
|
o.log = l
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewHTTPSTT builds an HTTPSTT for the given endpoint and API key.
|
||||||
|
func NewHTTPSTT(endpoint, apiKey string, opts ...HTTPOption) *HTTPSTT {
|
||||||
|
h := &httpConfig{client: defaultHTTPClient(), log: discardLogger()}
|
||||||
|
for _, o := range opts {
|
||||||
|
o(h)
|
||||||
|
}
|
||||||
|
return &HTTPSTT{endpoint: strings.TrimRight(endpoint, "/"), apiKey: apiKey, client: h.client, log: h.log}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewHTTPTTS builds an HTTPTTS for the given endpoint and API key.
|
||||||
|
func NewHTTPTTS(endpoint, apiKey string, opts ...HTTPOption) *HTTPTTS {
|
||||||
|
h := &httpConfig{client: defaultHTTPClient(), log: discardLogger()}
|
||||||
|
for _, o := range opts {
|
||||||
|
o(h)
|
||||||
|
}
|
||||||
|
return &HTTPTTS{endpoint: strings.TrimRight(endpoint, "/"), apiKey: apiKey, client: h.client, log: h.log}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sttRequest is the JSON body sent to the STT endpoint.
|
||||||
|
type sttRequest struct {
|
||||||
|
Audio string `json:"audio"`
|
||||||
|
Format string `json:"format"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// sttResponse is the transcription returned by the STT endpoint.
|
||||||
|
type sttResponse struct {
|
||||||
|
Text string `json:"text"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// STT posts the audio and returns the recognized text. A blank endpoint returns
|
||||||
|
// ErrNotConfigured so the chain degrades.
|
||||||
|
func (s *HTTPSTT) STT(ctx context.Context, audio Audio) (string, error) {
|
||||||
|
if s.endpoint == "" {
|
||||||
|
return "", ErrNotConfigured
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(sttRequest{Audio: base64.StdEncoding.EncodeToString(audio.Data), Format: audio.Format})
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err)
|
||||||
|
}
|
||||||
|
resp, err := postJSON(ctx, s.client, s.apiKey, s.endpoint, payload)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
var decoded sttResponse
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
|
||||||
|
return "", fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err)
|
||||||
|
}
|
||||||
|
return decoded.Text, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ttsRequest is the JSON body sent to the TTS endpoint.
|
||||||
|
type ttsRequest struct {
|
||||||
|
Text string `json:"text"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ttsResponse is the synthesized audio returned by the TTS endpoint.
|
||||||
|
type ttsResponse struct {
|
||||||
|
Audio string `json:"audio"`
|
||||||
|
Format string `json:"format"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TTS posts the text and returns the decoded audio. A blank endpoint returns
|
||||||
|
// ErrNotConfigured so the chain degrades.
|
||||||
|
func (s *HTTPTTS) TTS(ctx context.Context, text string) (Audio, error) {
|
||||||
|
if s.endpoint == "" {
|
||||||
|
return Audio{}, ErrNotConfigured
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(text) == "" {
|
||||||
|
return Audio{}, ErrEmptyText
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return Audio{}, err
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(ttsRequest{Text: text})
|
||||||
|
if err != nil {
|
||||||
|
return Audio{}, fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err)
|
||||||
|
}
|
||||||
|
resp, err := postJSON(ctx, s.client, s.apiKey, s.endpoint, payload)
|
||||||
|
if err != nil {
|
||||||
|
return Audio{}, err
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
var decoded ttsResponse
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
|
||||||
|
return Audio{}, fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err)
|
||||||
|
}
|
||||||
|
data, err := base64.StdEncoding.DecodeString(decoded.Audio)
|
||||||
|
if err != nil {
|
||||||
|
return Audio{}, fmt.Errorf("voice: %w: decode audio: %v", ErrProviderFailed, err)
|
||||||
|
}
|
||||||
|
format := decoded.Format
|
||||||
|
if format == "" {
|
||||||
|
format = "mp3"
|
||||||
|
}
|
||||||
|
return Audio{Format: format, Data: data}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// postJSON POSTs a JSON payload, wires auth, and checks the status code.
|
||||||
|
func postJSON(ctx context.Context, client *http.Client, apiKey, url string, payload []byte) (*http.Response, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("voice: %w: build: %v", ErrProviderFailed, err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
if apiKey != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||||
|
}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("voice: %w: request: %v", ErrProviderFailed, err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
return nil, fmt.Errorf("voice: %w: status %d", ErrProviderFailed, resp.StatusCode)
|
||||||
|
}
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultHTTPClient builds a client with a request timeout. A fresh client is
|
||||||
|
// returned per call rather than a package global, which the constraints forbid.
|
||||||
|
func defaultHTTPClient() *http.Client {
|
||||||
|
return &http.Client{Transport: http.DefaultTransport, Timeout: cloudHTTPTimeout}
|
||||||
|
}
|
||||||
|
|
||||||
|
// discardLogger returns a logger that drops all output.
|
||||||
|
func discardLogger() *slog.Logger {
|
||||||
|
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
// newDashboardSTT returns the cloud STT for a config. The endpoint is an
|
||||||
|
// adaptation point and is empty until config gains a voice URL; a call then
|
||||||
|
// returns ErrNotConfigured and the chain degrades.
|
||||||
|
func newDashboardSTT(cfg config.VoiceConfig) STT {
|
||||||
|
return NewHTTPSTT("", cfg.DashscopeAPIKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newDashboardTTS returns the cloud TTS for a config.
|
||||||
|
func newDashboardTTS(cfg config.VoiceConfig) TTS {
|
||||||
|
return NewHTTPTTS("", cfg.DashscopeAPIKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newEdgeTTS returns the edge-tts adapter point (not yet implemented).
|
||||||
|
func newEdgeTTS() TTS { return notImplementedTTS{} }
|
||||||
|
|
||||||
|
// newEspeak returns the espeak adapter point (not yet implemented).
|
||||||
|
func newEspeak() TTS { return notImplementedTTS{} }
|
||||||
|
|
||||||
|
// notImplementedTTS is an adaptation point that always degrades.
|
||||||
|
type notImplementedTTS struct{}
|
||||||
|
|
||||||
|
func (notImplementedTTS) TTS(_ context.Context, _ string) (Audio, error) {
|
||||||
|
return Audio{}, ErrNotImplemented
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
package voice
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHTTPSTT(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var body map[string]any
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if r.Header.Get("Authorization") != "Bearer sk" {
|
||||||
|
t.Errorf("auth = %q, want Bearer sk", r.Header.Get("Authorization"))
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"text":"hello world"}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
stt := NewHTTPSTT(srv.URL, "sk")
|
||||||
|
text, err := stt.STT(context.Background(), Audio{Format: "wav", Data: []byte("raw")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("STT error: %v", err)
|
||||||
|
}
|
||||||
|
if text != "hello world" {
|
||||||
|
t.Errorf("text = %q, want hello world", text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPSTT_NotConfigured(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
stt := NewHTTPSTT("", "")
|
||||||
|
if _, err := stt.STT(context.Background(), Audio{}); err != ErrNotConfigured {
|
||||||
|
t.Fatalf("err = %v, want ErrNotConfigured", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPTTS(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
audioBytes := []byte{1, 2, 3}
|
||||||
|
b64 := base64.StdEncoding.EncodeToString(audioBytes)
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var body map[string]any
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if body["text"] != "hi" {
|
||||||
|
t.Errorf("text = %v, want hi", body["text"])
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"audio":"` + b64 + `","format":"wav"}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
tts := NewHTTPTTS(srv.URL, "")
|
||||||
|
audio, err := tts.TTS(context.Background(), "hi")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("TTS error: %v", err)
|
||||||
|
}
|
||||||
|
if audio.Format != "wav" || hex.EncodeToString(audio.Data) != hex.EncodeToString(audioBytes) {
|
||||||
|
t.Errorf("audio = %+v, want wav/010203", audio)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPTTS_EmptyText(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||||
|
defer srv.Close()
|
||||||
|
tts := NewHTTPTTS(srv.URL, "")
|
||||||
|
if _, err := tts.TTS(context.Background(), " "); err != ErrEmptyText {
|
||||||
|
t.Fatalf("err = %v, want ErrEmptyText", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package voice
|
||||||
|
|
||||||
|
import "errors"
|
||||||
|
|
||||||
|
// Sentinel errors returned by the voice package. Each is wrapped with %w so
|
||||||
|
// callers can test for a category while keeping the cause.
|
||||||
|
var (
|
||||||
|
// ErrProviderFailed is returned when every provider in a chain fails.
|
||||||
|
ErrProviderFailed = errors.New("voice: all providers failed")
|
||||||
|
// ErrNotConfigured is returned when a provider has no endpoint/setting.
|
||||||
|
ErrNotConfigured = errors.New("voice: provider not configured")
|
||||||
|
// ErrNotImplemented is returned by an adaptation-point provider.
|
||||||
|
ErrNotImplemented = errors.New("voice: provider not implemented")
|
||||||
|
// ErrEmptyText is returned when TTS receives empty text.
|
||||||
|
ErrEmptyText = errors.New("voice: empty text")
|
||||||
|
// ErrClosed is returned when a service call arrives after Close.
|
||||||
|
ErrClosed = errors.New("voice: service closed")
|
||||||
|
)
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package voice
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// Provider is one rung in the voice degradation chain. It may implement zero,
|
||||||
|
// one, or both directions; a nil return means that direction is not available
|
||||||
|
// on this rung and the chain moves to the next.
|
||||||
|
type Provider interface {
|
||||||
|
// Name returns a stable provider identifier.
|
||||||
|
Name() string
|
||||||
|
// STT returns the speech-to-text implementation, or nil when the provider
|
||||||
|
// cannot transcribe.
|
||||||
|
STT() STT
|
||||||
|
// TTS returns the text-to-speech implementation, or nil when the provider
|
||||||
|
// cannot synthesize.
|
||||||
|
TTS() TTS
|
||||||
|
}
|
||||||
|
|
||||||
|
// StaticProvider is a Provider with a fixed name whose STT/TTS are set at
|
||||||
|
// construction. It is used by the concrete providers below.
|
||||||
|
type StaticProvider struct {
|
||||||
|
name string
|
||||||
|
stt STT
|
||||||
|
tts TTS
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the provider identifier.
|
||||||
|
func (p *StaticProvider) Name() string { return p.name }
|
||||||
|
|
||||||
|
// STT returns the configured STT, or nil.
|
||||||
|
func (p *StaticProvider) STT() STT { return p.stt }
|
||||||
|
|
||||||
|
// TTS returns the configured TTS, or nil.
|
||||||
|
func (p *StaticProvider) TTS() TTS { return p.tts }
|
||||||
|
|
||||||
|
// NewProvider builds a Provider with the given name and optional directions it
|
||||||
|
// supports.
|
||||||
|
func NewProvider(name string, stt STT, tts TTS) *StaticProvider {
|
||||||
|
return &StaticProvider{name: name, stt: stt, tts: tts}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloudProvider is the cloud rung of the chain. It is an adaptation point: the
|
||||||
|
// real STT/TTS implementations are wired in via NewProvider with cloud adapters.
|
||||||
|
func CloudProvider(stt STT, tts TTS) *StaticProvider {
|
||||||
|
return NewProvider("cloud", stt, tts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LocalProvider is the local-model rung of the chain.
|
||||||
|
func LocalProvider(stt STT, tts TTS) *StaticProvider {
|
||||||
|
return NewProvider("local", stt, tts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EdgeTTSProvider is the edge-tts rung (text-to-speech only).
|
||||||
|
func EdgeTTSProvider(tts TTS) *StaticProvider {
|
||||||
|
return NewProvider("edge-tts", nil, tts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EspeakProvider is the espeak rung (text-to-speech only).
|
||||||
|
func EspeakProvider(tts TTS) *StaticProvider {
|
||||||
|
return NewProvider("espeak", nil, tts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SilentProvider is the terminal rung. It always succeeds, returning an empty
|
||||||
|
// transcription and empty audio, so a chain that includes it never errors.
|
||||||
|
func SilentProvider() *StaticProvider {
|
||||||
|
return NewProvider("silent", silenceSTT{}, silenceTTS{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// silenceSTT is the silent STT rung; it returns an empty transcription.
|
||||||
|
type silenceSTT struct{}
|
||||||
|
|
||||||
|
func (silenceSTT) STT(_ context.Context, _ Audio) (string, error) { return "", nil }
|
||||||
|
|
||||||
|
// silenceTTS is the silent TTS rung; it returns empty audio.
|
||||||
|
type silenceTTS struct{}
|
||||||
|
|
||||||
|
func (silenceTTS) TTS(_ context.Context, _ string) (Audio, error) {
|
||||||
|
return Audio{}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
package voice
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Service runs STT/TTS requests on a background worker pool so caller
|
||||||
|
// goroutines (the agent loop) are never blocked by provider latency. Each
|
||||||
|
// request is enqueued on a jobs channel and answered on a per-job result
|
||||||
|
// channel; a canceled caller context or a closed service unblocks the caller
|
||||||
|
// immediately and no worker goroutine is leaked. It is safe for concurrent use.
|
||||||
|
type Service struct {
|
||||||
|
chain *Chain
|
||||||
|
jobs chan job
|
||||||
|
bgCtx context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
nWorkers int
|
||||||
|
wg sync.WaitGroup
|
||||||
|
mu sync.RWMutex
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServiceOption customizes a Service.
|
||||||
|
type ServiceOption func(*Service)
|
||||||
|
|
||||||
|
// WithWorkers sets the worker-pool size. Values <= 0 are ignored and the
|
||||||
|
// default single worker is kept.
|
||||||
|
func WithWorkers(n int) ServiceOption {
|
||||||
|
return func(s *Service) {
|
||||||
|
if n > 0 {
|
||||||
|
s.nWorkers = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// jobKind selects the operation a worker performs.
|
||||||
|
type jobKind int
|
||||||
|
|
||||||
|
const (
|
||||||
|
jobSTT jobKind = iota
|
||||||
|
jobTTS
|
||||||
|
)
|
||||||
|
|
||||||
|
// job is one request enqueued by the caller.
|
||||||
|
type job struct {
|
||||||
|
kind jobKind
|
||||||
|
audio Audio
|
||||||
|
text string
|
||||||
|
res chan jobResult
|
||||||
|
}
|
||||||
|
|
||||||
|
// jobResult carries the outcome written back to the caller.
|
||||||
|
type jobResult struct {
|
||||||
|
text string
|
||||||
|
audio Audio
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewService builds a Service backed by chain and starts the worker pool.
|
||||||
|
func NewService(chain *Chain, opts ...ServiceOption) *Service {
|
||||||
|
bg, cancel := context.WithCancel(context.Background())
|
||||||
|
s := &Service{
|
||||||
|
chain: chain,
|
||||||
|
jobs: make(chan job),
|
||||||
|
bgCtx: bg,
|
||||||
|
cancel: cancel,
|
||||||
|
nWorkers: 1,
|
||||||
|
}
|
||||||
|
for _, o := range opts {
|
||||||
|
o(s)
|
||||||
|
}
|
||||||
|
for i := 0; i < s.nWorkers; i++ {
|
||||||
|
s.wg.Add(1)
|
||||||
|
go s.worker()
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// worker consumes jobs until the service closes.
|
||||||
|
func (s *Service) worker() {
|
||||||
|
defer s.wg.Done()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-s.bgCtx.Done():
|
||||||
|
return
|
||||||
|
case j := <-s.jobs:
|
||||||
|
s.run(j)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// run executes a job and delivers the result. Providers run under the service's
|
||||||
|
// background context so Close cancels in-flight work and no worker goroutine is
|
||||||
|
// leaked; the result is written to a buffered channel so a caller that already
|
||||||
|
// returned never blocks the worker.
|
||||||
|
func (s *Service) run(j job) {
|
||||||
|
res := jobResult{}
|
||||||
|
switch j.kind {
|
||||||
|
case jobSTT:
|
||||||
|
res.text, res.err = s.chain.STT(s.bgCtx, j.audio)
|
||||||
|
case jobTTS:
|
||||||
|
res.audio, res.err = s.chain.TTS(s.bgCtx, j.text)
|
||||||
|
}
|
||||||
|
j.res <- res
|
||||||
|
}
|
||||||
|
|
||||||
|
// STT enqueues a transcription request and waits for the result. ctx cancels
|
||||||
|
// the wait and the underlying provider call.
|
||||||
|
func (s *Service) STT(ctx context.Context, audio Audio) (string, error) {
|
||||||
|
res, err := s.submit(ctx, job{kind: jobSTT, audio: audio, res: make(chan jobResult, 1)})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return res.text, res.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// TTS enqueues a synthesis request and waits for the result. ctx cancels the
|
||||||
|
// wait and the underlying provider call.
|
||||||
|
func (s *Service) TTS(ctx context.Context, text string) (Audio, error) {
|
||||||
|
res, err := s.submit(ctx, job{kind: jobTTS, text: text, res: make(chan jobResult, 1)})
|
||||||
|
if err != nil {
|
||||||
|
return Audio{}, err
|
||||||
|
}
|
||||||
|
return res.audio, res.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// submit enqueues j and blocks until the result or a cancellation.
|
||||||
|
func (s *Service) submit(ctx context.Context, j job) (jobResult, error) {
|
||||||
|
select {
|
||||||
|
case s.jobs <- j:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return jobResult{}, fmt.Errorf("voice: %w", ctx.Err())
|
||||||
|
case <-s.bgCtx.Done():
|
||||||
|
return jobResult{}, ErrClosed
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case r := <-j.res:
|
||||||
|
return r, nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return jobResult{}, fmt.Errorf("voice: %w", ctx.Err())
|
||||||
|
case <-s.bgCtx.Done():
|
||||||
|
return jobResult{}, ErrClosed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close cancels the worker pool and waits for in-flight jobs to finish. It is
|
||||||
|
// safe to call once; later calls are no-ops. In-flight provider calls are
|
||||||
|
// bounded by each provider's own context timeout, so Close returns promptly.
|
||||||
|
func (s *Service) Close() {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.closed {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.closed = true
|
||||||
|
s.cancel()
|
||||||
|
s.wg.Wait()
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package voice
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestService_STTAndTTS(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
chain := NewChain(
|
||||||
|
NewProvider("cloud", &stubSTT{text: "transcribed"}, &stubTTS{audio: Audio{Format: "mp3", Data: []byte("a")}}),
|
||||||
|
)
|
||||||
|
svc := NewService(chain)
|
||||||
|
defer svc.Close()
|
||||||
|
|
||||||
|
text, err := svc.STT(context.Background(), Audio{Format: "wav", Data: []byte("x")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("STT error: %v", err)
|
||||||
|
}
|
||||||
|
if text != "transcribed" {
|
||||||
|
t.Errorf("text = %q, want transcribed", text)
|
||||||
|
}
|
||||||
|
|
||||||
|
audio, err := svc.TTS(context.Background(), "hello")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("TTS error: %v", err)
|
||||||
|
}
|
||||||
|
if string(audio.Data) != "a" || audio.Format != "mp3" {
|
||||||
|
t.Errorf("audio = %+v, want mp3/a", audio)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestService_CloseStopsNewCalls(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
svc := NewService(NewChain(SilentProvider()))
|
||||||
|
svc.Close()
|
||||||
|
_, err := svc.STT(context.Background(), Audio{})
|
||||||
|
if !errors.Is(err, ErrClosed) {
|
||||||
|
t.Fatalf("err = %v, want ErrClosed", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// blockingSTT blocks until its context is canceled, marking that it started.
|
||||||
|
type blockingSTT struct {
|
||||||
|
started chan struct{}
|
||||||
|
once sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *blockingSTT) STT(ctx context.Context, _ Audio) (string, error) {
|
||||||
|
b.once.Do(func() { close(b.started) })
|
||||||
|
<-ctx.Done()
|
||||||
|
return "", ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestService_CloseCancelsInFlightWorker(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
blk := &blockingSTT{started: make(chan struct{})}
|
||||||
|
chain := NewChain(NewProvider("cloud", blk, nil))
|
||||||
|
svc := NewService(chain)
|
||||||
|
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
_, _ = svc.STT(context.Background(), Audio{Format: "wav", Data: []byte("x")})
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-blk.started:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("provider never started")
|
||||||
|
}
|
||||||
|
|
||||||
|
svc.Close()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("STT call did not return after Close; worker likely leaked")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestService_CallerCancelUnblocks(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
blk := &blockingSTT{started: make(chan struct{})}
|
||||||
|
svc := NewService(NewChain(NewProvider("cloud", blk, nil)))
|
||||||
|
defer svc.Close()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
_, _ = svc.STT(ctx, Audio{Format: "wav", Data: []byte("x")})
|
||||||
|
}()
|
||||||
|
|
||||||
|
<-blk.started
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("caller cancellation did not unblock STT")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
// Package voice implements asynchronous speech-to-text and text-to-speech for
|
||||||
|
// the Cyrene runtime, following DevDocs/design/06-多模态.md §1.
|
||||||
|
//
|
||||||
|
// Voice input (STT) is transcribed to text and fed into the same agent pipeline
|
||||||
|
// as a chat message; the reply is synthesized (TTS) back to audio. Both
|
||||||
|
// directions degrade through a provider chain — cloud → local → edge-tts/espeak
|
||||||
|
// → silent — so an unavailable provider falls through to the next rung instead
|
||||||
|
// of failing the whole turn. The stub default implementations are HTTP-based and
|
||||||
|
// testable with a stdlib httptest server; real cloud/local providers remain
|
||||||
|
// adaptation points for later milestones.
|
||||||
|
package voice
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// Audio is a piece of audio data plus its encoding.
|
||||||
|
type Audio struct {
|
||||||
|
// Format is the encoding, e.g. "wav", "mp3", "ogg".
|
||||||
|
Format string
|
||||||
|
// Data is the raw audio bytes.
|
||||||
|
Data []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// STT converts audio to text.
|
||||||
|
type STT interface {
|
||||||
|
// STT transcribes audio and returns the spoken text. ctx controls
|
||||||
|
// cancellation and must be honored.
|
||||||
|
STT(ctx context.Context, audio Audio) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TTS synthesizes speech audio from text.
|
||||||
|
type TTS interface {
|
||||||
|
// TTS synthesizes audio from text. ctx controls cancellation and must be
|
||||||
|
// honored.
|
||||||
|
TTS(ctx context.Context, text string) (Audio, error)
|
||||||
|
}
|
||||||
Reference in new issue
Block a user