From 0a936a3202e7553900f0a92268520da906e53ad8 Mon Sep 17 00:00:00 2001 From: Aska Date: Sun, 6 Sep 2026 05:12:36 +0800 Subject: [PATCH] =?UTF-8?q?feat(multimodal):=20=E8=A7=86=E8=A7=89=E5=88=86?= =?UTF-8?q?=E5=B1=82=E8=B7=AF=E7=94=B1=EF=BC=88=E5=A4=9A=E6=A8=A1=E6=80=81?= =?UTF-8?q?=E6=A3=80=E6=B5=8B=E2=86=92=E4=B8=BB/=E8=A7=86=E8=A7=89/OCR=20?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=EF=BC=89+=20=E8=AF=AD=E9=9F=B3=20STT?= =?UTF-8?q?=E2=86=92TTS=20=E9=99=8D=E7=BA=A7=E9=93=BE=EF=BC=88=E5=BC=82?= =?UTF-8?q?=E6=AD=A5=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/runtime/multimodal/doc.go | 17 ++ internal/runtime/multimodal/vision/adapter.go | 217 +++++++++++++++++ .../runtime/multimodal/vision/adapter_test.go | 53 ++++ internal/runtime/multimodal/vision/errors.go | 22 ++ internal/runtime/multimodal/vision/image.go | 178 ++++++++++++++ internal/runtime/multimodal/vision/router.go | 128 ++++++++++ .../runtime/multimodal/vision/router_test.go | 178 ++++++++++++++ internal/runtime/multimodal/vision/tool.go | 148 ++++++++++++ internal/runtime/multimodal/vision/vision.go | 79 ++++++ .../runtime/multimodal/vision/vision_test.go | 95 ++++++++ internal/runtime/multimodal/voice/chain.go | 112 +++++++++ .../runtime/multimodal/voice/chain_test.go | 186 ++++++++++++++ internal/runtime/multimodal/voice/cloud.go | 227 ++++++++++++++++++ .../runtime/multimodal/voice/cloud_test.go | 81 +++++++ internal/runtime/multimodal/voice/errors.go | 18 ++ internal/runtime/multimodal/voice/provider.go | 79 ++++++ internal/runtime/multimodal/voice/service.go | 160 ++++++++++++ .../runtime/multimodal/voice/service_test.go | 108 +++++++++ internal/runtime/multimodal/voice/voice.go | 35 +++ 19 files changed, 2121 insertions(+) create mode 100644 internal/runtime/multimodal/doc.go create mode 100644 internal/runtime/multimodal/vision/adapter.go create mode 100644 internal/runtime/multimodal/vision/adapter_test.go create mode 100644 internal/runtime/multimodal/vision/errors.go create mode 100644 internal/runtime/multimodal/vision/image.go create mode 100644 internal/runtime/multimodal/vision/router.go create mode 100644 internal/runtime/multimodal/vision/router_test.go create mode 100644 internal/runtime/multimodal/vision/tool.go create mode 100644 internal/runtime/multimodal/vision/vision.go create mode 100644 internal/runtime/multimodal/vision/vision_test.go create mode 100644 internal/runtime/multimodal/voice/chain.go create mode 100644 internal/runtime/multimodal/voice/chain_test.go create mode 100644 internal/runtime/multimodal/voice/cloud.go create mode 100644 internal/runtime/multimodal/voice/cloud_test.go create mode 100644 internal/runtime/multimodal/voice/errors.go create mode 100644 internal/runtime/multimodal/voice/provider.go create mode 100644 internal/runtime/multimodal/voice/service.go create mode 100644 internal/runtime/multimodal/voice/service_test.go create mode 100644 internal/runtime/multimodal/voice/voice.go diff --git a/internal/runtime/multimodal/doc.go b/internal/runtime/multimodal/doc.go new file mode 100644 index 0000000..76a487c --- /dev/null +++ b/internal/runtime/multimodal/doc.go @@ -0,0 +1,17 @@ +// Package multimodal groups the runtime's non-text input/output handling: +// image understanding (vision) and speech (voice). +// +// It follows DevDocs/design/06-多模态.md: +// +// - The vision subpackage routes image messages through a layered strategy +// (main model first, then a tool-shaped vision/OCR adapter) and registers +// those adapters as sdk.Tool capabilities so the M2 tool loop can invoke +// them without putting the image in the main-model context. +// - The voice subpackage provides asynchronous STT/TTS through a degrading +// provider chain (cloud → local → edge-tts/espeak → silent). +// +// Both subpackages keep real provider calls behind interfaces with an HTTP +// default implementation, so they are testable with a stdlib httptest server +// and leave the real cloud/local providers as adaptation points for later +// milestones. +package multimodal diff --git a/internal/runtime/multimodal/vision/adapter.go b/internal/runtime/multimodal/vision/adapter.go new file mode 100644 index 0000000..ab0b2a3 --- /dev/null +++ b/internal/runtime/multimodal/vision/adapter.go @@ -0,0 +1,217 @@ +package vision + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/config" +) + +// visionMaxTokens caps the vision/OCR response length. +const visionMaxTokens = 512 + +// VisionAdapter understands an image and returns a description of it. It is the +// backend behind the vision tool; real providers implement it without exposing +// the image to the main-model context. +type VisionAdapter interface { + // Describe returns a description of img. prompt is an optional instruction + // overriding the default "describe this image" ask. + Describe(ctx context.Context, img ImageRef, prompt string) (string, error) +} + +// OCRAdapter extracts text from an image. It is the backend behind the OCR tool. +type OCRAdapter interface { + // Extract returns the text found in img. + Extract(ctx context.Context, img ImageRef) (string, error) +} + +// OpenAICompatibleAdapter implements both VisionAdapter and OCRAdapter by +// calling an OpenAI-compatible /chat/completions endpoint with the image +// embedded as a content part. It is the ready default implementation; real +// cloud vision/OCR providers expose the same protocol. +type OpenAICompatibleAdapter struct { + cfg config.LLMConfig + model string + client *http.Client + log *slog.Logger +} + +// OpenAIOption customizes an OpenAICompatibleAdapter. +type OpenAIOption func(*OpenAICompatibleAdapter) + +// WithModel overrides the adapter model (defaults to cfg.Model). +func WithModel(m string) OpenAIOption { + return func(a *OpenAICompatibleAdapter) { + if m != "" { + a.model = m + } + } +} + +// WithHTTPClient sets the *http.Client used for the request; nil keeps default. +func WithHTTPClient(c *http.Client) OpenAIOption { + return func(a *OpenAICompatibleAdapter) { + if c != nil { + a.client = c + } + } +} + +// WithLogger sets the logger used on failures; nil keeps a discard logger. +func WithLogger(l *slog.Logger) OpenAIOption { + return func(a *OpenAICompatibleAdapter) { + if l != nil { + a.log = l + } + } +} + +// NewOpenAICompatible builds an adapter from cfg, using cfg.Model unless +// overridden. +func NewOpenAICompatible(cfg config.LLMConfig, opts ...OpenAIOption) *OpenAICompatibleAdapter { + a := &OpenAICompatibleAdapter{ + cfg: cfg, + model: cfg.Model, + client: defaultHTTPClient(), + log: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + for _, o := range opts { + o(a) + } + return a +} + +// Describe returns an image description from the underlying chat endpoint. +func (a *OpenAICompatibleAdapter) Describe(ctx context.Context, img ImageRef, prompt string) (string, error) { + if prompt == "" { + prompt = "Describe this image in detail." + } + return a.ask(ctx, img, prompt) +} + +// Extract returns text found in the image via the same endpoint. +func (a *OpenAICompatibleAdapter) Extract(ctx context.Context, img ImageRef) (string, error) { + return a.ask(ctx, img, "Extract all text visible in this image. Return only the text.") +} + +// ask resolves the image, builds a chat request and returns the assistant text. +func (a *OpenAICompatibleAdapter) ask(ctx context.Context, img ImageRef, prompt string) (string, error) { + if strings.TrimSpace(a.cfg.BaseURL) == "" { + return "", ErrNotConfigured + } + if strings.TrimSpace(a.model) == "" { + return "", ErrNotConfigured + } + if err := ctx.Err(); err != nil { + return "", err + } + imageURL, err := a.imageURL(ctx, img) + if err != nil { + return "", err + } + payload, err := json.Marshal(a.request(imageURL, prompt)) + if err != nil { + return "", fmt.Errorf("vision: %w: marshal request: %v", ErrUpstream, err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(a.cfg.BaseURL, "/")+"/chat/completions", bytes.NewReader(payload)) + if err != nil { + return "", fmt.Errorf("vision: %w: build request: %v", ErrUpstream, err) + } + req.Header.Set("Content-Type", "application/json") + if a.cfg.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+a.cfg.APIKey) + } + resp, err := a.client.Do(req) + if err != nil { + return "", fmt.Errorf("vision: %w: %v", ErrUpstream, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, maxFetchedBody)) + return "", fmt.Errorf("vision: %w: status %d: %s", ErrUpstream, resp.StatusCode, strings.TrimSpace(string(snippet))) + } + return a.decode(resp.Body) +} + +// imageURL converts an ImageRef to the image_url value the wire expects: an +// http(s) URL passes through, anything else becomes a base64 data: URI. +func (a *OpenAICompatibleAdapter) imageURL(ctx context.Context, img ImageRef) (string, error) { + if img.Source == SourceURL { + return img.Data, nil + } + mime, data, err := Resolve(ctx, img, a.client) + if err != nil { + return "", err + } + return DataURI(mime, data), nil +} + +// openAIVisionRequest is the JSON body sent to /chat/completions. +type openAIVisionRequest struct { + Model string `json:"model"` + Messages []openAIMessage `json:"messages"` + MaxTokens int `json:"max_tokens"` +} + +type openAIMessage struct { + Role string `json:"role"` + Content any `json:"content"` +} + +type openAIPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *openAIImageURL `json:"image_url,omitempty"` +} + +type openAIImageURL struct { + URL string `json:"url"` +} + +type openAIVisionResponse struct { + Choices []openAIChoice `json:"choices"` +} + +type openAIChoice struct { + Message openAIResponseMessage `json:"message"` +} + +type openAIResponseMessage struct { + Content string `json:"content"` +} + +// request builds the wire body for an image+prompt. +func (a *OpenAICompatibleAdapter) request(imageURL, prompt string) openAIVisionRequest { + return openAIVisionRequest{ + Model: a.model, + Messages: []openAIMessage{ + {Role: "user", Content: []openAIPart{ + {Type: "text", Text: prompt}, + {Type: "image_url", ImageURL: &openAIImageURL{URL: imageURL}}, + }}, + }, + MaxTokens: visionMaxTokens, + } +} + +// decode extracts the first choice's text from a response body. +func (a *OpenAICompatibleAdapter) decode(r io.Reader) (string, error) { + var wire openAIVisionResponse + if err := json.NewDecoder(r).Decode(&wire); err != nil { + return "", fmt.Errorf("vision: %w: decode response: %v", ErrUpstream, err) + } + if len(wire.Choices) == 0 { + return "", fmt.Errorf("vision: %w: empty choices", ErrUpstream) + } + return wire.Choices[0].Message.Content, nil +} + +// interface assertion: one concrete adapter serves both directions. +var _ VisionAdapter = (*OpenAICompatibleAdapter)(nil) +var _ OCRAdapter = (*OpenAICompatibleAdapter)(nil) diff --git a/internal/runtime/multimodal/vision/adapter_test.go b/internal/runtime/multimodal/vision/adapter_test.go new file mode 100644 index 0000000..4b4dcfb --- /dev/null +++ b/internal/runtime/multimodal/vision/adapter_test.go @@ -0,0 +1,53 @@ +package vision + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/config" +) + +func TestOpenAICompatibleAdapter_Describe(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/chat/completions" { + t.Errorf("path = %s, want /chat/completions", r.URL.Path) + } + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + msgs, _ := body["messages"].([]any) + if len(msgs) != 1 { + t.Fatalf("messages = %d, want 1", len(msgs)) + } + content, _ := msgs[0].(map[string]any)["content"].([]any) + if len(content) != 2 { + t.Fatalf("content parts = %d, want 2", len(content)) + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"choices":[{"message":{"content":"a red car"}}]}`) + })) + defer srv.Close() + + ad := NewOpenAICompatible(config.LLMConfig{BaseURL: srv.URL, Model: "vision-"}) + text, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: "aGVsbG8="}, "") + if err != nil { + t.Fatalf("Describe error: %v", err) + } + if text != "a red car" { + t.Errorf("text = %q, want a red car", text) + } +} + +func TestOpenAICompatibleAdapter_NotConfigured(t *testing.T) { + t.Parallel() + ad := NewOpenAICompatible(config.LLMConfig{}) + if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: "aGVsbG8="}, ""); err == nil { + t.Fatal("want error for empty base url, got nil") + } +} diff --git a/internal/runtime/multimodal/vision/errors.go b/internal/runtime/multimodal/vision/errors.go new file mode 100644 index 0000000..9cce9dc --- /dev/null +++ b/internal/runtime/multimodal/vision/errors.go @@ -0,0 +1,22 @@ +package vision + +import "errors" + +// Sentinel errors returned by the vision package. Each is wrapped with %w on +// the way out so callers can test for a category while keeping the cause. +var ( + // ErrNoImage is returned when an image source string is empty. + ErrNoImage = errors.New("vision: empty image source") + // ErrNotConfigured is returned by adapters when no endpoint/model is set. + ErrNotConfigured = errors.New("vision: adapter not configured") + // ErrUnsupportedSource is returned when an image source cannot be handled. + ErrUnsupportedSource = errors.New("vision: unsupported image source") + // ErrFetch is returned when a URL image cannot be downloaded. + ErrFetch = errors.New("vision: cannot fetch image") + // ErrUpstream is returned when the vision/OCR provider request fails. + ErrUpstream = errors.New("vision: upstream request failed") + // ErrBadImage is returned when an image cannot be read or decoded. + ErrBadImage = errors.New("vision: cannot read image") + // ErrBadArgs is returned by a tool when its arguments are malformed. + ErrBadArgs = errors.New("vision: bad tool arguments") +) diff --git a/internal/runtime/multimodal/vision/image.go b/internal/runtime/multimodal/vision/image.go new file mode 100644 index 0000000..5540278 --- /dev/null +++ b/internal/runtime/multimodal/vision/image.go @@ -0,0 +1,178 @@ +package vision + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + "os" + "strings" +) + +// Size caps for reading image bytes and response bodies, so a hostile or +// malformed source cannot exhaust memory. The 50MB cap matches the file-upload +// limit in DevDocs/02-development-constraints.md §11. +const ( + maxImageBytes = 50 << 20 + maxFetchedBody = 50 << 20 +) + +// DataURI builds a base64 data: image URI from mime and raw bytes. +func DataURI(mime string, data []byte) string { + return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data) +} + +// Resolve loads the bytes for an ImageRef. Paths and base64 payloads are read +// locally; URLs are fetched with an http.Client (the provided client, or a +// default one when nil). For base64 input the Data field is returned verbatim as +// the payload and the MIME type is guessed from a data: prefix. +func Resolve(ctx context.Context, ref ImageRef, client *http.Client) (mime string, data []byte, err error) { + switch ref.Source { + case SourcePath: + return readFile(ref.Data) + case SourceBase64: + return decodeBase64(ref.Data) + case SourceURL: + return fetchURL(ctx, ref.Data, client) + default: + return "", nil, fmt.Errorf("vision: %w: %q", ErrUnsupportedSource, ref.Source) + } +} + +// readFile reads an image file and guesses its MIME type from the extension. +func readFile(path string) (string, []byte, error) { + f, err := os.Open(path) + if err != nil { + return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err) + } + defer func() { _ = f.Close() }() + data, err := io.ReadAll(io.LimitReader(f, maxImageBytes)) + if err != nil { + return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err) + } + return mimeByExt(path), data, nil +} + +// decodeBase64 returns the decoded bytes of a base64/data: payload and its MIME. +func decodeBase64(src string) (string, []byte, error) { + mime, encoded, isDataURI := splitDataURI(src) + if isDataURI { + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err) + } + return mime, decoded, nil + } + decoded, err := base64.StdEncoding.DecodeString(src) + if err != nil { + return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err) + } + return "image/png", decoded, nil +} + +// fetchURL downloads an image over http(s); only http/https schemes are allowed. +func fetchURL(ctx context.Context, rawURL string, client *http.Client) (string, []byte, error) { + if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") { + return "", nil, fmt.Errorf("vision: %w: %q", ErrUnsupportedSource, rawURL) + } + if client == nil { + client = defaultHTTPClient() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err) + } + resp, err := client.Do(req) + if err != nil { + return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return "", nil, fmt.Errorf("vision: %w: status %d", ErrFetch, resp.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxFetchedBody)) + if err != nil { + return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err) + } + return mimeByContentType(resp.Header.Get("Content-Type")), data, nil +} + +// splitDataURI parses a "data:;base64," prefix. The second return +// is false when src is not a data: URI. +func splitDataURI(src string) (mime, payload string, ok bool) { + if !strings.HasPrefix(src, "data:") { + return "", "", false + } + rest := strings.TrimPrefix(src, "data:") + comma := strings.IndexByte(rest, ',') + if comma < 0 { + return "", "", false + } + meta := rest[:comma] + payload = rest[comma+1:] + mime = "image/png" + if strings.HasPrefix(meta, ";base64") { + return mime, payload, true + } + if idx := strings.Index(meta, ";base64"); idx >= 0 { + if t := strings.TrimSuffix(meta[:idx], ";base64"); t != "" { + mime = t + } + return mime, payload, true + } + return mime, payload, true +} + +// mimeByExt guesses a MIME type from a file path extension. +func mimeByExt(path string) string { + switch strings.ToLower(pathExtension(path)) { + case ".png": + return "image/png" + case ".jpg", ".jpeg": + return "image/jpeg" + case ".gif": + return "image/gif" + case ".webp": + return "image/webp" + default: + return "image/png" + } +} + +// mimeByContentType returns a MIME type from an HTTP Content-Type header, +// defaulting to image/png when absent or non-image. +func mimeByContentType(ct string) string { + if ct == "" || !strings.HasPrefix(ct, "image/") { + return "image/png" + } + return strings.TrimSpace(strings.Split(ct, ";")[0]) +} + +// pathExtension returns the file extension with its dot, or empty when none. +func pathExtension(path string) string { + base := path + if idx := strings.LastIndexByte(path, '/'); idx >= 0 { + base = path[idx+1:] + } else if idx := strings.LastIndexByte(path, '\\'); idx >= 0 { + base = path[idx+1:] + } + dot := strings.LastIndexByte(base, '.') + if dot < 0 { + return "" + } + return base[dot:] +} + +// fileExists reports whether path exists on disk and is a regular file. +func fileExists(path string) bool { + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() +} + +// defaultHTTPClient builds the HTTP client described in +// DevDocs/02-development-constraints.md §2.3. A fresh client is returned per +// call rather than a package global, which the constraints forbid. +func defaultHTTPClient() *http.Client { + return &http.Client{Transport: http.DefaultTransport} +} diff --git a/internal/runtime/multimodal/vision/router.go b/internal/runtime/multimodal/vision/router.go new file mode 100644 index 0000000..006399c --- /dev/null +++ b/internal/runtime/multimodal/vision/router.go @@ -0,0 +1,128 @@ +package vision + +// Assessment is the model's own confidence in reading an image. +type Assessment string + +const ( + // AssessmentUnknown means the model reported no self-assessment. + AssessmentUnknown Assessment = "unknown" + // AssessmentGood means the model is confident in its read. + AssessmentGood Assessment = "good" + // AssessmentPoor means the model judged its recognition as poor. + AssessmentPoor Assessment = "poor" +) + +// ModelInfo describes the main model's vision capability. +type ModelInfo struct { + // Name identifies the model. + Name string + // SupportsVision reports whether the model accepts image input. + SupportsVision bool + // SelfAssessment is the model's own confidence for the current image. + // It drives the layer-2 tool route when SupportsVision is true. + SelfAssessment Assessment +} + +// DecisionKind is the chosen layered route. +type DecisionKind string + +const ( + // KindMain routes the image to the main model with the image in context. + KindMain DecisionKind = "main" + // KindVisionTool routes to a dedicated vision tool, returning only the tool + // result so the image never occupies main-model context. + KindVisionTool DecisionKind = "vision_tool" + // KindOCRTool routes to an OCR tool, returning only the extracted text. + KindOCRTool DecisionKind = "ocr_tool" + // KindUnsupported means no route can process the image. + KindUnsupported DecisionKind = "unsupported" +) + +// Decision is the outcome of a routing call. +type Decision struct { + // Kind is the chosen route. + Kind DecisionKind + // Tool names the vision or OCR tool to invoke, set when Kind is a tool route. + Tool string + // Reason is a human-readable explanation for observability and logs. + Reason string +} + +// RouteRequest carries routing hints from the caller on top of the model info. +type RouteRequest struct { + // TextPrimary reports that the user indicated the image is mainly text, so + // an OCR tool is preferred over a general vision model. + TextPrimary bool + // MainSelfAssessedPoor is set when the main model has already judged its own + // read of the image as poor, requesting a vision/OCR tool route. + MainSelfAssessedPoor bool +} + +// RouterOption configures a Router. +type RouterOption func(*Router) + +// WithVisionTool binds the name of the registered vision-understanding tool. +func WithVisionTool(name string) RouterOption { + return func(r *Router) { + if name != "" { + r.visionTool = name + } + } +} + +// WithOCRTool binds the name of the registered OCR tool. +func WithOCRTool(name string) RouterOption { + return func(r *Router) { + if name != "" { + r.ocrTool = name + } + } +} + +// Router implements the layered vision routing policy. It does no I/O; the +// caller uses the returned Decision to dispatch the request. +type Router struct { + visionTool string + ocrTool string +} + +// NewRouter builds a Router. Unknown (empty) tool names leave the corresponding +// route unavailable, so the router degrades to KindMain or KindUnsupported. +func NewRouter(opts ...RouterOption) *Router { + r := &Router{} + for _, o := range opts { + o(r) + } + return r +} + +// Route picks a layered route for an image request given the target model info +// and caller hints. The order is: the main model directly when it can see and +// has not self-assessed poorly (layer 1); otherwise a vision or OCR tool whose +// result replaces the image in context (layers 2 and 3); otherwise +// KindUnsupported when nothing can process the image. +func (r *Router) Route(target ModelInfo, req RouteRequest) Decision { + poor := target.SelfAssessment == AssessmentPoor || req.MainSelfAssessedPoor + // Layer 1: the main model accepts images and is confident, so use it first. + if target.SupportsVision && !poor { + return Decision{ + Kind: KindMain, + Reason: "main model supports vision and is confident; use it directly", + } + } + // OCR is preferred when the user says the image is mostly text. + if req.TextPrimary && r.ocrTool != "" { + return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "image is mainly text; route to OCR"} + } + if r.visionTool != "" { + return Decision{Kind: KindVisionTool, Tool: r.visionTool, Reason: "route to dedicated vision tool"} + } + if r.ocrTool != "" { + return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "route to OCR tool"} + } + // No tool is bound. A sighted main model remains the only option. + if target.SupportsVision { + return Decision{Kind: KindMain, Reason: "no vision/OCR tool bound; keep image with main model"} + } + return Decision{Kind: KindUnsupported, Reason: "main model cannot see and no vision/OCR tool is bound"} +} diff --git a/internal/runtime/multimodal/vision/router_test.go b/internal/runtime/multimodal/vision/router_test.go new file mode 100644 index 0000000..5ac26a1 --- /dev/null +++ b/internal/runtime/multimodal/vision/router_test.go @@ -0,0 +1,178 @@ +package vision + +import ( + "context" + "testing" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/capability/registry" +) + +func TestRouter_Route(t *testing.T) { + t.Parallel() + cases := []struct { + name string + router *Router + target ModelInfo + req RouteRequest + want DecisionKind + tool string + }{ + { + name: "sighted confident main -> main", + router: NewRouter(WithVisionTool("v"), WithOCRTool("o")), + target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentGood}, + want: KindMain, + }, + { + name: "poor self-assessment sighted main -> vision tool", + router: NewRouter(WithVisionTool("v"), WithOCRTool("o")), + target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor}, + req: RouteRequest{MainSelfAssessedPoor: true}, + want: KindVisionTool, + tool: "v", + }, + { + name: "poor self-assessment only OCR -> OCR tool", + router: NewRouter(WithOCRTool("o")), + target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor}, + want: KindOCRTool, + tool: "o", + }, + { + name: "blind main with vision tool -> vision tool", + router: NewRouter(WithVisionTool("v")), + target: ModelInfo{Name: "m", SupportsVision: false}, + want: KindVisionTool, + tool: "v", + }, + { + name: "blind main text primary -> OCR tool", + router: NewRouter(WithVisionTool("v"), WithOCRTool("o")), + target: ModelInfo{Name: "m", SupportsVision: false}, + req: RouteRequest{TextPrimary: true}, + want: KindOCRTool, + tool: "o", + }, + { + name: "blind main only OCR non-text -> OCR fallback", + router: NewRouter(WithOCRTool("o")), + target: ModelInfo{Name: "m", SupportsVision: false}, + want: KindOCRTool, + tool: "o", + }, + { + name: "blind main no tools -> unsupported", + router: NewRouter(), + target: ModelInfo{Name: "m", SupportsVision: false}, + want: KindUnsupported, + }, + { + name: "sighted main no tools -> main", + router: NewRouter(), + target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentGood}, + want: KindMain, + }, + { + name: "sighted main poor no tools -> main fallback", + router: NewRouter(), + target: ModelInfo{Name: "m", SupportsVision: true, SelfAssessment: AssessmentPoor}, + want: KindMain, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := tc.router.Route(tc.target, tc.req) + if got.Kind != tc.want { + t.Fatalf("Kind = %q, want %q (reason: %s)", got.Kind, tc.want, got.Reason) + } + if tc.tool != "" && got.Tool != tc.tool { + t.Errorf("Tool = %q, want %q", got.Tool, tc.tool) + } + if got.Reason == "" { + t.Error("Reason must be non-empty") + } + }) + } +} + +// fakeVision implements VisionAdapter for tool tests. +type fakeVision struct{ desc string } + +func (f *fakeVision) Describe(_ context.Context, _ ImageRef, _ string) (string, error) { + return f.desc, nil +} + +// fakeOCR implements OCRAdapter for tool tests. +type fakeOCR struct{ text string } + +func (f *fakeOCR) Extract(_ context.Context, _ ImageRef) (string, error) { + return f.text, nil +} + +func TestRegisterTools(t *testing.T) { + t.Parallel() + reg := registry.New() + v := NewVisionTool("vision_understand", &fakeVision{desc: "a cat"}) + o := NewOCRTool("ocr_extract", &fakeOCR{text: "HELLO"}) + + if err := RegisterTools(reg, v, o); err != nil { + t.Fatalf("RegisterTools error: %v", err) + } + if capv, err := reg.Get("vision_understand"); err != nil || capv.Kind != registry.KindTool { + t.Fatalf("Get(vision_understand) = %+v, %v", capv, err) + } + if capo, err := reg.Get("ocr_extract"); err != nil || capo.Kind != registry.KindTool { + t.Fatalf("Get(ocr_extract) = %+v, %v", capo, err) + } + if len(reg.List(registry.KindTool)) != 2 { + t.Errorf("List(tool) = %d, want 2", len(reg.List(registry.KindTool))) + } +} + +func TestRegisterTools_NilRegistrar(t *testing.T) { + t.Parallel() + if err := RegisterTools(nil, nil, nil); err == nil { + t.Fatal("want error for nil registrar, got nil") + } +} + +func TestVisionTool_Execute(t *testing.T) { + t.Parallel() + v := NewVisionTool("vision_understand", &fakeVision{desc: "a cat"}) + res, err := v.Execute(context.Background(), map[string]any{ + "image": "data:image/png;base64,aGVsbG8=", + "prompt": "what is this", + }) + if err != nil { + t.Fatalf("Execute error: %v", err) + } + m, ok := res.(map[string]any) + if !ok || m["description"] != "a cat" { + t.Errorf("result = %v, want description=a cat", res) + } +} + +func TestVisionTool_Execute_BadArgs(t *testing.T) { + t.Parallel() + v := NewVisionTool("vision_understand", &fakeVision{desc: ""}) + if _, err := v.Execute(context.Background(), map[string]any{"prompt": "x"}); err == nil { + t.Fatal("want error for missing image, got nil") + } + if _, err := v.Execute(context.Background(), "not-a-map"); err == nil { + t.Fatal("want error for non-map args, got nil") + } +} + +func TestOCRTool_Execute(t *testing.T) { + t.Parallel() + o := NewOCRTool("ocr_extract", &fakeOCR{text: "HELLO"}) + res, err := o.Execute(context.Background(), map[string]any{"image": "aGVsbG8="}) + if err != nil { + t.Fatalf("Execute error: %v", err) + } + m, ok := res.(map[string]any) + if !ok || m["text"] != "HELLO" { + t.Errorf("result = %v, want text=HELLO", res) + } +} diff --git a/internal/runtime/multimodal/vision/tool.go b/internal/runtime/multimodal/vision/tool.go new file mode 100644 index 0000000..02ed7cd --- /dev/null +++ b/internal/runtime/multimodal/vision/tool.go @@ -0,0 +1,148 @@ +package vision + +import ( + "context" + "fmt" + + "git.yeij.top/AskaEth/Cyrene_v2/pkg/plugins/sdk" +) + +// toolParameters returns the shared input JSON Schema for the vision and OCR +// tools. It builds a fresh map each call rather than sharing a package-level +// variable. +func toolParameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "image": map[string]any{ + "type": "string", + "description": "The image: a local path, an http(s) URL, or a base64/data: URI.", + }, + "prompt": map[string]any{ + "type": "string", + "description": "Optional instruction for the vision model (vision tool only).", + }, + }, + "required": []any{"image"}, + } +} + +// VisionTool wraps a VisionAdapter as an sdk.Tool so the M2 tool loop can invoke +// it and return its description as a tool result, keeping the image out of the +// main-model context. It is safe for concurrent use. +type VisionTool struct { + name string + adapter VisionAdapter +} + +// NewVisionTool builds a VisionTool with the given name and adapter. +func NewVisionTool(name string, adapter VisionAdapter) *VisionTool { + return &VisionTool{name: name, adapter: adapter} +} + +// Name returns the tool identifier the agent uses to invoke it. +func (t *VisionTool) Name() string { return t.name } + +// Description explains what the tool does. +func (t *VisionTool) Description() string { + return "Understand an image and return a description of its contents." +} + +// Parameters returns the input JSON Schema. +func (t *VisionTool) Parameters() (map[string]any, error) { + return toolParameters(), nil +} + +// Execute runs the adapter on the image from args and returns the description. +func (t *VisionTool) Execute(ctx context.Context, args any) (any, error) { + image, prompt, err := toolArgs(args) + if err != nil { + return nil, err + } + ref, err := ParseImageRef(image) + if err != nil { + return nil, err + } + desc, err := t.adapter.Describe(ctx, ref, prompt) + if err != nil { + return nil, err + } + return map[string]any{"description": desc}, nil +} + +// OCRTool wraps an OCRAdapter as an sdk.Tool. It returns the extracted text as a +// tool result. It is safe for concurrent use. +type OCRTool struct { + name string + adapter OCRAdapter +} + +// NewOCRTool builds an OCRTool with the given name and adapter. +func NewOCRTool(name string, adapter OCRAdapter) *OCRTool { + return &OCRTool{name: name, adapter: adapter} +} + +// Name returns the tool identifier. +func (t *OCRTool) Name() string { return t.name } + +// Description explains what the tool does. +func (t *OCRTool) Description() string { + return "Extract text from an image (OCR) and return the recognized text." +} + +// Parameters returns the input JSON Schema. +func (t *OCRTool) Parameters() (map[string]any, error) { + return toolParameters(), nil +} + +// Execute runs the OCR adapter on the image from args and returns the text. +func (t *OCRTool) Execute(ctx context.Context, args any) (any, error) { + image, _, err := toolArgs(args) + if err != nil { + return nil, err + } + ref, err := ParseImageRef(image) + if err != nil { + return nil, err + } + text, err := t.adapter.Extract(ctx, ref) + if err != nil { + return nil, err + } + return map[string]any{"text": text}, nil +} + +// RegisterTools registers the vision and OCR tools with r via the SDK helper. +// Each tool is only registered when its pointer is non-nil. It returns the +// first registration error, or nil when every tool registered. +func RegisterTools(r sdk.Registrar, tool *VisionTool, ocr *OCRTool) error { + if r == nil { + return sdk.ErrNilRegistrar + } + if tool != nil { + if err := sdk.RegisterTool(r, tool); err != nil { + return fmt.Errorf("vision: register tool %q: %w", tool.Name(), err) + } + } + if ocr != nil { + if err := sdk.RegisterTool(r, ocr); err != nil { + return fmt.Errorf("vision: register tool %q: %w", ocr.Name(), err) + } + } + return nil +} + +// toolArgs decodes sdk.Tool.Execute args into the image string and an optional +// prompt. Args must be a map[string]any with a non-empty "image" string. +func toolArgs(args any) (image, prompt string, err error) { + m, ok := args.(map[string]any) + if !ok { + return "", "", fmt.Errorf("vision: %w: args must be an object", ErrBadArgs) + } + image, _ = m["image"].(string) + if image == "" { + return "", "", fmt.Errorf("vision: %w: missing image", ErrBadArgs) + } + prompt, _ = m["prompt"].(string) + return image, prompt, nil +} diff --git a/internal/runtime/multimodal/vision/vision.go b/internal/runtime/multimodal/vision/vision.go new file mode 100644 index 0000000..42bf06f --- /dev/null +++ b/internal/runtime/multimodal/vision/vision.go @@ -0,0 +1,79 @@ +// Package vision implements the layered image routing described in +// DevDocs/design/06-多模态.md §0. +// +// A request that carries image content is first checked by DetectMultimodal. +// The VisionRouter then picks one of the layered routes: the main model +// directly (when it accepts images), a tool-shaped vision/OCR adapter (when the +// main model self-assesses poorly or cannot see), or the OCR tool (when the +// image is mainly text). Vision/OCR adapters are registered as sdk.Tool +// capabilities through the capability registry, and their output is a tool +// result, so the image itself never occupies main-model context. +// +// Images may come from a local path, an http(s) URL, or a base64/data: URI. +package vision + +import ( + "strings" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model" +) + +// DetectMultimodal reports whether msgs contains any non-text content part, +// i.e. whether multimodal handling is required. Today model only distinguishes +// text from image content, so this is true exactly when a message carries an +// image part; the check is written against "not text" so future content types +// (audio) extend it without rework. +func DetectMultimodal(msgs []model.Message) bool { + for _, m := range msgs { + for _, part := range m.Content { + if part.Type != model.ContentTypeText { + return true + } + } + } + return false +} + +// ImageSource classifies where an image's bytes come from. +type ImageSource string + +const ( + // SourcePath is a local filesystem path. + SourcePath ImageSource = "path" + // SourceURL is an http(s):// URL. + SourceURL ImageSource = "url" + // SourceBase64 is a base64 payload, optionally wrapped in a data: URI. + SourceBase64 ImageSource = "base64" +) + +// ImageRef is a single image reference plus its resolved source kind. +type ImageRef struct { + // Source is the classified source kind. + Source ImageSource + // Data is the local path, URL, or base64/data: payload. + Data string +} + +// ParseImageRef classifies a raw image source string into an ImageRef. Empty +// input is rejected. Local paths that exist on disk are classified as paths; +// http(s) URLs as URLs; every other value, including a data: URI, is treated as +// a base64 payload. This keeps routing deterministic without guessing MIME. +func ParseImageRef(src string) (ImageRef, error) { + if src == "" { + return ImageRef{}, ErrNoImage + } + if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { + return ImageRef{Source: SourceURL, Data: src}, nil + } + if strings.HasPrefix(src, "file://") { + return ImageRef{Source: SourcePath, Data: strings.TrimPrefix(src, "file://")}, nil + } + // A data: URI is base64 already; treat as base64, not path. + if strings.HasPrefix(src, "data:") { + return ImageRef{Source: SourceBase64, Data: src}, nil + } + if fileExists(src) { + return ImageRef{Source: SourcePath, Data: src}, nil + } + return ImageRef{Source: SourceBase64, Data: src}, nil +} diff --git a/internal/runtime/multimodal/vision/vision_test.go b/internal/runtime/multimodal/vision/vision_test.go new file mode 100644 index 0000000..8844bf3 --- /dev/null +++ b/internal/runtime/multimodal/vision/vision_test.go @@ -0,0 +1,95 @@ +package vision + +import ( + "context" + "strings" + "testing" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model" +) + +func TestDetectMultimodal(t *testing.T) { + t.Parallel() + tests := []struct { + name string + msgs []model.Message + want bool + }{ + {name: "text only", msgs: []model.Message{model.TextMessage(model.RoleUser, "hi")}, want: false}, + {name: "empty", msgs: nil, want: false}, + {name: "single image", msgs: []model.Message{model.ImageMessage(model.RoleUser, "what", "data:image/png;base64,abc")}, want: true}, + {name: "image no text", msgs: []model.Message{model.ImageMessage(model.RoleUser, "", "https://x/y.png")}, want: true}, + {name: "image in later turn", msgs: []model.Message{ + model.TextMessage(model.RoleUser, "hi"), + model.ImageMessage(model.RoleAssistant, "", "http://x/z.png"), + }, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := DetectMultimodal(tt.msgs); got != tt.want { + t.Errorf("DetectMultimodal() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseImageRef(t *testing.T) { + t.Parallel() + tests := []struct { + name string + src string + wantSource ImageSource + }{ + {name: "http url", src: "https://a/b.png", wantSource: SourceURL}, + {name: "https url", src: "https://a/b.png", wantSource: SourceURL}, + {name: "file uri", src: "file:///tmp/a.png", wantSource: SourcePath}, + {name: "data uri", src: "data:image/png;base64,abc", wantSource: SourceBase64}, + {name: "raw base64", src: "aGVsbG8=", wantSource: SourceBase64}, + {name: "empty rejected", src: "", wantSource: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ref, err := ParseImageRef(tt.src) + if tt.src == "" { + if err == nil { + t.Fatal("want error for empty source, got nil") + } + return + } + if err != nil { + t.Fatalf("ParseImageRef error: %v", err) + } + if ref.Source != tt.wantSource { + t.Errorf("Source = %q, want %q", ref.Source, tt.wantSource) + } + }) + } +} + +func TestResolve_Base64(t *testing.T) { + t.Parallel() + ref, err := ParseImageRef("data:image/png;base64,aGVsbG8=") + if err != nil { + t.Fatalf("ParseImageRef error: %v", err) + } + mime, data, err := Resolve(context.Background(), ref, nil) + if err != nil { + t.Fatalf("Resolve error: %v", err) + } + if mime != "image/png" { + t.Errorf("mime = %q, want image/png", mime) + } + if string(data) != "hello" { + t.Errorf("data = %q, want hello", data) + } +} + +func TestResolve_DataURI(t *testing.T) { + t.Parallel() + got := DataURI("image/webp", []byte{1, 2, 3}) + if !strings.HasPrefix(got, "data:image/webp;base64,") { + t.Errorf("DataURI = %q, want data:image/webp;base64,", got) + } +} diff --git a/internal/runtime/multimodal/voice/chain.go b/internal/runtime/multimodal/voice/chain.go new file mode 100644 index 0000000..32048e2 --- /dev/null +++ b/internal/runtime/multimodal/voice/chain.go @@ -0,0 +1,112 @@ +package voice + +import ( + "context" + "fmt" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/config" +) + +// Chain is a degrading provider list. STT and TTS walk the steps in order, +// returning the first successful result and moving to the next step when a +// provider is unavailable (nil direction) or fails. It implements STT and TTS so +// it can be used anywhere a single provider is expected. +type Chain struct { + steps []Provider +} + +// NewChain builds a degrading chain from the given providers in the order they +// should be tried. Passing a SilentProvider last guarantees the chain always +// succeeds. +func NewChain(steps ...Provider) *Chain { + return &Chain{steps: steps} +} + +// STT walks the chain for a transcription. +func (c *Chain) STT(ctx context.Context, audio Audio) (string, error) { + if len(c.steps) == 0 { + return "", ErrProviderFailed + } + var lastErr error + for _, p := range c.steps { + stt := p.STT() + if stt == nil { + continue + } + text, err := stt.STT(ctx, audio) + if err == nil { + return text, nil + } + lastErr = err + } + return "", fmt.Errorf("voice: %w: %v", ErrProviderFailed, lastErr) +} + +// TTS walks the chain for synthesized audio. +func (c *Chain) TTS(ctx context.Context, text string) (Audio, error) { + if len(c.steps) == 0 { + return Audio{}, ErrProviderFailed + } + var lastErr error + for _, p := range c.steps { + tts := p.TTS() + if tts == nil { + continue + } + audio, err := tts.TTS(ctx, text) + if err == nil { + return audio, nil + } + lastErr = err + } + return Audio{}, fmt.Errorf("voice: %w: %v", ErrProviderFailed, lastErr) +} + +// Option customizes a chain built by New. +type Option func(*chainConfig) + +// chainConfig carries the assembled steps for New. +type chainConfig struct { + steps []Provider +} + +// WithProviders overrides the provider steps assembled from config. Tests use it +// to inject deterministic mock providers instead of the real adaptation-point +// rungs. +func WithProviders(steps ...Provider) Option { + return func(c *chainConfig) { c.steps = steps } +} + +// New assembles a chain from cfg using the fixed degradation order +// cloud → local → edge-tts/espeak → silent. +// +// - cloud is included only when a cloud voice setting is present (today an +// API key) so a mis-configured build degrades quietly. +// - local is a pure adaptation point and is only included when supplied via +// WithProviders; by default it is skipped. +// - the edge-tts or espeak rung is included according to cfg.TTSProvider. +// - silent is always appended last so the chain never fails outright. +// +// opts may supply a fully assembled step list to override this mapping. +func New(cfg config.VoiceConfig, opts ...Option) *Chain { + conf := &chainConfig{} + for _, o := range opts { + o(conf) + } + if len(conf.steps) > 0 { + return NewChain(conf.steps...) + } + + steps := make([]Provider, 0, 5) + if cfg.DashscopeAPIKey != "" { + steps = append(steps, CloudProvider(newDashboardSTT(cfg), newDashboardTTS(cfg))) + } + switch cfg.TTSProvider { + case "edge-tts", "edge": + steps = append(steps, EdgeTTSProvider(newEdgeTTS())) + case "espeak": + steps = append(steps, EspeakProvider(newEspeak())) + } + steps = append(steps, SilentProvider()) + return NewChain(steps...) +} diff --git a/internal/runtime/multimodal/voice/chain_test.go b/internal/runtime/multimodal/voice/chain_test.go new file mode 100644 index 0000000..731d02b --- /dev/null +++ b/internal/runtime/multimodal/voice/chain_test.go @@ -0,0 +1,186 @@ +package voice + +import ( + "context" + "errors" + "testing" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/config" +) + +// stubSTT is a mock STT returning a fixed result. +type stubSTT struct { + text string + err error +} + +func (s *stubSTT) STT(_ context.Context, _ Audio) (string, error) { return s.text, s.err } + +// stubTTS is a mock TTS returning a fixed result. +type stubTTS struct { + audio Audio + err error +} + +func (s *stubTTS) TTS(_ context.Context, _ string) (Audio, error) { return s.audio, s.err } + +// stubProvider is a mock Provider carrying optional directions. +type stubProvider struct { + name string + stt STT + tts TTS +} + +func (s *stubProvider) Name() string { return s.name } +func (s *stubProvider) STT() STT { return s.stt } +func (s *stubProvider) TTS() TTS { return s.tts } + +func TestChain_STT_Degrades(t *testing.T) { + t.Parallel() + tests := []struct { + name string + steps []Provider + want string + wantErr bool + }{ + { + name: "first provider succeeds", + steps: []Provider{&stubProvider{name: "cloud", stt: &stubSTT{text: "hello"}}}, + want: "hello", + }, + { + name: "first fails second succeeds", + steps: []Provider{ + &stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}}, + &stubProvider{name: "local", stt: &stubSTT{text: "degraded"}}, + }, + want: "degraded", + }, + { + name: "nil direction skipped", + steps: []Provider{ + &stubProvider{name: "edge", stt: nil}, + &stubProvider{name: "silent", stt: &stubSTT{text: "silence"}}, + }, + want: "silence", + }, + { + name: "all fail no silent -> error", + steps: []Provider{ + &stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}}, + }, + wantErr: true, + }, + { + name: "all fail silent last -> no error", + steps: []Provider{ + &stubProvider{name: "cloud", stt: &stubSTT{err: ErrNotConfigured}}, + SilentProvider(), + }, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := NewChain(tt.steps...).STT(context.Background(), Audio{Format: "wav", Data: []byte("x")}) + if tt.wantErr { + if !errors.Is(err, ErrProviderFailed) { + t.Fatalf("err = %v, want ErrProviderFailed", err) + } + return + } + if err != nil { + t.Fatalf("STT error: %v", err) + } + if got != tt.want { + t.Errorf("text = %q, want %q", got, tt.want) + } + }) + } +} + +func TestChain_TTS_Degrades(t *testing.T) { + t.Parallel() + tests := []struct { + name string + steps []Provider + want Audio + wantErr bool + }{ + { + name: "first provider succeeds", + steps: []Provider{&stubProvider{name: "cloud", tts: &stubTTS{audio: Audio{Format: "mp3", Data: []byte("a")}}}}, + want: Audio{Format: "mp3", Data: []byte("a")}, + }, + { + name: "first fails second succeeds", + steps: []Provider{ + &stubProvider{name: "cloud", tts: &stubTTS{err: ErrNotConfigured}}, + &stubProvider{name: "edge-tts", tts: &stubTTS{audio: Audio{Format: "mp3", Data: []byte("b")}}}, + }, + want: Audio{Format: "mp3", Data: []byte("b")}, + }, + { + name: "all fail no silent -> error", + steps: []Provider{ + &stubProvider{name: "cloud", tts: &stubTTS{err: ErrNotConfigured}}, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := NewChain(tt.steps...).TTS(context.Background(), "hi") + if tt.wantErr { + if !errors.Is(err, ErrProviderFailed) { + t.Fatalf("err = %v, want ErrProviderFailed", err) + } + return + } + if err != nil { + t.Fatalf("TTS error: %v", err) + } + if got.Format != tt.want.Format || string(got.Data) != string(tt.want.Data) { + t.Errorf("audio = %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestChain_Empty(t *testing.T) { + t.Parallel() + if _, err := NewChain().STT(context.Background(), Audio{}); !errors.Is(err, ErrProviderFailed) { + t.Fatalf("err = %v, want ErrProviderFailed", err) + } +} + +func TestNew_ConfigAssembly(t *testing.T) { + t.Parallel() + // With a cloud key and default edge-tts TTS, the chain starts at cloud and + // ends at silent. + c := New(config.VoiceConfig{DashscopeAPIKey: "k"}) + names := make([]string, 0, len(c.steps)) + for _, p := range c.steps { + names = append(names, p.Name()) + } + if len(names) < 2 || names[len(names)-1] != "silent" { + t.Fatalf("steps = %v, want to end at silent", names) + } + if names[0] != "cloud" { + t.Errorf("first step = %q, want cloud", names[0]) + } + + // Without a key the cloud rung is skipped. + c2 := New(config.VoiceConfig{}) + foundCloud := false + for _, p := range c2.steps { + if p.Name() == "cloud" { + foundCloud = true + } + } + if foundCloud { + t.Error("cloud step present without an API key") + } +} diff --git a/internal/runtime/multimodal/voice/cloud.go b/internal/runtime/multimodal/voice/cloud.go new file mode 100644 index 0000000..1e0e7be --- /dev/null +++ b/internal/runtime/multimodal/voice/cloud.go @@ -0,0 +1,227 @@ +package voice + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + "time" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/config" +) + +// cloudMaxResponseBytes caps how much of a cloud voice response is read, so an +// oversized or malformed response cannot exhaust memory. The request timeout +// bounds a single call. +const ( + cloudMaxResponseBytes = 8 << 20 + cloudHTTPTimeout = 180 * time.Second +) + +// HTTPSTT transcribes audio by POSTing a JSON body to a cloud endpoint and +// reading {"text": "..."} back. It is the stubbed default STT implementation; +// the real DashScope/whisper-facing adapter is an adaptation point. +type HTTPSTT struct { + endpoint string + apiKey string + client *http.Client + log *slog.Logger +} + +// HTTPTTS synthesizes speech by POSTing a JSON body and reading the returned +// base64 audio. It is the stubbed default TTS implementation. +type HTTPTTS struct { + endpoint string + apiKey string + client *http.Client + log *slog.Logger +} + +// HTTPOption customizes the client and logger of an HTTPSTT/HTTPTTS. +type HTTPOption func(*httpConfig) + +type httpConfig struct { + client *http.Client + log *slog.Logger +} + +// WithHTTPClient sets the *http.Client used for the call; nil keeps the default. +func WithHTTPClient(c *http.Client) HTTPOption { + return func(o *httpConfig) { + if c != nil { + o.client = c + } + } +} + +// WithLogger sets the logger used on failures; nil keeps a discard logger. +func WithLogger(l *slog.Logger) HTTPOption { + return func(o *httpConfig) { + if l != nil { + o.log = l + } + } +} + +// NewHTTPSTT builds an HTTPSTT for the given endpoint and API key. +func NewHTTPSTT(endpoint, apiKey string, opts ...HTTPOption) *HTTPSTT { + h := &httpConfig{client: defaultHTTPClient(), log: discardLogger()} + for _, o := range opts { + o(h) + } + return &HTTPSTT{endpoint: strings.TrimRight(endpoint, "/"), apiKey: apiKey, client: h.client, log: h.log} +} + +// NewHTTPTTS builds an HTTPTTS for the given endpoint and API key. +func NewHTTPTTS(endpoint, apiKey string, opts ...HTTPOption) *HTTPTTS { + h := &httpConfig{client: defaultHTTPClient(), log: discardLogger()} + for _, o := range opts { + o(h) + } + return &HTTPTTS{endpoint: strings.TrimRight(endpoint, "/"), apiKey: apiKey, client: h.client, log: h.log} +} + +// sttRequest is the JSON body sent to the STT endpoint. +type sttRequest struct { + Audio string `json:"audio"` + Format string `json:"format"` +} + +// sttResponse is the transcription returned by the STT endpoint. +type sttResponse struct { + Text string `json:"text"` +} + +// STT posts the audio and returns the recognized text. A blank endpoint returns +// ErrNotConfigured so the chain degrades. +func (s *HTTPSTT) STT(ctx context.Context, audio Audio) (string, error) { + if s.endpoint == "" { + return "", ErrNotConfigured + } + if err := ctx.Err(); err != nil { + return "", err + } + payload, err := json.Marshal(sttRequest{Audio: base64.StdEncoding.EncodeToString(audio.Data), Format: audio.Format}) + if err != nil { + return "", fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err) + } + resp, err := postJSON(ctx, s.client, s.apiKey, s.endpoint, payload) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + var decoded sttResponse + if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil { + return "", fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err) + } + return decoded.Text, nil +} + +// ttsRequest is the JSON body sent to the TTS endpoint. +type ttsRequest struct { + Text string `json:"text"` +} + +// ttsResponse is the synthesized audio returned by the TTS endpoint. +type ttsResponse struct { + Audio string `json:"audio"` + Format string `json:"format"` +} + +// TTS posts the text and returns the decoded audio. A blank endpoint returns +// ErrNotConfigured so the chain degrades. +func (s *HTTPTTS) TTS(ctx context.Context, text string) (Audio, error) { + if s.endpoint == "" { + return Audio{}, ErrNotConfigured + } + if strings.TrimSpace(text) == "" { + return Audio{}, ErrEmptyText + } + if err := ctx.Err(); err != nil { + return Audio{}, err + } + payload, err := json.Marshal(ttsRequest{Text: text}) + if err != nil { + return Audio{}, fmt.Errorf("voice: %w: marshal: %v", ErrProviderFailed, err) + } + resp, err := postJSON(ctx, s.client, s.apiKey, s.endpoint, payload) + if err != nil { + return Audio{}, err + } + defer func() { _ = resp.Body.Close() }() + var decoded ttsResponse + if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil { + return Audio{}, fmt.Errorf("voice: %w: decode: %v", ErrProviderFailed, err) + } + data, err := base64.StdEncoding.DecodeString(decoded.Audio) + if err != nil { + return Audio{}, fmt.Errorf("voice: %w: decode audio: %v", ErrProviderFailed, err) + } + format := decoded.Format + if format == "" { + format = "mp3" + } + return Audio{Format: format, Data: data}, nil +} + +// postJSON POSTs a JSON payload, wires auth, and checks the status code. +func postJSON(ctx context.Context, client *http.Client, apiKey, url string, payload []byte) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + return nil, fmt.Errorf("voice: %w: build: %v", ErrProviderFailed, err) + } + req.Header.Set("Content-Type", "application/json") + if apiKey != "" { + req.Header.Set("Authorization", "Bearer "+apiKey) + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("voice: %w: request: %v", ErrProviderFailed, err) + } + if resp.StatusCode != http.StatusOK { + _ = resp.Body.Close() + return nil, fmt.Errorf("voice: %w: status %d", ErrProviderFailed, resp.StatusCode) + } + return resp, nil +} + +// defaultHTTPClient builds a client with a request timeout. A fresh client is +// returned per call rather than a package global, which the constraints forbid. +func defaultHTTPClient() *http.Client { + return &http.Client{Transport: http.DefaultTransport, Timeout: cloudHTTPTimeout} +} + +// discardLogger returns a logger that drops all output. +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// newDashboardSTT returns the cloud STT for a config. The endpoint is an +// adaptation point and is empty until config gains a voice URL; a call then +// returns ErrNotConfigured and the chain degrades. +func newDashboardSTT(cfg config.VoiceConfig) STT { + return NewHTTPSTT("", cfg.DashscopeAPIKey) +} + +// newDashboardTTS returns the cloud TTS for a config. +func newDashboardTTS(cfg config.VoiceConfig) TTS { + return NewHTTPTTS("", cfg.DashscopeAPIKey) +} + +// newEdgeTTS returns the edge-tts adapter point (not yet implemented). +func newEdgeTTS() TTS { return notImplementedTTS{} } + +// newEspeak returns the espeak adapter point (not yet implemented). +func newEspeak() TTS { return notImplementedTTS{} } + +// notImplementedTTS is an adaptation point that always degrades. +type notImplementedTTS struct{} + +func (notImplementedTTS) TTS(_ context.Context, _ string) (Audio, error) { + return Audio{}, ErrNotImplemented +} diff --git a/internal/runtime/multimodal/voice/cloud_test.go b/internal/runtime/multimodal/voice/cloud_test.go new file mode 100644 index 0000000..8efa2bd --- /dev/null +++ b/internal/runtime/multimodal/voice/cloud_test.go @@ -0,0 +1,81 @@ +package voice + +import ( + "context" + "encoding/base64" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestHTTPSTT(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("decode: %v", err) + } + if r.Header.Get("Authorization") != "Bearer sk" { + t.Errorf("auth = %q, want Bearer sk", r.Header.Get("Authorization")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"text":"hello world"}`)) + })) + defer srv.Close() + + stt := NewHTTPSTT(srv.URL, "sk") + text, err := stt.STT(context.Background(), Audio{Format: "wav", Data: []byte("raw")}) + if err != nil { + t.Fatalf("STT error: %v", err) + } + if text != "hello world" { + t.Errorf("text = %q, want hello world", text) + } +} + +func TestHTTPSTT_NotConfigured(t *testing.T) { + t.Parallel() + stt := NewHTTPSTT("", "") + if _, err := stt.STT(context.Background(), Audio{}); err != ErrNotConfigured { + t.Fatalf("err = %v, want ErrNotConfigured", err) + } +} + +func TestHTTPTTS(t *testing.T) { + t.Parallel() + audioBytes := []byte{1, 2, 3} + b64 := base64.StdEncoding.EncodeToString(audioBytes) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("decode: %v", err) + } + if body["text"] != "hi" { + t.Errorf("text = %v, want hi", body["text"]) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"audio":"` + b64 + `","format":"wav"}`)) + })) + defer srv.Close() + + tts := NewHTTPTTS(srv.URL, "") + audio, err := tts.TTS(context.Background(), "hi") + if err != nil { + t.Fatalf("TTS error: %v", err) + } + if audio.Format != "wav" || hex.EncodeToString(audio.Data) != hex.EncodeToString(audioBytes) { + t.Errorf("audio = %+v, want wav/010203", audio) + } +} + +func TestHTTPTTS_EmptyText(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer srv.Close() + tts := NewHTTPTTS(srv.URL, "") + if _, err := tts.TTS(context.Background(), " "); err != ErrEmptyText { + t.Fatalf("err = %v, want ErrEmptyText", err) + } +} diff --git a/internal/runtime/multimodal/voice/errors.go b/internal/runtime/multimodal/voice/errors.go new file mode 100644 index 0000000..69ebf7f --- /dev/null +++ b/internal/runtime/multimodal/voice/errors.go @@ -0,0 +1,18 @@ +package voice + +import "errors" + +// Sentinel errors returned by the voice package. Each is wrapped with %w so +// callers can test for a category while keeping the cause. +var ( + // ErrProviderFailed is returned when every provider in a chain fails. + ErrProviderFailed = errors.New("voice: all providers failed") + // ErrNotConfigured is returned when a provider has no endpoint/setting. + ErrNotConfigured = errors.New("voice: provider not configured") + // ErrNotImplemented is returned by an adaptation-point provider. + ErrNotImplemented = errors.New("voice: provider not implemented") + // ErrEmptyText is returned when TTS receives empty text. + ErrEmptyText = errors.New("voice: empty text") + // ErrClosed is returned when a service call arrives after Close. + ErrClosed = errors.New("voice: service closed") +) diff --git a/internal/runtime/multimodal/voice/provider.go b/internal/runtime/multimodal/voice/provider.go new file mode 100644 index 0000000..38b8fb3 --- /dev/null +++ b/internal/runtime/multimodal/voice/provider.go @@ -0,0 +1,79 @@ +package voice + +import "context" + +// Provider is one rung in the voice degradation chain. It may implement zero, +// one, or both directions; a nil return means that direction is not available +// on this rung and the chain moves to the next. +type Provider interface { + // Name returns a stable provider identifier. + Name() string + // STT returns the speech-to-text implementation, or nil when the provider + // cannot transcribe. + STT() STT + // TTS returns the text-to-speech implementation, or nil when the provider + // cannot synthesize. + TTS() TTS +} + +// StaticProvider is a Provider with a fixed name whose STT/TTS are set at +// construction. It is used by the concrete providers below. +type StaticProvider struct { + name string + stt STT + tts TTS +} + +// Name returns the provider identifier. +func (p *StaticProvider) Name() string { return p.name } + +// STT returns the configured STT, or nil. +func (p *StaticProvider) STT() STT { return p.stt } + +// TTS returns the configured TTS, or nil. +func (p *StaticProvider) TTS() TTS { return p.tts } + +// NewProvider builds a Provider with the given name and optional directions it +// supports. +func NewProvider(name string, stt STT, tts TTS) *StaticProvider { + return &StaticProvider{name: name, stt: stt, tts: tts} +} + +// CloudProvider is the cloud rung of the chain. It is an adaptation point: the +// real STT/TTS implementations are wired in via NewProvider with cloud adapters. +func CloudProvider(stt STT, tts TTS) *StaticProvider { + return NewProvider("cloud", stt, tts) +} + +// LocalProvider is the local-model rung of the chain. +func LocalProvider(stt STT, tts TTS) *StaticProvider { + return NewProvider("local", stt, tts) +} + +// EdgeTTSProvider is the edge-tts rung (text-to-speech only). +func EdgeTTSProvider(tts TTS) *StaticProvider { + return NewProvider("edge-tts", nil, tts) +} + +// EspeakProvider is the espeak rung (text-to-speech only). +func EspeakProvider(tts TTS) *StaticProvider { + return NewProvider("espeak", nil, tts) +} + +// SilentProvider is the terminal rung. It always succeeds, returning an empty +// transcription and empty audio, so a chain that includes it never errors. +func SilentProvider() *StaticProvider { + return NewProvider("silent", silenceSTT{}, silenceTTS{}) +} + +// silenceSTT is the silent STT rung; it returns an empty transcription. +type silenceSTT struct{} + +func (silenceSTT) STT(_ context.Context, _ Audio) (string, error) { return "", nil } + +// silenceTTS is the silent TTS rung; it returns empty audio. +type silenceTTS struct{} + +func (silenceTTS) TTS(_ context.Context, _ string) (Audio, error) { + return Audio{}, nil +} diff --git a/internal/runtime/multimodal/voice/service.go b/internal/runtime/multimodal/voice/service.go new file mode 100644 index 0000000..fa6f347 --- /dev/null +++ b/internal/runtime/multimodal/voice/service.go @@ -0,0 +1,160 @@ +package voice + +import ( + "context" + "fmt" + "sync" +) + +// Service runs STT/TTS requests on a background worker pool so caller +// goroutines (the agent loop) are never blocked by provider latency. Each +// request is enqueued on a jobs channel and answered on a per-job result +// channel; a canceled caller context or a closed service unblocks the caller +// immediately and no worker goroutine is leaked. It is safe for concurrent use. +type Service struct { + chain *Chain + jobs chan job + bgCtx context.Context + cancel context.CancelFunc + nWorkers int + wg sync.WaitGroup + mu sync.RWMutex + closed bool +} + +// ServiceOption customizes a Service. +type ServiceOption func(*Service) + +// WithWorkers sets the worker-pool size. Values <= 0 are ignored and the +// default single worker is kept. +func WithWorkers(n int) ServiceOption { + return func(s *Service) { + if n > 0 { + s.nWorkers = n + } + } +} + +// jobKind selects the operation a worker performs. +type jobKind int + +const ( + jobSTT jobKind = iota + jobTTS +) + +// job is one request enqueued by the caller. +type job struct { + kind jobKind + audio Audio + text string + res chan jobResult +} + +// jobResult carries the outcome written back to the caller. +type jobResult struct { + text string + audio Audio + err error +} + +// NewService builds a Service backed by chain and starts the worker pool. +func NewService(chain *Chain, opts ...ServiceOption) *Service { + bg, cancel := context.WithCancel(context.Background()) + s := &Service{ + chain: chain, + jobs: make(chan job), + bgCtx: bg, + cancel: cancel, + nWorkers: 1, + } + for _, o := range opts { + o(s) + } + for i := 0; i < s.nWorkers; i++ { + s.wg.Add(1) + go s.worker() + } + return s +} + +// worker consumes jobs until the service closes. +func (s *Service) worker() { + defer s.wg.Done() + for { + select { + case <-s.bgCtx.Done(): + return + case j := <-s.jobs: + s.run(j) + } + } +} + +// run executes a job and delivers the result. Providers run under the service's +// background context so Close cancels in-flight work and no worker goroutine is +// leaked; the result is written to a buffered channel so a caller that already +// returned never blocks the worker. +func (s *Service) run(j job) { + res := jobResult{} + switch j.kind { + case jobSTT: + res.text, res.err = s.chain.STT(s.bgCtx, j.audio) + case jobTTS: + res.audio, res.err = s.chain.TTS(s.bgCtx, j.text) + } + j.res <- res +} + +// STT enqueues a transcription request and waits for the result. ctx cancels +// the wait and the underlying provider call. +func (s *Service) STT(ctx context.Context, audio Audio) (string, error) { + res, err := s.submit(ctx, job{kind: jobSTT, audio: audio, res: make(chan jobResult, 1)}) + if err != nil { + return "", err + } + return res.text, res.err +} + +// TTS enqueues a synthesis request and waits for the result. ctx cancels the +// wait and the underlying provider call. +func (s *Service) TTS(ctx context.Context, text string) (Audio, error) { + res, err := s.submit(ctx, job{kind: jobTTS, text: text, res: make(chan jobResult, 1)}) + if err != nil { + return Audio{}, err + } + return res.audio, res.err +} + +// submit enqueues j and blocks until the result or a cancellation. +func (s *Service) submit(ctx context.Context, j job) (jobResult, error) { + select { + case s.jobs <- j: + case <-ctx.Done(): + return jobResult{}, fmt.Errorf("voice: %w", ctx.Err()) + case <-s.bgCtx.Done(): + return jobResult{}, ErrClosed + } + select { + case r := <-j.res: + return r, nil + case <-ctx.Done(): + return jobResult{}, fmt.Errorf("voice: %w", ctx.Err()) + case <-s.bgCtx.Done(): + return jobResult{}, ErrClosed + } +} + +// Close cancels the worker pool and waits for in-flight jobs to finish. It is +// safe to call once; later calls are no-ops. In-flight provider calls are +// bounded by each provider's own context timeout, so Close returns promptly. +func (s *Service) Close() { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return + } + s.closed = true + s.cancel() + s.wg.Wait() +} diff --git a/internal/runtime/multimodal/voice/service_test.go b/internal/runtime/multimodal/voice/service_test.go new file mode 100644 index 0000000..417dfa9 --- /dev/null +++ b/internal/runtime/multimodal/voice/service_test.go @@ -0,0 +1,108 @@ +package voice + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +func TestService_STTAndTTS(t *testing.T) { + t.Parallel() + chain := NewChain( + NewProvider("cloud", &stubSTT{text: "transcribed"}, &stubTTS{audio: Audio{Format: "mp3", Data: []byte("a")}}), + ) + svc := NewService(chain) + defer svc.Close() + + text, err := svc.STT(context.Background(), Audio{Format: "wav", Data: []byte("x")}) + if err != nil { + t.Fatalf("STT error: %v", err) + } + if text != "transcribed" { + t.Errorf("text = %q, want transcribed", text) + } + + audio, err := svc.TTS(context.Background(), "hello") + if err != nil { + t.Fatalf("TTS error: %v", err) + } + if string(audio.Data) != "a" || audio.Format != "mp3" { + t.Errorf("audio = %+v, want mp3/a", audio) + } +} + +func TestService_CloseStopsNewCalls(t *testing.T) { + t.Parallel() + svc := NewService(NewChain(SilentProvider())) + svc.Close() + _, err := svc.STT(context.Background(), Audio{}) + if !errors.Is(err, ErrClosed) { + t.Fatalf("err = %v, want ErrClosed", err) + } +} + +// blockingSTT blocks until its context is canceled, marking that it started. +type blockingSTT struct { + started chan struct{} + once sync.Once +} + +func (b *blockingSTT) STT(ctx context.Context, _ Audio) (string, error) { + b.once.Do(func() { close(b.started) }) + <-ctx.Done() + return "", ctx.Err() +} + +func TestService_CloseCancelsInFlightWorker(t *testing.T) { + t.Parallel() + blk := &blockingSTT{started: make(chan struct{})} + chain := NewChain(NewProvider("cloud", blk, nil)) + svc := NewService(chain) + + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = svc.STT(context.Background(), Audio{Format: "wav", Data: []byte("x")}) + }() + + select { + case <-blk.started: + case <-time.After(2 * time.Second): + t.Fatal("provider never started") + } + + svc.Close() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("STT call did not return after Close; worker likely leaked") + } +} + +func TestService_CallerCancelUnblocks(t *testing.T) { + t.Parallel() + blk := &blockingSTT{started: make(chan struct{})} + svc := NewService(NewChain(NewProvider("cloud", blk, nil))) + defer svc.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = svc.STT(ctx, Audio{Format: "wav", Data: []byte("x")}) + }() + + <-blk.started + cancel() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("caller cancellation did not unblock STT") + } +} diff --git a/internal/runtime/multimodal/voice/voice.go b/internal/runtime/multimodal/voice/voice.go new file mode 100644 index 0000000..3637dca --- /dev/null +++ b/internal/runtime/multimodal/voice/voice.go @@ -0,0 +1,35 @@ +// Package voice implements asynchronous speech-to-text and text-to-speech for +// the Cyrene runtime, following DevDocs/design/06-多模态.md §1. +// +// Voice input (STT) is transcribed to text and fed into the same agent pipeline +// as a chat message; the reply is synthesized (TTS) back to audio. Both +// directions degrade through a provider chain — cloud → local → edge-tts/espeak +// → silent — so an unavailable provider falls through to the next rung instead +// of failing the whole turn. The stub default implementations are HTTP-based and +// testable with a stdlib httptest server; real cloud/local providers remain +// adaptation points for later milestones. +package voice + +import "context" + +// Audio is a piece of audio data plus its encoding. +type Audio struct { + // Format is the encoding, e.g. "wav", "mp3", "ogg". + Format string + // Data is the raw audio bytes. + Data []byte +} + +// STT converts audio to text. +type STT interface { + // STT transcribes audio and returns the spoken text. ctx controls + // cancellation and must be honored. + STT(ctx context.Context, audio Audio) (string, error) +} + +// TTS synthesizes speech audio from text. +type TTS interface { + // TTS synthesizes audio from text. ctx controls cancellation and must be + // honored. + TTS(ctx context.Context, text string) (Audio, error) +}