feat(multimodal): 视觉分层路由(多模态检测→主/视觉/OCR 工具)+ 语音 STT→TTS 降级链(异步)
This commit is contained in:
1 parent
02aae2bbb4
commit
0a936a3202
19 files changed
+2121
No files matched your search
@@ -0,0 +1,148 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/pkg/plugins/sdk"
|
||||
)
|
||||
|
||||
// toolParameters returns the shared input JSON Schema for the vision and OCR
|
||||
// tools. It builds a fresh map each call rather than sharing a package-level
|
||||
// variable.
|
||||
func toolParameters() map[string]any {
|
||||
return map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"image": map[string]any{
|
||||
"type": "string",
|
||||
"description": "The image: a local path, an http(s) URL, or a base64/data: URI.",
|
||||
},
|
||||
"prompt": map[string]any{
|
||||
"type": "string",
|
||||
"description": "Optional instruction for the vision model (vision tool only).",
|
||||
},
|
||||
},
|
||||
"required": []any{"image"},
|
||||
}
|
||||
}
|
||||
|
||||
// VisionTool wraps a VisionAdapter as an sdk.Tool so the M2 tool loop can invoke
|
||||
// it and return its description as a tool result, keeping the image out of the
|
||||
// main-model context. It is safe for concurrent use.
|
||||
type VisionTool struct {
|
||||
name string
|
||||
adapter VisionAdapter
|
||||
}
|
||||
|
||||
// NewVisionTool builds a VisionTool with the given name and adapter.
|
||||
func NewVisionTool(name string, adapter VisionAdapter) *VisionTool {
|
||||
return &VisionTool{name: name, adapter: adapter}
|
||||
}
|
||||
|
||||
// Name returns the tool identifier the agent uses to invoke it.
|
||||
func (t *VisionTool) Name() string { return t.name }
|
||||
|
||||
// Description explains what the tool does.
|
||||
func (t *VisionTool) Description() string {
|
||||
return "Understand an image and return a description of its contents."
|
||||
}
|
||||
|
||||
// Parameters returns the input JSON Schema.
|
||||
func (t *VisionTool) Parameters() (map[string]any, error) {
|
||||
return toolParameters(), nil
|
||||
}
|
||||
|
||||
// Execute runs the adapter on the image from args and returns the description.
|
||||
func (t *VisionTool) Execute(ctx context.Context, args any) (any, error) {
|
||||
image, prompt, err := toolArgs(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ref, err := ParseImageRef(image)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
desc, err := t.adapter.Describe(ctx, ref, prompt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"description": desc}, nil
|
||||
}
|
||||
|
||||
// OCRTool wraps an OCRAdapter as an sdk.Tool. It returns the extracted text as a
|
||||
// tool result. It is safe for concurrent use.
|
||||
type OCRTool struct {
|
||||
name string
|
||||
adapter OCRAdapter
|
||||
}
|
||||
|
||||
// NewOCRTool builds an OCRTool with the given name and adapter.
|
||||
func NewOCRTool(name string, adapter OCRAdapter) *OCRTool {
|
||||
return &OCRTool{name: name, adapter: adapter}
|
||||
}
|
||||
|
||||
// Name returns the tool identifier.
|
||||
func (t *OCRTool) Name() string { return t.name }
|
||||
|
||||
// Description explains what the tool does.
|
||||
func (t *OCRTool) Description() string {
|
||||
return "Extract text from an image (OCR) and return the recognized text."
|
||||
}
|
||||
|
||||
// Parameters returns the input JSON Schema.
|
||||
func (t *OCRTool) Parameters() (map[string]any, error) {
|
||||
return toolParameters(), nil
|
||||
}
|
||||
|
||||
// Execute runs the OCR adapter on the image from args and returns the text.
|
||||
func (t *OCRTool) Execute(ctx context.Context, args any) (any, error) {
|
||||
image, _, err := toolArgs(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ref, err := ParseImageRef(image)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
text, err := t.adapter.Extract(ctx, ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"text": text}, nil
|
||||
}
|
||||
|
||||
// RegisterTools registers the vision and OCR tools with r via the SDK helper.
|
||||
// Each tool is only registered when its pointer is non-nil. It returns the
|
||||
// first registration error, or nil when every tool registered.
|
||||
func RegisterTools(r sdk.Registrar, tool *VisionTool, ocr *OCRTool) error {
|
||||
if r == nil {
|
||||
return sdk.ErrNilRegistrar
|
||||
}
|
||||
if tool != nil {
|
||||
if err := sdk.RegisterTool(r, tool); err != nil {
|
||||
return fmt.Errorf("vision: register tool %q: %w", tool.Name(), err)
|
||||
}
|
||||
}
|
||||
if ocr != nil {
|
||||
if err := sdk.RegisterTool(r, ocr); err != nil {
|
||||
return fmt.Errorf("vision: register tool %q: %w", ocr.Name(), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// toolArgs decodes sdk.Tool.Execute args into the image string and an optional
|
||||
// prompt. Args must be a map[string]any with a non-empty "image" string.
|
||||
func toolArgs(args any) (image, prompt string, err error) {
|
||||
m, ok := args.(map[string]any)
|
||||
if !ok {
|
||||
return "", "", fmt.Errorf("vision: %w: args must be an object", ErrBadArgs)
|
||||
}
|
||||
image, _ = m["image"].(string)
|
||||
if image == "" {
|
||||
return "", "", fmt.Errorf("vision: %w: missing image", ErrBadArgs)
|
||||
}
|
||||
prompt, _ = m["prompt"].(string)
|
||||
return image, prompt, nil
|
||||
}
|
||||
Reference in new issue
Block a user