feat(multimodal): 视觉分层路由(多模态检测→主/视觉/OCR 工具)+ 语音 STT→TTS 降级链(异步)
This commit is contained in:
1 parent
02aae2bbb4
commit
0a936a3202
19 files changed
+2121
No files matched your search
@@ -0,0 +1,79 @@
|
||||
// Package vision implements the layered image routing described in
|
||||
// DevDocs/design/06-多模态.md §0.
|
||||
//
|
||||
// A request that carries image content is first checked by DetectMultimodal.
|
||||
// The VisionRouter then picks one of the layered routes: the main model
|
||||
// directly (when it accepts images), a tool-shaped vision/OCR adapter (when the
|
||||
// main model self-assesses poorly or cannot see), or the OCR tool (when the
|
||||
// image is mainly text). Vision/OCR adapters are registered as sdk.Tool
|
||||
// capabilities through the capability registry, and their output is a tool
|
||||
// result, so the image itself never occupies main-model context.
|
||||
//
|
||||
// Images may come from a local path, an http(s) URL, or a base64/data: URI.
|
||||
package vision
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||
)
|
||||
|
||||
// DetectMultimodal reports whether msgs contains any non-text content part,
|
||||
// i.e. whether multimodal handling is required. Today model only distinguishes
|
||||
// text from image content, so this is true exactly when a message carries an
|
||||
// image part; the check is written against "not text" so future content types
|
||||
// (audio) extend it without rework.
|
||||
func DetectMultimodal(msgs []model.Message) bool {
|
||||
for _, m := range msgs {
|
||||
for _, part := range m.Content {
|
||||
if part.Type != model.ContentTypeText {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ImageSource classifies where an image's bytes come from.
|
||||
type ImageSource string
|
||||
|
||||
const (
|
||||
// SourcePath is a local filesystem path.
|
||||
SourcePath ImageSource = "path"
|
||||
// SourceURL is an http(s):// URL.
|
||||
SourceURL ImageSource = "url"
|
||||
// SourceBase64 is a base64 payload, optionally wrapped in a data: URI.
|
||||
SourceBase64 ImageSource = "base64"
|
||||
)
|
||||
|
||||
// ImageRef is a single image reference plus its resolved source kind.
|
||||
type ImageRef struct {
|
||||
// Source is the classified source kind.
|
||||
Source ImageSource
|
||||
// Data is the local path, URL, or base64/data: payload.
|
||||
Data string
|
||||
}
|
||||
|
||||
// ParseImageRef classifies a raw image source string into an ImageRef. Empty
|
||||
// input is rejected. Local paths that exist on disk are classified as paths;
|
||||
// http(s) URLs as URLs; every other value, including a data: URI, is treated as
|
||||
// a base64 payload. This keeps routing deterministic without guessing MIME.
|
||||
func ParseImageRef(src string) (ImageRef, error) {
|
||||
if src == "" {
|
||||
return ImageRef{}, ErrNoImage
|
||||
}
|
||||
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
|
||||
return ImageRef{Source: SourceURL, Data: src}, nil
|
||||
}
|
||||
if strings.HasPrefix(src, "file://") {
|
||||
return ImageRef{Source: SourcePath, Data: strings.TrimPrefix(src, "file://")}, nil
|
||||
}
|
||||
// A data: URI is base64 already; treat as base64, not path.
|
||||
if strings.HasPrefix(src, "data:") {
|
||||
return ImageRef{Source: SourceBase64, Data: src}, nil
|
||||
}
|
||||
if fileExists(src) {
|
||||
return ImageRef{Source: SourcePath, Data: src}, nil
|
||||
}
|
||||
return ImageRef{Source: SourceBase64, Data: src}, nil
|
||||
}
|
||||
Reference in new issue
Block a user