80 lines
2.8 KiB
Go
80 lines
2.8 KiB
Go
// Package vision implements the layered image routing described in
|
|
// DevDocs/design/06-多模态.md §0.
|
|
//
|
|
// A request that carries image content is first checked by DetectMultimodal.
|
|
// The VisionRouter then picks one of the layered routes: the main model
|
|
// directly (when it accepts images), a tool-shaped vision/OCR adapter (when the
|
|
// main model self-assesses poorly or cannot see), or the OCR tool (when the
|
|
// image is mainly text). Vision/OCR adapters are registered as sdk.Tool
|
|
// capabilities through the capability registry, and their output is a tool
|
|
// result, so the image itself never occupies main-model context.
|
|
//
|
|
// Images may come from a local path, an http(s) URL, or a base64/data: URI.
|
|
package vision
|
|
|
|
import (
|
|
"strings"
|
|
|
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
|
)
|
|
|
|
// DetectMultimodal reports whether msgs contains any non-text content part,
|
|
// i.e. whether multimodal handling is required. Today model only distinguishes
|
|
// text from image content, so this is true exactly when a message carries an
|
|
// image part; the check is written against "not text" so future content types
|
|
// (audio) extend it without rework.
|
|
func DetectMultimodal(msgs []model.Message) bool {
|
|
for _, m := range msgs {
|
|
for _, part := range m.Content {
|
|
if part.Type != model.ContentTypeText {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ImageSource classifies where an image's bytes come from.
|
|
type ImageSource string
|
|
|
|
const (
|
|
// SourcePath is a local filesystem path.
|
|
SourcePath ImageSource = "path"
|
|
// SourceURL is an http(s):// URL.
|
|
SourceURL ImageSource = "url"
|
|
// SourceBase64 is a base64 payload, optionally wrapped in a data: URI.
|
|
SourceBase64 ImageSource = "base64"
|
|
)
|
|
|
|
// ImageRef is a single image reference plus its resolved source kind.
|
|
type ImageRef struct {
|
|
// Source is the classified source kind.
|
|
Source ImageSource
|
|
// Data is the local path, URL, or base64/data: payload.
|
|
Data string
|
|
}
|
|
|
|
// ParseImageRef classifies a raw image source string into an ImageRef. Empty
|
|
// input is rejected. Local paths that exist on disk are classified as paths;
|
|
// http(s) URLs as URLs; every other value, including a data: URI, is treated as
|
|
// a base64 payload. This keeps routing deterministic without guessing MIME.
|
|
func ParseImageRef(src string) (ImageRef, error) {
|
|
if src == "" {
|
|
return ImageRef{}, ErrNoImage
|
|
}
|
|
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
|
|
return ImageRef{Source: SourceURL, Data: src}, nil
|
|
}
|
|
if strings.HasPrefix(src, "file://") {
|
|
return ImageRef{Source: SourcePath, Data: strings.TrimPrefix(src, "file://")}, nil
|
|
}
|
|
// A data: URI is base64 already; treat as base64, not path.
|
|
if strings.HasPrefix(src, "data:") {
|
|
return ImageRef{Source: SourceBase64, Data: src}, nil
|
|
}
|
|
if fileExists(src) {
|
|
return ImageRef{Source: SourcePath, Data: src}, nil
|
|
}
|
|
return ImageRef{Source: SourceBase64, Data: src}, nil
|
|
}
|