// Package vision implements the layered image routing described in // DevDocs/design/06-多模态.md §0. // // A request that carries image content is first checked by DetectMultimodal. // The VisionRouter then picks one of the layered routes: the main model // directly (when it accepts images), a tool-shaped vision/OCR adapter (when the // main model self-assesses poorly or cannot see), or the OCR tool (when the // image is mainly text). Vision/OCR adapters are registered as sdk.Tool // capabilities through the capability registry, and their output is a tool // result, so the image itself never occupies main-model context. // // Images may come from a local path, an http(s) URL, or a base64/data: URI. package vision import ( "strings" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model" ) // DetectMultimodal reports whether msgs contains any non-text content part, // i.e. whether multimodal handling is required. Today model only distinguishes // text from image content, so this is true exactly when a message carries an // image part; the check is written against "not text" so future content types // (audio) extend it without rework. func DetectMultimodal(msgs []model.Message) bool { for _, m := range msgs { for _, part := range m.Content { if part.Type != model.ContentTypeText { return true } } } return false } // ImageSource classifies where an image's bytes come from. type ImageSource string const ( // SourcePath is a local filesystem path. SourcePath ImageSource = "path" // SourceURL is an http(s):// URL. SourceURL ImageSource = "url" // SourceBase64 is a base64 payload, optionally wrapped in a data: URI. SourceBase64 ImageSource = "base64" ) // ImageRef is a single image reference plus its resolved source kind. type ImageRef struct { // Source is the classified source kind. Source ImageSource // Data is the local path, URL, or base64/data: payload. Data string } // ParseImageRef classifies a raw image source string into an ImageRef. Empty // input is rejected. Local paths that exist on disk are classified as paths; // http(s) URLs as URLs; every other value, including a data: URI, is treated as // a base64 payload. This keeps routing deterministic without guessing MIME. func ParseImageRef(src string) (ImageRef, error) { if src == "" { return ImageRef{}, ErrNoImage } if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { return ImageRef{Source: SourceURL, Data: src}, nil } if strings.HasPrefix(src, "file://") { return ImageRef{Source: SourcePath, Data: strings.TrimPrefix(src, "file://")}, nil } // A data: URI is base64 already; treat as base64, not path. if strings.HasPrefix(src, "data:") { return ImageRef{Source: SourceBase64, Data: src}, nil } if fileExists(src) { return ImageRef{Source: SourcePath, Data: src}, nil } return ImageRef{Source: SourceBase64, Data: src}, nil }