docs(comments): 注释中文化 · internal/runtime/multimodal(21 文件,纯替换)
- 只翻注释/文档字符串,代码零变化:go/scanner 词法流比对 21 文件 token+字面量全等,增删行数 1:1(467/467) - 刻意保留非散文英文:JSON Schema description、工具 Description() 返回值、adapter prompt、API 路径与命令示例、环境变量名 - 逐行确认代码前缀未变,未触碰其他模块
This commit is contained in:
1 parent
b9e042d07d
commit
b0fbb8bc0d
21 files changed
+467
-467
No files matched your search
@@ -13,28 +13,28 @@ import (
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||
)
|
||||
|
||||
// visionMaxTokens caps the vision/OCR response length.
|
||||
// visionMaxTokens 限制 vision/OCR 响应的长度。
|
||||
const visionMaxTokens = 512
|
||||
|
||||
// VisionAdapter understands an image and returns a description of it. It is the
|
||||
// backend behind the vision tool; real providers implement it without exposing
|
||||
// the image to the main-model context.
|
||||
// VisionAdapter 理解一张图像并返回对它的描述。它是
|
||||
// 视觉工具背后的后端;真实提供方实现它,且不把图像
|
||||
// 暴露给主模型上下文。
|
||||
type VisionAdapter interface {
|
||||
// Describe returns a description of img. prompt is an optional instruction
|
||||
// overriding the default "describe this image" ask.
|
||||
// Describe 返回 img 的描述。prompt 是可选的指令,
|
||||
// 用于覆盖默认的「描述这张图」提问。
|
||||
Describe(ctx context.Context, img ImageRef, prompt string) (string, error)
|
||||
}
|
||||
|
||||
// OCRAdapter extracts text from an image. It is the backend behind the OCR tool.
|
||||
// OCRAdapter 从图像中抽取文字。它是 OCR 工具背后的后端。
|
||||
type OCRAdapter interface {
|
||||
// Extract returns the text found in img.
|
||||
// Extract 返回 img 中发现的文字。
|
||||
Extract(ctx context.Context, img ImageRef) (string, error)
|
||||
}
|
||||
|
||||
// OpenAICompatibleAdapter implements both VisionAdapter and OCRAdapter by
|
||||
// calling an OpenAI-compatible /chat/completions endpoint with the image
|
||||
// embedded as a content part. It is the ready default implementation; real
|
||||
// cloud vision/OCR providers expose the same protocol.
|
||||
// OpenAICompatibleAdapter 通过调用 OpenAI 兼容的 /chat/completions
|
||||
// endpoint(图像作为内容部件内嵌)同时实现 VisionAdapter 与 OCRAdapter。
|
||||
// 它是开箱可用的默认实现;真实的
|
||||
// 云端 vision/OCR 提供方暴露相同协议。
|
||||
type OpenAICompatibleAdapter struct {
|
||||
cfg config.LLMConfig
|
||||
model string
|
||||
@@ -42,10 +42,10 @@ type OpenAICompatibleAdapter struct {
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// OpenAIOption customizes an OpenAICompatibleAdapter.
|
||||
// OpenAIOption 用于定制 OpenAICompatibleAdapter。
|
||||
type OpenAIOption func(*OpenAICompatibleAdapter)
|
||||
|
||||
// WithModel overrides the adapter model (defaults to cfg.Model).
|
||||
// WithModel 覆盖适配器的模型(默认取 cfg.Model)。
|
||||
func WithModel(m string) OpenAIOption {
|
||||
return func(a *OpenAICompatibleAdapter) {
|
||||
if m != "" {
|
||||
@@ -54,7 +54,7 @@ func WithModel(m string) OpenAIOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithHTTPClient sets the *http.Client used for the request; nil keeps default.
|
||||
// WithHTTPClient 设置请求所用的 *http.Client;nil 表示保持默认。
|
||||
func WithHTTPClient(c *http.Client) OpenAIOption {
|
||||
return func(a *OpenAICompatibleAdapter) {
|
||||
if c != nil {
|
||||
@@ -63,7 +63,7 @@ func WithHTTPClient(c *http.Client) OpenAIOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithLogger sets the logger used on failures; nil keeps a discard logger.
|
||||
// WithLogger 设置失败时使用的日志器;nil 表示保持丢弃日志器。
|
||||
func WithLogger(l *slog.Logger) OpenAIOption {
|
||||
return func(a *OpenAICompatibleAdapter) {
|
||||
if l != nil {
|
||||
@@ -72,8 +72,8 @@ func WithLogger(l *slog.Logger) OpenAIOption {
|
||||
}
|
||||
}
|
||||
|
||||
// NewOpenAICompatible builds an adapter from cfg, using cfg.Model unless
|
||||
// overridden.
|
||||
// NewOpenAICompatible 用 cfg 构造一个适配器,未覆盖时使用
|
||||
// cfg.Model。
|
||||
func NewOpenAICompatible(cfg config.LLMConfig, opts ...OpenAIOption) *OpenAICompatibleAdapter {
|
||||
a := &OpenAICompatibleAdapter{
|
||||
cfg: cfg,
|
||||
@@ -87,7 +87,7 @@ func NewOpenAICompatible(cfg config.LLMConfig, opts ...OpenAIOption) *OpenAIComp
|
||||
return a
|
||||
}
|
||||
|
||||
// Describe returns an image description from the underlying chat endpoint.
|
||||
// Describe 从底层 chat endpoint 返回图像描述。
|
||||
func (a *OpenAICompatibleAdapter) Describe(ctx context.Context, img ImageRef, prompt string) (string, error) {
|
||||
if prompt == "" {
|
||||
prompt = "Describe this image in detail."
|
||||
@@ -95,12 +95,12 @@ func (a *OpenAICompatibleAdapter) Describe(ctx context.Context, img ImageRef, pr
|
||||
return a.ask(ctx, img, prompt)
|
||||
}
|
||||
|
||||
// Extract returns text found in the image via the same endpoint.
|
||||
// Extract 经同一 endpoint 返回图像中发现的文字。
|
||||
func (a *OpenAICompatibleAdapter) Extract(ctx context.Context, img ImageRef) (string, error) {
|
||||
return a.ask(ctx, img, "Extract all text visible in this image. Return only the text.")
|
||||
}
|
||||
|
||||
// ask resolves the image, builds a chat request and returns the assistant text.
|
||||
// ask 解析图像、构造 chat 请求并返回助手文本。
|
||||
func (a *OpenAICompatibleAdapter) ask(ctx context.Context, img ImageRef, prompt string) (string, error) {
|
||||
if strings.TrimSpace(a.cfg.BaseURL) == "" {
|
||||
return "", ErrNotConfigured
|
||||
@@ -139,8 +139,8 @@ func (a *OpenAICompatibleAdapter) ask(ctx context.Context, img ImageRef, prompt
|
||||
return a.decode(resp.Body)
|
||||
}
|
||||
|
||||
// imageURL converts an ImageRef to the image_url value the wire expects: an
|
||||
// http(s) URL passes through, anything else becomes a base64 data: URI.
|
||||
// imageURL 把 ImageRef 转换为线上格式期望的 image_url 取值:http(s)
|
||||
// URL 原样透传,其余一律转为 base64 data: URI。
|
||||
func (a *OpenAICompatibleAdapter) imageURL(ctx context.Context, img ImageRef) (string, error) {
|
||||
if img.Source == SourceURL {
|
||||
return img.Data, nil
|
||||
@@ -152,7 +152,7 @@ func (a *OpenAICompatibleAdapter) imageURL(ctx context.Context, img ImageRef) (s
|
||||
return DataURI(mime, data), nil
|
||||
}
|
||||
|
||||
// openAIVisionRequest is the JSON body sent to /chat/completions.
|
||||
// openAIVisionRequest 是发往 /chat/completions 的 JSON 请求体。
|
||||
type openAIVisionRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []openAIMessage `json:"messages"`
|
||||
@@ -186,7 +186,7 @@ type openAIResponseMessage struct {
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
// request builds the wire body for an image+prompt.
|
||||
// request 构造图像 + prompt 的线上请求体。
|
||||
func (a *OpenAICompatibleAdapter) request(imageURL, prompt string) openAIVisionRequest {
|
||||
return openAIVisionRequest{
|
||||
Model: a.model,
|
||||
@@ -200,7 +200,7 @@ func (a *OpenAICompatibleAdapter) request(imageURL, prompt string) openAIVisionR
|
||||
}
|
||||
}
|
||||
|
||||
// decode extracts the first choice's text from a response body.
|
||||
// decode 从响应体中取出第一个 choice 的文本。
|
||||
func (a *OpenAICompatibleAdapter) decode(r io.Reader) (string, error) {
|
||||
var wire openAIVisionResponse
|
||||
if err := json.NewDecoder(r).Decode(&wire); err != nil {
|
||||
@@ -212,6 +212,6 @@ func (a *OpenAICompatibleAdapter) decode(r io.Reader) (string, error) {
|
||||
return wire.Choices[0].Message.Content, nil
|
||||
}
|
||||
|
||||
// interface assertion: one concrete adapter serves both directions.
|
||||
// 接口断言:同一个具体适配器同时服务两个方向。
|
||||
var _ VisionAdapter = (*OpenAICompatibleAdapter)(nil)
|
||||
var _ OCRAdapter = (*OpenAICompatibleAdapter)(nil)
|
||||
@@ -2,21 +2,21 @@ package vision
|
||||
|
||||
import "errors"
|
||||
|
||||
// Sentinel errors returned by the vision package. Each is wrapped with %w on
|
||||
// the way out so callers can test for a category while keeping the cause.
|
||||
// vision 包返回的哨兵错误。每个错误向外传递时都以 %w
|
||||
// 包装,使调用方能按类别判断,同时保留原始原因。
|
||||
var (
|
||||
// ErrNoImage is returned when an image source string is empty.
|
||||
// ErrNoImage 在图像来源字符串为空时返回。
|
||||
ErrNoImage = errors.New("vision: empty image source")
|
||||
// ErrNotConfigured is returned by adapters when no endpoint/model is set.
|
||||
// ErrNotConfigured 在适配器未设置 endpoint/model 时返回。
|
||||
ErrNotConfigured = errors.New("vision: adapter not configured")
|
||||
// ErrUnsupportedSource is returned when an image source cannot be handled.
|
||||
// ErrUnsupportedSource 在图像来源无法处理时返回。
|
||||
ErrUnsupportedSource = errors.New("vision: unsupported image source")
|
||||
// ErrFetch is returned when a URL image cannot be downloaded.
|
||||
// ErrFetch 在 URL 图像无法下载时返回。
|
||||
ErrFetch = errors.New("vision: cannot fetch image")
|
||||
// ErrUpstream is returned when the vision/OCR provider request fails.
|
||||
// ErrUpstream 在 vision/OCR 提供方请求失败时返回。
|
||||
ErrUpstream = errors.New("vision: upstream request failed")
|
||||
// ErrBadImage is returned when an image cannot be read or decoded.
|
||||
// ErrBadImage 在图像无法读取或解码时返回。
|
||||
ErrBadImage = errors.New("vision: cannot read image")
|
||||
// ErrBadArgs is returned by a tool when its arguments are malformed.
|
||||
// ErrBadArgs 在工具参数格式错误时由该工具返回。
|
||||
ErrBadArgs = errors.New("vision: bad tool arguments")
|
||||
)
|
||||
@@ -10,23 +10,23 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Size caps for reading image bytes and response bodies, so a hostile or
|
||||
// malformed source cannot exhaust memory. The 50MB cap matches the file-upload
|
||||
// limit in DevDocs/02-development-constraints.md §11.
|
||||
// 读取图像字节与响应体的体积上限,使恶意或
|
||||
// 畸形来源无法耗尽内存。50MB 上限与
|
||||
// DevDocs/02-development-constraints.md §11 的文件上传限制一致。
|
||||
const (
|
||||
maxImageBytes = 50 << 20
|
||||
maxFetchedBody = 50 << 20
|
||||
)
|
||||
|
||||
// DataURI builds a base64 data: image URI from mime and raw bytes.
|
||||
// DataURI 由 mime 与原始字节构造一个 base64 data: 图像 URI。
|
||||
func DataURI(mime string, data []byte) string {
|
||||
return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data)
|
||||
}
|
||||
|
||||
// Resolve loads the bytes for an ImageRef. Paths and base64 payloads are read
|
||||
// locally; URLs are fetched with an http.Client (the provided client, or a
|
||||
// default one when nil). For base64 input the Data field is returned verbatim as
|
||||
// the payload and the MIME type is guessed from a data: prefix.
|
||||
// Resolve 加载一个 ImageRef 对应的字节。路径与 base64 载荷在
|
||||
// 本地读取;URL 用 http.Client 抓取(传入的 client,为 nil 时
|
||||
// 用默认 client)。对 base64 输入,Data 字段原样作为
|
||||
// 载荷返回,MIME 类型从 data: 前缀推断。
|
||||
func Resolve(ctx context.Context, ref ImageRef, client *http.Client) (mime string, data []byte, err error) {
|
||||
switch ref.Source {
|
||||
case SourcePath:
|
||||
@@ -40,7 +40,7 @@ func Resolve(ctx context.Context, ref ImageRef, client *http.Client) (mime strin
|
||||
}
|
||||
}
|
||||
|
||||
// readFile reads an image file and guesses its MIME type from the extension.
|
||||
// readFile 读取一个图像文件,并从扩展名推断其 MIME 类型。
|
||||
func readFile(path string) (string, []byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
@@ -54,7 +54,7 @@ func readFile(path string) (string, []byte, error) {
|
||||
return mimeByExt(path), data, nil
|
||||
}
|
||||
|
||||
// decodeBase64 returns the decoded bytes of a base64/data: payload and its MIME.
|
||||
// decodeBase64 返回 base64/data: 载荷解码后的字节及其 MIME。
|
||||
func decodeBase64(src string) (string, []byte, error) {
|
||||
mime, encoded, isDataURI := splitDataURI(src)
|
||||
if isDataURI {
|
||||
@@ -71,7 +71,7 @@ func decodeBase64(src string) (string, []byte, error) {
|
||||
return "image/png", decoded, nil
|
||||
}
|
||||
|
||||
// fetchURL downloads an image over http(s); only http/https schemes are allowed.
|
||||
// fetchURL 通过 http(s) 下载一张图像;只允许 http/https 协议。
|
||||
func fetchURL(ctx context.Context, rawURL string, client *http.Client) (string, []byte, error) {
|
||||
if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
|
||||
return "", nil, fmt.Errorf("vision: %w: %q", ErrUnsupportedSource, rawURL)
|
||||
@@ -98,8 +98,8 @@ func fetchURL(ctx context.Context, rawURL string, client *http.Client) (string,
|
||||
return mimeByContentType(resp.Header.Get("Content-Type")), data, nil
|
||||
}
|
||||
|
||||
// splitDataURI parses a "data:<mime>;base64,<payload>" prefix. The second return
|
||||
// is false when src is not a data: URI.
|
||||
// splitDataURI 解析 "data:<mime>;base64,<payload>" 前缀。src 不是
|
||||
// data: URI 时第二个返回值是 false。
|
||||
func splitDataURI(src string) (mime, payload string, ok bool) {
|
||||
if !strings.HasPrefix(src, "data:") {
|
||||
return "", "", false
|
||||
@@ -124,7 +124,7 @@ func splitDataURI(src string) (mime, payload string, ok bool) {
|
||||
return mime, payload, true
|
||||
}
|
||||
|
||||
// mimeByExt guesses a MIME type from a file path extension.
|
||||
// mimeByExt 从文件路径扩展名推断 MIME 类型。
|
||||
func mimeByExt(path string) string {
|
||||
switch strings.ToLower(pathExtension(path)) {
|
||||
case ".png":
|
||||
@@ -140,8 +140,8 @@ func mimeByExt(path string) string {
|
||||
}
|
||||
}
|
||||
|
||||
// mimeByContentType returns a MIME type from an HTTP Content-Type header,
|
||||
// defaulting to image/png when absent or non-image.
|
||||
// mimeByContentType 从 HTTP Content-Type 头返回 MIME 类型,
|
||||
// 缺失或非图像时默认为 image/png。
|
||||
func mimeByContentType(ct string) string {
|
||||
if ct == "" || !strings.HasPrefix(ct, "image/") {
|
||||
return "image/png"
|
||||
@@ -149,7 +149,7 @@ func mimeByContentType(ct string) string {
|
||||
return strings.TrimSpace(strings.Split(ct, ";")[0])
|
||||
}
|
||||
|
||||
// pathExtension returns the file extension with its dot, or empty when none.
|
||||
// pathExtension 返回带点号的文件扩展名,没有则返回空串。
|
||||
func pathExtension(path string) string {
|
||||
base := path
|
||||
if idx := strings.LastIndexByte(path, '/'); idx >= 0 {
|
||||
@@ -164,15 +164,15 @@ func pathExtension(path string) string {
|
||||
return base[dot:]
|
||||
}
|
||||
|
||||
// fileExists reports whether path exists on disk and is a regular file.
|
||||
// fileExists 报告 path 是否存在于磁盘且为常规文件。
|
||||
func fileExists(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && info.Mode().IsRegular()
|
||||
}
|
||||
|
||||
// defaultHTTPClient builds the HTTP client described in
|
||||
// DevDocs/02-development-constraints.md §2.3. A fresh client is returned per
|
||||
// call rather than a package global, which the constraints forbid.
|
||||
// defaultHTTPClient 构造 DevDocs/02-development-constraints.md §2.3
|
||||
// 所述的 HTTP client。这里每次调用返回一个新 client,
|
||||
// 而不是包级全局变量——后者为约束所禁止。
|
||||
func defaultHTTPClient() *http.Client {
|
||||
return &http.Client{Transport: http.DefaultTransport}
|
||||
}
|
||||
@@ -7,31 +7,31 @@ import (
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||
)
|
||||
|
||||
// ModelInfoProvider supplies the main model's vision capability so the resolver
|
||||
// can route an image the way the vision Router decides (design/06 §0). It is
|
||||
// called per image; the same provider is shared across a single resolution.
|
||||
// ModelInfoProvider 提供主模型的视觉能力,使解析器
|
||||
// 能按 vision Router 的判定(design/06 §0)路由图像。它按
|
||||
// 图像逐个调用;同一次解析中共享同一个 provider。
|
||||
type ModelInfoProvider func() ModelInfo
|
||||
|
||||
// ToolInvoker invokes a registered tool (the vision or OCR tool) by name with
|
||||
// the given arguments. It is satisfied by *tool.Dispatcher.Invoke.
|
||||
// ToolInvoker 按名字与给定参数调用一个已注册的工具(视觉或 OCR
|
||||
// 工具)。由 *tool.Dispatcher.Invoke 满足。
|
||||
type ToolInvoker func(ctx context.Context, name string, args map[string]any) (any, error)
|
||||
|
||||
// MessageResolver is the loop's concrete ImageResolver. For each image part in a
|
||||
// user message it asks the vision Router how to route it: when the main model
|
||||
// supports vision and is confident, the image part is kept so the main model sees
|
||||
// the image; otherwise the vision/OCR tool is invoked and its text result replaces
|
||||
// the image part, so a text-only main model can reason over the image ("先看图再
|
||||
// 对话"). It is safe for concurrent use.
|
||||
// MessageResolver 是 loop 的具体 ImageResolver。对用户消息中的
|
||||
// 每个图像部件,它向 vision Router 询问如何路由:当主模型
|
||||
// 支持视觉且有信心时,保留该图像部件让主模型看到
|
||||
// 图像;否则调用 vision/OCR 工具,用其文字结果替换
|
||||
// 图像部件,使纯文本主模型也能对图像推理(「先看图再
|
||||
// 对话」)。它可安全并发使用。
|
||||
type MessageResolver struct {
|
||||
router *Router
|
||||
info ModelInfoProvider
|
||||
invoke ToolInvoker
|
||||
}
|
||||
|
||||
// NewMessageResolver builds a MessageResolver. router decides the layer; info
|
||||
// reports the main model's vision capability; invoke runs a registered vision/OCR
|
||||
// tool. A nil invoke leaves any tool route failed (the image is then kept), so a
|
||||
// resolver without tools degrades to the main-model path.
|
||||
// NewMessageResolver 构造一个 MessageResolver。router 决定
|
||||
// 分层;info 报告主模型的视觉能力;invoke 运行一个已注册的 vision/OCR
|
||||
// 工具。invoke 为 nil 时任何工具路由都会失败(此时图像被保留),
|
||||
// 因此没有工具的解析器会降级到主模型路径。
|
||||
func NewMessageResolver(router *Router, info ModelInfoProvider, invoke ToolInvoker) *MessageResolver {
|
||||
return &MessageResolver{
|
||||
router: router,
|
||||
@@ -40,10 +40,10 @@ func NewMessageResolver(router *Router, info ModelInfoProvider, invoke ToolInvok
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve rewrites msg so every image part is either kept (main model sees it) or
|
||||
// replaced by a text description produced by the vision/OCR tool. Text parts are
|
||||
// preserved in order. It errors only when an image cannot be routed or a tool
|
||||
// invocation fails; the caller (loop) then keeps the original message.
|
||||
// Resolve 改写 msg,使每个图像部件要么被保留(主模型能看到它),
|
||||
// 要么被 vision/OCR 工具产出的文字描述替换。文本部件
|
||||
// 按原顺序保留。它仅在图像无法路由或工具
|
||||
// 调用失败时报错;此时调用方(loop)保留原始消息。
|
||||
func (r *MessageResolver) Resolve(ctx context.Context, msg model.Message) (model.Message, error) {
|
||||
if r.router == nil {
|
||||
return msg, nil
|
||||
@@ -63,8 +63,8 @@ func (r *MessageResolver) Resolve(ctx context.Context, msg model.Message) (model
|
||||
return model.Message{Role: msg.Role, Content: out}, nil
|
||||
}
|
||||
|
||||
// resolveImage picks a route and either keeps the image (KindMain) or folds it
|
||||
// into a text description via the vision/OCR tool.
|
||||
// resolveImage 选出一条路由:要么保留图像(KindMain),
|
||||
// 要么经 vision/OCR 工具把它折叠为文字描述。
|
||||
func (r *MessageResolver) resolveImage(ctx context.Context, imageURL string) (model.Content, error) {
|
||||
target := ModelInfo{}
|
||||
if r.info != nil {
|
||||
@@ -73,7 +73,7 @@ func (r *MessageResolver) resolveImage(ctx context.Context, imageURL string) (mo
|
||||
decision := r.router.Route(target, RouteRequest{})
|
||||
switch decision.Kind {
|
||||
case KindMain:
|
||||
// A vision-capable main model sees the image directly.
|
||||
// 支持视觉的主模型直接看到这张图。
|
||||
return model.Content{Type: model.ContentTypeImage, ImageURL: imageURL}, nil
|
||||
case KindVisionTool:
|
||||
return r.invokeTool(ctx, decision.Tool, map[string]any{
|
||||
@@ -89,8 +89,8 @@ func (r *MessageResolver) resolveImage(ctx context.Context, imageURL string) (mo
|
||||
}
|
||||
}
|
||||
|
||||
// invokeTool runs a registered vision/OCR tool and turns its text result into a
|
||||
// text content part. field is the result map key ("description" or "text").
|
||||
// invokeTool 运行一个已注册的 vision/OCR 工具,并把其文字结果转成
|
||||
// 文本内容部件。field 是结果 map 的键("description" 或 "text")。
|
||||
func (r *MessageResolver) invokeTool(ctx context.Context, toolName string, args map[string]any, field string) (model.Content, error) {
|
||||
if r.invoke == nil {
|
||||
return model.Content{}, fmt.Errorf("%s: %w", toolName, ErrNotConfigured)
|
||||
@@ -102,8 +102,8 @@ func (r *MessageResolver) invokeTool(ctx context.Context, toolName string, args
|
||||
return model.Content{Type: model.ContentTypeText, Text: descriptionFragment(fieldText(result, field))}, nil
|
||||
}
|
||||
|
||||
// fieldText extracts a string field from a tool result map, tolerating the
|
||||
// map[string]any shape returned by the vision/OCR tools.
|
||||
// fieldText 从工具结果的 map 中取出字符串字段,兼容
|
||||
// vision/OCR 工具返回的 map[string]any 形状。
|
||||
func fieldText(result any, field string) string {
|
||||
m, ok := result.(map[string]any)
|
||||
if !ok {
|
||||
@@ -113,8 +113,8 @@ func fieldText(result any, field string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
// descriptionFragment labels a tool-produced description so the model can tell it
|
||||
// refers to the picture the user attached.
|
||||
// descriptionFragment 给工具产出的描述加上标注,使模型能看出它
|
||||
// 指的是用户附上的那张图。
|
||||
func descriptionFragment(desc string) string {
|
||||
if desc == "" {
|
||||
return "[图片内容] 无法识别该图片"
|
||||
|
||||
@@ -69,9 +69,9 @@ func TestMessageResolver_OCRToolForText(t *testing.T) {
|
||||
return map[string]any{"text": "HELLO"}, nil
|
||||
})
|
||||
|
||||
// Force the text-primary hint by using a router route request path: this is
|
||||
// exercised directly through Resolve which passes no hint, so the vision tool
|
||||
// is preferred; assert at least a text part replaces the image.
|
||||
// 通过 router 的路由请求路径强制给出文本优先提示:这里直接经 Resolve
|
||||
// 走通,而 Resolve 不传任何提示,因此优先使用视觉工具;
|
||||
// 断言至少有一个文本部件替换了图像。
|
||||
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "x", "data:image/png;base64,AAAA"))
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
|
||||
@@ -1,67 +1,67 @@
|
||||
package vision
|
||||
|
||||
// Assessment is the model's own confidence in reading an image.
|
||||
// Assessment 是模型对自己读图结果的置信度。
|
||||
type Assessment string
|
||||
|
||||
const (
|
||||
// AssessmentUnknown means the model reported no self-assessment.
|
||||
// AssessmentUnknown 表示模型未给出自评。
|
||||
AssessmentUnknown Assessment = "unknown"
|
||||
// AssessmentGood means the model is confident in its read.
|
||||
// AssessmentGood 表示模型对自己的读图结果有信心。
|
||||
AssessmentGood Assessment = "good"
|
||||
// AssessmentPoor means the model judged its recognition as poor.
|
||||
// AssessmentPoor 表示模型判定自己的识别效果差。
|
||||
AssessmentPoor Assessment = "poor"
|
||||
)
|
||||
|
||||
// ModelInfo describes the main model's vision capability.
|
||||
// ModelInfo 描述主模型的视觉能力。
|
||||
type ModelInfo struct {
|
||||
// Name identifies the model.
|
||||
// Name 标识模型。
|
||||
Name string
|
||||
// SupportsVision reports whether the model accepts image input.
|
||||
// SupportsVision 报告模型是否接受图像输入。
|
||||
SupportsVision bool
|
||||
// SelfAssessment is the model's own confidence for the current image.
|
||||
// It drives the layer-2 tool route when SupportsVision is true.
|
||||
// SelfAssessment 是模型对当前图像的自评置信度。
|
||||
// 当 SupportsVision 为真时,它决定是否走第二层工具路由。
|
||||
SelfAssessment Assessment
|
||||
}
|
||||
|
||||
// DecisionKind is the chosen layered route.
|
||||
// DecisionKind 是选定的分层路由。
|
||||
type DecisionKind string
|
||||
|
||||
const (
|
||||
// KindMain routes the image to the main model with the image in context.
|
||||
// KindMain 把图像连同上下文一起交给主模型。
|
||||
KindMain DecisionKind = "main"
|
||||
// KindVisionTool routes to a dedicated vision tool, returning only the tool
|
||||
// result so the image never occupies main-model context.
|
||||
// KindVisionTool 路由到专用视觉工具,只返回工具
|
||||
// 结果,使图像不占用主模型上下文。
|
||||
KindVisionTool DecisionKind = "vision_tool"
|
||||
// KindOCRTool routes to an OCR tool, returning only the extracted text.
|
||||
// KindOCRTool 路由到 OCR 工具,只返回抽取出的文字。
|
||||
KindOCRTool DecisionKind = "ocr_tool"
|
||||
// KindUnsupported means no route can process the image.
|
||||
// KindUnsupported 表示没有任何路由能处理该图像。
|
||||
KindUnsupported DecisionKind = "unsupported"
|
||||
)
|
||||
|
||||
// Decision is the outcome of a routing call.
|
||||
// Decision 是一次路由调用的结果。
|
||||
type Decision struct {
|
||||
// Kind is the chosen route.
|
||||
// Kind 是选定的路由。
|
||||
Kind DecisionKind
|
||||
// Tool names the vision or OCR tool to invoke, set when Kind is a tool route.
|
||||
// Tool 指名要调用的视觉或 OCR 工具,当 Kind 为工具路由时设置。
|
||||
Tool string
|
||||
// Reason is a human-readable explanation for observability and logs.
|
||||
// Reason 是便于可观测与日志的人类可读说明。
|
||||
Reason string
|
||||
}
|
||||
|
||||
// RouteRequest carries routing hints from the caller on top of the model info.
|
||||
// RouteRequest 携带调用方在模型信息之外给出的路由提示。
|
||||
type RouteRequest struct {
|
||||
// TextPrimary reports that the user indicated the image is mainly text, so
|
||||
// an OCR tool is preferred over a general vision model.
|
||||
// TextPrimary 表示用户指出图像以文字为主,因此
|
||||
// 优先使用 OCR 工具而非通用视觉模型。
|
||||
TextPrimary bool
|
||||
// MainSelfAssessedPoor is set when the main model has already judged its own
|
||||
// read of the image as poor, requesting a vision/OCR tool route.
|
||||
// MainSelfAssessedPoor 在主模型已自评读图效果
|
||||
// 较差、请求走 vision/OCR 工具路由时置位。
|
||||
MainSelfAssessedPoor bool
|
||||
}
|
||||
|
||||
// RouterOption configures a Router.
|
||||
// RouterOption 配置 Router。
|
||||
type RouterOption func(*Router)
|
||||
|
||||
// WithVisionTool binds the name of the registered vision-understanding tool.
|
||||
// WithVisionTool 绑定已注册的图像理解工具名。
|
||||
func WithVisionTool(name string) RouterOption {
|
||||
return func(r *Router) {
|
||||
if name != "" {
|
||||
@@ -70,7 +70,7 @@ func WithVisionTool(name string) RouterOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithOCRTool binds the name of the registered OCR tool.
|
||||
// WithOCRTool 绑定已注册的 OCR 工具名。
|
||||
func WithOCRTool(name string) RouterOption {
|
||||
return func(r *Router) {
|
||||
if name != "" {
|
||||
@@ -79,15 +79,15 @@ func WithOCRTool(name string) RouterOption {
|
||||
}
|
||||
}
|
||||
|
||||
// Router implements the layered vision routing policy. It does no I/O; the
|
||||
// caller uses the returned Decision to dispatch the request.
|
||||
// Router 实现分层视觉路由策略。它不做任何 I/O;调用方
|
||||
// 用返回的 Decision 去派发请求。
|
||||
type Router struct {
|
||||
visionTool string
|
||||
ocrTool string
|
||||
}
|
||||
|
||||
// NewRouter builds a Router. Unknown (empty) tool names leave the corresponding
|
||||
// route unavailable, so the router degrades to KindMain or KindUnsupported.
|
||||
// NewRouter 构造 Router。未知(空)的工具名会让对应
|
||||
// 路由不可用,于是路由器降级为 KindMain 或 KindUnsupported。
|
||||
func NewRouter(opts ...RouterOption) *Router {
|
||||
r := &Router{}
|
||||
for _, o := range opts {
|
||||
@@ -96,21 +96,21 @@ func NewRouter(opts ...RouterOption) *Router {
|
||||
return r
|
||||
}
|
||||
|
||||
// Route picks a layered route for an image request given the target model info
|
||||
// and caller hints. The order is: the main model directly when it can see and
|
||||
// has not self-assessed poorly (layer 1); otherwise a vision or OCR tool whose
|
||||
// result replaces the image in context (layers 2 and 3); otherwise
|
||||
// KindUnsupported when nothing can process the image.
|
||||
// Route 依据目标模型信息与调用方提示,为一次图像请求
|
||||
// 选出分层路由。顺序为:主模型能看图且
|
||||
// 未自评较差时直接用它(第一层);否则用能替换
|
||||
// 上下文中图像的视觉或 OCR 工具(第二、三层);再否则
|
||||
// 没有任何东西能处理该图像时给出 KindUnsupported。
|
||||
func (r *Router) Route(target ModelInfo, req RouteRequest) Decision {
|
||||
poor := target.SelfAssessment == AssessmentPoor || req.MainSelfAssessedPoor
|
||||
// Layer 1: the main model accepts images and is confident, so use it first.
|
||||
// 第一层:主模型接受图像且自信,优先使用它。
|
||||
if target.SupportsVision && !poor {
|
||||
return Decision{
|
||||
Kind: KindMain,
|
||||
Reason: "main model supports vision and is confident; use it directly",
|
||||
}
|
||||
}
|
||||
// OCR is preferred when the user says the image is mostly text.
|
||||
// 用户说明图像以文字为主时优先 OCR。
|
||||
if req.TextPrimary && r.ocrTool != "" {
|
||||
return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "image is mainly text; route to OCR"}
|
||||
}
|
||||
@@ -120,7 +120,7 @@ func (r *Router) Route(target ModelInfo, req RouteRequest) Decision {
|
||||
if r.ocrTool != "" {
|
||||
return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "route to OCR tool"}
|
||||
}
|
||||
// No tool is bound. A sighted main model remains the only option.
|
||||
// 未绑定任何工具。能看图的主模型仍是唯一选择。
|
||||
if target.SupportsVision {
|
||||
return Decision{Kind: KindMain, Reason: "no vision/OCR tool bound; keep image with main model"}
|
||||
}
|
||||
|
||||
@@ -96,14 +96,14 @@ func TestRouter_Route(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// fakeVision implements VisionAdapter for tool tests.
|
||||
// fakeVision 为工具测试实现 VisionAdapter。
|
||||
type fakeVision struct{ desc string }
|
||||
|
||||
func (f *fakeVision) Describe(_ context.Context, _ ImageRef, _ string) (string, error) {
|
||||
return f.desc, nil
|
||||
}
|
||||
|
||||
// fakeOCR implements OCRAdapter for tool tests.
|
||||
// fakeOCR 为工具测试实现 OCRAdapter。
|
||||
type fakeOCR struct{ text string }
|
||||
|
||||
func (f *fakeOCR) Extract(_ context.Context, _ ImageRef) (string, error) {
|
||||
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/pkg/plugins/sdk"
|
||||
)
|
||||
|
||||
// toolParameters returns the shared input JSON Schema for the vision and OCR
|
||||
// tools. It builds a fresh map each call rather than sharing a package-level
|
||||
// variable.
|
||||
// toolParameters 返回视觉工具与 OCR 工具共用的输入 JSON Schema。
|
||||
// 它每次调用都新建一个 map,而不是共享包级
|
||||
// 变量。
|
||||
func toolParameters() map[string]any {
|
||||
return map[string]any{
|
||||
"type": "object",
|
||||
@@ -27,33 +27,33 @@ func toolParameters() map[string]any {
|
||||
}
|
||||
}
|
||||
|
||||
// VisionTool wraps a VisionAdapter as an sdk.Tool so the M2 tool loop can invoke
|
||||
// it and return its description as a tool result, keeping the image out of the
|
||||
// main-model context. It is safe for concurrent use.
|
||||
// VisionTool 把 VisionAdapter 包装成 sdk.Tool,使 M2 工具循环能调用
|
||||
// 它并把描述作为工具结果返回,从而让图像不进入
|
||||
// 主模型上下文。它可安全并发使用。
|
||||
type VisionTool struct {
|
||||
name string
|
||||
adapter VisionAdapter
|
||||
}
|
||||
|
||||
// NewVisionTool builds a VisionTool with the given name and adapter.
|
||||
// NewVisionTool 用给定名字与适配器构造一个 VisionTool。
|
||||
func NewVisionTool(name string, adapter VisionAdapter) *VisionTool {
|
||||
return &VisionTool{name: name, adapter: adapter}
|
||||
}
|
||||
|
||||
// Name returns the tool identifier the agent uses to invoke it.
|
||||
// Name 返回智能体用来调用该工具的工具标识。
|
||||
func (t *VisionTool) Name() string { return t.name }
|
||||
|
||||
// Description explains what the tool does.
|
||||
// Description 说明该工具做什么。
|
||||
func (t *VisionTool) Description() string {
|
||||
return "Understand an image and return a description of its contents."
|
||||
}
|
||||
|
||||
// Parameters returns the input JSON Schema.
|
||||
// Parameters 返回输入 JSON Schema。
|
||||
func (t *VisionTool) Parameters() (map[string]any, error) {
|
||||
return toolParameters(), nil
|
||||
}
|
||||
|
||||
// Execute runs the adapter on the image from args and returns the description.
|
||||
// Execute 对 args 中的图像运行适配器并返回描述。
|
||||
func (t *VisionTool) Execute(ctx context.Context, args any) (any, error) {
|
||||
image, prompt, err := toolArgs(args)
|
||||
if err != nil {
|
||||
@@ -70,32 +70,32 @@ func (t *VisionTool) Execute(ctx context.Context, args any) (any, error) {
|
||||
return map[string]any{"description": desc}, nil
|
||||
}
|
||||
|
||||
// OCRTool wraps an OCRAdapter as an sdk.Tool. It returns the extracted text as a
|
||||
// tool result. It is safe for concurrent use.
|
||||
// OCRTool 把 OCRAdapter 包装成 sdk.Tool。它把抽取出的文字作为
|
||||
// 工具结果返回。它可安全并发使用。
|
||||
type OCRTool struct {
|
||||
name string
|
||||
adapter OCRAdapter
|
||||
}
|
||||
|
||||
// NewOCRTool builds an OCRTool with the given name and adapter.
|
||||
// NewOCRTool 用给定名字与适配器构造一个 OCRTool。
|
||||
func NewOCRTool(name string, adapter OCRAdapter) *OCRTool {
|
||||
return &OCRTool{name: name, adapter: adapter}
|
||||
}
|
||||
|
||||
// Name returns the tool identifier.
|
||||
// Name 返回工具标识。
|
||||
func (t *OCRTool) Name() string { return t.name }
|
||||
|
||||
// Description explains what the tool does.
|
||||
// Description 说明该工具做什么。
|
||||
func (t *OCRTool) Description() string {
|
||||
return "Extract text from an image (OCR) and return the recognized text."
|
||||
}
|
||||
|
||||
// Parameters returns the input JSON Schema.
|
||||
// Parameters 返回输入 JSON Schema。
|
||||
func (t *OCRTool) Parameters() (map[string]any, error) {
|
||||
return toolParameters(), nil
|
||||
}
|
||||
|
||||
// Execute runs the OCR adapter on the image from args and returns the text.
|
||||
// Execute 对 args 中的图像运行 OCR 适配器并返回文字。
|
||||
func (t *OCRTool) Execute(ctx context.Context, args any) (any, error) {
|
||||
image, _, err := toolArgs(args)
|
||||
if err != nil {
|
||||
@@ -112,9 +112,9 @@ func (t *OCRTool) Execute(ctx context.Context, args any) (any, error) {
|
||||
return map[string]any{"text": text}, nil
|
||||
}
|
||||
|
||||
// RegisterTools registers the vision and OCR tools with r via the SDK helper.
|
||||
// Each tool is only registered when its pointer is non-nil. It returns the
|
||||
// first registration error, or nil when every tool registered.
|
||||
// RegisterTools 经 SDK 辅助函数把视觉工具与 OCR 工具注册到 r。
|
||||
// 每个工具仅在其指针非 nil 时注册。它返回
|
||||
// 第一个注册错误;全部注册成功时返回 nil。
|
||||
func RegisterTools(r sdk.Registrar, tool *VisionTool, ocr *OCRTool) error {
|
||||
if r == nil {
|
||||
return sdk.ErrNilRegistrar
|
||||
@@ -132,8 +132,8 @@ func RegisterTools(r sdk.Registrar, tool *VisionTool, ocr *OCRTool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// toolArgs decodes sdk.Tool.Execute args into the image string and an optional
|
||||
// prompt. Args must be a map[string]any with a non-empty "image" string.
|
||||
// toolArgs 把 sdk.Tool.Execute 的 args 解码为图像字符串与一个可选的
|
||||
// prompt。args 必须是含非空 "image" 字符串的 map[string]any。
|
||||
func toolArgs(args any) (image, prompt string, err error) {
|
||||
m, ok := args.(map[string]any)
|
||||
if !ok {
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
// Package vision implements the layered image routing described in
|
||||
// DevDocs/design/06-多模态.md §0.
|
||||
// Package vision 实现 DevDocs/design/06-多模态.md §0 所述的
|
||||
// 分层图像路由。
|
||||
//
|
||||
// A request that carries image content is first checked by DetectMultimodal.
|
||||
// The VisionRouter then picks one of the layered routes: the main model
|
||||
// directly (when it accepts images), a tool-shaped vision/OCR adapter (when the
|
||||
// main model self-assesses poorly or cannot see), or the OCR tool (when the
|
||||
// image is mainly text). Vision/OCR adapters are registered as sdk.Tool
|
||||
// capabilities through the capability registry, and their output is a tool
|
||||
// result, so the image itself never occupies main-model context.
|
||||
// 携带图像内容的请求先由 DetectMultimodal 检查。
|
||||
// 随后 VisionRouter 选出其中一条分层路由:主模型
|
||||
// 直连(当它接受图像时)、工具形态的 vision/OCR 适配器(当主模型
|
||||
// 自评较差或看不见图像时),或 OCR 工具(当
|
||||
// 图像以文字为主时)。vision/OCR 适配器经能力注册中心
|
||||
// 注册为 sdk.Tool 能力,其输出即工具
|
||||
// 结果,因此图像本身从不占用主模型上下文。
|
||||
//
|
||||
// Images may come from a local path, an http(s) URL, or a base64/data: URI.
|
||||
// 图像可来自本地路径、http(s) URL 或 base64/data: URI。
|
||||
package vision
|
||||
|
||||
import (
|
||||
@@ -18,11 +18,11 @@ import (
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||
)
|
||||
|
||||
// DetectMultimodal reports whether msgs contains any non-text content part,
|
||||
// i.e. whether multimodal handling is required. Today model only distinguishes
|
||||
// text from image content, so this is true exactly when a message carries an
|
||||
// image part; the check is written against "not text" so future content types
|
||||
// (audio) extend it without rework.
|
||||
// DetectMultimodal 报告 msgs 是否含任何非文本内容部件,
|
||||
// 即是否需要多模态处理。当前 model 只区分
|
||||
// 文本与图像内容,因此这等价于某条消息携带
|
||||
// 图像部件;判断写成「非文本」是为了让未来的内容类型
|
||||
// (音频)无需返工即可扩展。
|
||||
func DetectMultimodal(msgs []model.Message) bool {
|
||||
for _, m := range msgs {
|
||||
for _, part := range m.Content {
|
||||
@@ -34,30 +34,30 @@ func DetectMultimodal(msgs []model.Message) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// ImageSource classifies where an image's bytes come from.
|
||||
// ImageSource 归类一副图像的字节来自何处。
|
||||
type ImageSource string
|
||||
|
||||
const (
|
||||
// SourcePath is a local filesystem path.
|
||||
// SourcePath 是本地文件系统路径。
|
||||
SourcePath ImageSource = "path"
|
||||
// SourceURL is an http(s):// URL.
|
||||
// SourceURL 是 http(s):// URL。
|
||||
SourceURL ImageSource = "url"
|
||||
// SourceBase64 is a base64 payload, optionally wrapped in a data: URI.
|
||||
// SourceBase64 是 base64 载荷,可选地包在 data: URI 中。
|
||||
SourceBase64 ImageSource = "base64"
|
||||
)
|
||||
|
||||
// ImageRef is a single image reference plus its resolved source kind.
|
||||
// ImageRef 是单个图像引用及其已归类的来源种类。
|
||||
type ImageRef struct {
|
||||
// Source is the classified source kind.
|
||||
// Source 是归类后的来源种类。
|
||||
Source ImageSource
|
||||
// Data is the local path, URL, or base64/data: payload.
|
||||
// Data 是本地路径、URL 或 base64/data: 载荷。
|
||||
Data string
|
||||
}
|
||||
|
||||
// ParseImageRef classifies a raw image source string into an ImageRef. Empty
|
||||
// input is rejected. Local paths that exist on disk are classified as paths;
|
||||
// http(s) URLs as URLs; every other value, including a data: URI, is treated as
|
||||
// a base64 payload. This keeps routing deterministic without guessing MIME.
|
||||
// ParseImageRef 把原始图像来源字符串归类为 ImageRef。空
|
||||
// 输入被拒绝。磁盘上存在的本地路径归类为路径;
|
||||
// http(s) URL 归类为 URL;其余所有值(含 data: URI)都视为
|
||||
// base64 载荷。这样无需猜测 MIME 即可保持路由确定性。
|
||||
func ParseImageRef(src string) (ImageRef, error) {
|
||||
if src == "" {
|
||||
return ImageRef{}, ErrNoImage
|
||||
@@ -68,7 +68,7 @@ func ParseImageRef(src string) (ImageRef, error) {
|
||||
if strings.HasPrefix(src, "file://") {
|
||||
return ImageRef{Source: SourcePath, Data: strings.TrimPrefix(src, "file://")}, nil
|
||||
}
|
||||
// A data: URI is base64 already; treat as base64, not path.
|
||||
// data: URI 本身就是 base64;按 base64 处理,而非路径。
|
||||
if strings.HasPrefix(src, "data:") {
|
||||
return ImageRef{Source: SourceBase64, Data: src}, nil
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user