AskaEth
983bc49218
- 原缺陷：vision 适配器把 http(s) URL **原样转给上游 provider**，而 upload 返回的是同源局域网地址 ⇒ 云 provider 根本取不到 ⇒ **B1（大图绕开 WS 4096 字节上限）在自部署形态下等于没做** - 修法：① `imageURL` → `localizeRef`：**彻底不再直传 URL**；同源 upload 产物走**本地读取**， 其余 http(s) 由服务端自己取回 → 一律内联为 `data:` URI 再交 provider（OpenAI 与 Anthropic 双协议都能收图） - 顺手修「类型瞎猜」：新增 `vision/mime.go`，**字节嗅探是唯一裁决**——判得出白名单光栅图就用字节结果； 字节是文本（HTML/PROXY 错误页、SVG）→ `ErrNotImage`；判不定但来源明确声明 `image/*` 才采信声明。 错误信息只含类型名，**不含响应体/URL**（不打印图片内容） - 防护：取回复用既有 http/https 白名单 + 50MiB 上限，新增 **30s 取回超时**； 本地读取路径 `localUploadPath` 严格认定 `<host>/api/v1/files/<name>`（无 query/fragment、单段路径）+ 32 位小写十六进制名 + `filepath.Dir` 包含性第二道 ⇒ `../`、`%2e%2e%2f`、反斜杠等编码变体在触碰 FS 前即拒 - 生产接线（主会话补）：`cmd/cyrene/vision.go` 传 `vision.WithUploadsDir(<DataDir>/uploads)` - 复核：build/vet 绿；新增 51 条断言（含 httptest 真取回 → 断言上游收到 data URI 且不含源 URL、 同源 upload 用不可达 host 证明走本地、HTML 谎报 image/png 被字节否决、越界名 19 例）； **3 条变异反证**（恢复 URL 直传 8 红 / 文本静默回退 png 红 / 去掉穿越检查 12 红）均已还原 - 未做（已登记）：历史里 URL 形态图片每轮重新取回（缓解＝持久化时内联/缓存）； Anthropic 只认 png/jpeg/gif/webp 而白名单含 bmp；`fetchURL` 无 IP 过滤（链路本地 169.254.169.254 可达）→ 需收紧
48 lines
1.9 KiB
Go
48 lines
1.9 KiB
Go
package main
|
|
|
|
import (
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/logger"
|
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/multimodal/vision"
|
|
)
|
|
|
|
// visionToolName and ocrToolName are the registered names of the vision and OCR
|
|
// tools. They are bound to the tool registration here and advertised to the
|
|
// agent by the tool dispatcher.
|
|
const (
|
|
visionToolName = "vision"
|
|
ocrToolName = "ocr"
|
|
)
|
|
|
|
// wireVision registers the vision and OCR tools and the vision model routing.
|
|
//
|
|
// The vision adapter is an OpenAI-compatible /chat/completions endpoint on the
|
|
// same base URL as the chat LLM (LLM_BASE_URL), using the dedicated model
|
|
// (LLM_VISION_MODEL) when set. When no base URL is configured the tools are
|
|
// skipped with a warning so startup never blocks; the model router is still
|
|
// bound to the vision model (in wireRuntime) whenever one is configured.
|
|
func (a *app) wireVision() {
|
|
log := logger.WithSource(a.log, "cmd/cyrene")
|
|
if strings.TrimSpace(a.cfg.LLM.BaseURL) == "" {
|
|
log.Warn("vision/ocr tools skipped: no LLM base url configured")
|
|
return
|
|
}
|
|
adapter := vision.NewOpenAICompatible(a.cfg.LLM,
|
|
vision.WithModel(a.cfg.LLM.VisionModel),
|
|
// 同源 upload 产物（`/api/v1/files/<name>`）走**本地读取**：与自己取回相比更快，
|
|
// 且在「服务端路由不到自己的广告 host」（如安卓模拟器 10.0.2.2、反向代理改写）
|
|
// 的部署下才取得到图。目录必须与 upload 的落盘位置一致 = `<DataDir>/uploads`。
|
|
vision.WithUploadsDir(filepath.Join(resolveDataDir(a.cfg), "uploads")),
|
|
vision.WithLogger(a.log))
|
|
if err := vision.RegisterTools(a.caps,
|
|
vision.NewVisionTool(visionToolName, adapter),
|
|
vision.NewOCRTool(ocrToolName, adapter),
|
|
); err != nil {
|
|
log.Warn("vision/ocr tool register skipped", "error", err)
|
|
return
|
|
}
|
|
log.Info("vision/ocr tools registered", "vision_model", a.cfg.LLM.VisionModel)
|
|
}
|