fix(vision): 服务端自己取回图片 URL 并内联为 data URI,修自部署下取不到图(F3,优先级「高」)
- 原缺陷:vision 适配器把 http(s) URL **原样转给上游 provider**,而 upload 返回的是同源局域网地址 ⇒ 云 provider 根本取不到 ⇒ **B1(大图绕开 WS 4096 字节上限)在自部署形态下等于没做** - 修法:① `imageURL` → `localizeRef`:**彻底不再直传 URL**;同源 upload 产物走**本地读取**, 其余 http(s) 由服务端自己取回 → 一律内联为 `data:` URI 再交 provider(OpenAI 与 Anthropic 双协议都能收图) - 顺手修「类型瞎猜」:新增 `vision/mime.go`,**字节嗅探是唯一裁决**——判得出白名单光栅图就用字节结果; 字节是文本(HTML/PROXY 错误页、SVG)→ `ErrNotImage`;判不定但来源明确声明 `image/*` 才采信声明。 错误信息只含类型名,**不含响应体/URL**(不打印图片内容) - 防护:取回复用既有 http/https 白名单 + 50MiB 上限,新增 **30s 取回超时**; 本地读取路径 `localUploadPath` 严格认定 `<host>/api/v1/files/<name>`(无 query/fragment、单段路径)+ 32 位小写十六进制名 + `filepath.Dir` 包含性第二道 ⇒ `../`、`%2e%2e%2f`、反斜杠等编码变体在触碰 FS 前即拒 - 生产接线(主会话补):`cmd/cyrene/vision.go` 传 `vision.WithUploadsDir(<DataDir>/uploads)` - 复核:build/vet 绿;新增 51 条断言(含 httptest 真取回 → 断言上游收到 data URI 且不含源 URL、 同源 upload 用不可达 host 证明走本地、HTML 谎报 image/png 被字节否决、越界名 19 例); **3 条变异反证**(恢复 URL 直传 8 红 / 文本静默回退 png 红 / 去掉穿越检查 12 红)均已还原 - 未做(已登记):历史里 URL 形态图片每轮重新取回(缓解=持久化时内联/缓存); Anthropic 只认 png/jpeg/gif/webp 而白名单含 bmp;`fetchURL` 无 IP 过滤(链路本地 169.254.169.254 可达)→ 需收紧
This commit is contained in:
1 parent
88d7dd0026
commit
983bc49218
11 files changed
+971
-55
No files matched your search
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/logger"
|
||||
@@ -30,6 +31,10 @@ func (a *app) wireVision() {
|
||||
}
|
||||
adapter := vision.NewOpenAICompatible(a.cfg.LLM,
|
||||
vision.WithModel(a.cfg.LLM.VisionModel),
|
||||
// 同源 upload 产物(`/api/v1/files/<name>`)走**本地读取**:与自己取回相比更快,
|
||||
// 且在「服务端路由不到自己的广告 host」(如安卓模拟器 10.0.2.2、反向代理改写)
|
||||
// 的部署下才取得到图。目录必须与 upload 的落盘位置一致 = `<DataDir>/uploads`。
|
||||
vision.WithUploadsDir(filepath.Join(resolveDataDir(a.cfg), "uploads")),
|
||||
vision.WithLogger(a.log))
|
||||
if err := vision.RegisterTools(a.caps,
|
||||
vision.NewVisionTool(visionToolName, adapter),
|
||||
|
||||
@@ -40,11 +40,12 @@ type OCRAdapter interface {
|
||||
// 模型 + OCR 模型」):Describe 用 model(LLM_VISION_MODEL),Extract 用
|
||||
// ocrModel(LLM_OCR_MODEL),未配 OCR 模型时回落到同一个模型。
|
||||
type OpenAICompatibleAdapter struct {
|
||||
cfg config.LLMConfig
|
||||
model string
|
||||
ocrModel string
|
||||
client *http.Client
|
||||
log *slog.Logger
|
||||
cfg config.LLMConfig
|
||||
model string
|
||||
ocrModel string
|
||||
client *http.Client
|
||||
log *slog.Logger
|
||||
uploadsDir string
|
||||
}
|
||||
|
||||
// OpenAIOption 用于定制 OpenAICompatibleAdapter。
|
||||
@@ -87,6 +88,18 @@ func WithOCRModel(m string) OpenAIOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithUploadsDir 配置本机上传目录(<DataDir>/uploads,即网关静态路由
|
||||
// /api/v1/files/ 的根),使指向本服务的上传 URL 直接走本地文件读取,
|
||||
// 而不是绕一圈网络回自己的 HTTP 路由。留空时上传 URL 按普通 URL 取回,
|
||||
// 行为与配置前一致。
|
||||
func WithUploadsDir(dir string) OpenAIOption {
|
||||
return func(a *OpenAICompatibleAdapter) {
|
||||
if d := strings.TrimSpace(dir); d != "" {
|
||||
a.uploadsDir = d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewOpenAICompatible 用 cfg 构造一个适配器,未覆盖时使用
|
||||
// cfg.Model。
|
||||
func NewOpenAICompatible(cfg config.LLMConfig, opts ...OpenAIOption) *OpenAICompatibleAdapter {
|
||||
@@ -163,19 +176,40 @@ func (a *OpenAICompatibleAdapter) ask(ctx context.Context, model string, img Ima
|
||||
return a.decode(resp.Body)
|
||||
}
|
||||
|
||||
// imageURL 把 ImageRef 转换为线上格式期望的 image_url 取值:http(s)
|
||||
// URL 原样透传,其余一律转为 base64 data: URI。
|
||||
// imageURL 返回线上 image_url 的取值。无论来源是 http(s) URL、本地路径
|
||||
// 还是 base64,送上游的字节一律由服务端自己取回并内联为 data: URI:
|
||||
// 把同源(局域网)URL 原样透传给云 provider,对方根本取不到,
|
||||
// 自部署形态下视觉链路会整条不通(B1 上传大图的实际意义也就落空)。
|
||||
// URL 取回沿用 Resolve→fetchURL 的既有守卫(http/https 白名单 + 50MiB
|
||||
// 体积上限),并加取回超时(imageFetchTimeout)。
|
||||
func (a *OpenAICompatibleAdapter) imageURL(ctx context.Context, img ImageRef) (string, error) {
|
||||
if img.Source == SourceURL {
|
||||
return img.Data, nil
|
||||
ref := a.localizeRef(img)
|
||||
if ref.Source == SourceURL {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, imageFetchTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
mime, data, err := Resolve(ctx, img, a.client)
|
||||
mime, data, err := Resolve(ctx, ref, a.client)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return DataURI(mime, data), nil
|
||||
}
|
||||
|
||||
// localizeRef 把指向本服务上传路由的 URL 改写成 uploads 目录下的本地路径。
|
||||
// uploadsDir 未配置、或 URL 不是本服务的上传形状时原样返回(此时仍会按
|
||||
// 普通 URL 取回,因此行为不退化);文件缺失等读取错误由 Resolve 如实报告。
|
||||
func (a *OpenAICompatibleAdapter) localizeRef(img ImageRef) ImageRef {
|
||||
if img.Source != SourceURL || a.uploadsDir == "" {
|
||||
return img
|
||||
}
|
||||
path, ok := localUploadPath(img.Data, a.uploadsDir)
|
||||
if !ok {
|
||||
return img
|
||||
}
|
||||
return ImageRef{Source: SourcePath, Data: path}
|
||||
}
|
||||
|
||||
// openAIVisionRequest 是发往 /chat/completions 的 JSON 请求体。
|
||||
type openAIVisionRequest struct {
|
||||
Model string `json:"model"`
|
||||
|
||||
@@ -17,6 +17,9 @@ var (
|
||||
ErrUpstream = errors.New("vision: upstream request failed")
|
||||
// ErrBadImage 在图像无法读取或解码时返回。
|
||||
ErrBadImage = errors.New("vision: cannot read image")
|
||||
// ErrNotImage 在来源声明的类型与真实字节都判不出图像时返回
|
||||
// (例如把 HTML/PROXY 错误页当成图片送进模型)。
|
||||
ErrNotImage = errors.New("vision: source is not an image")
|
||||
// ErrBadArgs 在工具参数格式错误时由该工具返回。
|
||||
ErrBadArgs = errors.New("vision: bad tool arguments")
|
||||
)
|
||||
@@ -0,0 +1,107 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||
)
|
||||
|
||||
// writeFileBytes 写一个测试用的字节文件。
|
||||
func writeFileBytes(path string, data []byte) error {
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
return fmt.Errorf("write test file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeFileBytesAll 在 dir 下写 name(含父目录),用于摆放测试用的上传产物。
|
||||
func writeFileBytesAll(dir, name string, data []byte) error {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return fmt.Errorf("create test dir: %w", err)
|
||||
}
|
||||
return writeFileBytes(filepath.Join(dir, name), data)
|
||||
}
|
||||
|
||||
// imageURLRecorder 是一个 /chat/completions endpoint,记录发往上游的
|
||||
// image_url 取值,使测试能断言「服务端真的把图取回并内联了」而不是
|
||||
// 原样透传 URL。
|
||||
type imageURLRecorder struct {
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
urls []string
|
||||
}
|
||||
|
||||
func (r *imageURLRecorder) seen() (int, []string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.calls, append([]string(nil), r.urls...)
|
||||
}
|
||||
|
||||
// newImageURLRecorderServer 起一个上游 endpoint,把每次请求里的
|
||||
// image_url.url 记下来。
|
||||
func newImageURLRecorderServer(t *testing.T) (*httptest.Server, *imageURLRecorder) {
|
||||
t.Helper()
|
||||
rec := &imageURLRecorder{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Messages []struct {
|
||||
Content []struct {
|
||||
ImageURL *struct {
|
||||
URL string `json:"url"`
|
||||
} `json:"image_url"`
|
||||
} `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Errorf("decode upstream body: %v", err)
|
||||
}
|
||||
rec.mu.Lock()
|
||||
rec.calls++
|
||||
for _, m := range body.Messages {
|
||||
for _, part := range m.Content {
|
||||
if part.ImageURL != nil {
|
||||
rec.urls = append(rec.urls, part.ImageURL.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
rec.mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"a small square"}}]}`)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv, rec
|
||||
}
|
||||
|
||||
// newImageServer 起一个提供图像字节的来源服务,contentType 为 "" 时
|
||||
// 刻意不发 Content-Type(考验字节嗅探)。
|
||||
func newImageServer(t *testing.T, data []byte, contentType string, hits *atomic.Int32) *httptest.Server {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if hits != nil {
|
||||
hits.Add(1)
|
||||
}
|
||||
if contentType == "" {
|
||||
w.Header()["Content-Type"] = nil
|
||||
w.Header().Set("Content-Type", "")
|
||||
} else {
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
}
|
||||
_, _ = w.Write(data)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
// newAdapter 构造一个指向 up 的适配器,opts 追加在必需项之后。
|
||||
func newAdapter(up string, opts ...OpenAIOption) *OpenAICompatibleAdapter {
|
||||
return NewOpenAICompatible(config.LLMConfig{BaseURL: up, Model: "vision-test"}, opts...)
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 读取图像字节与响应体的体积上限,使恶意或
|
||||
@@ -18,6 +19,11 @@ const (
|
||||
maxFetchedBody = 50 << 20
|
||||
)
|
||||
|
||||
// imageFetchTimeout 限制一次图像 URL 取回的时长。适配器与消息解析器会
|
||||
// 为模型代取 URL,没有上限时一个卡住或恶意的地址会一直占着这一轮
|
||||
// (fetchURL 复用默认 client,其本身不设整体超时)。
|
||||
const imageFetchTimeout = 30 * time.Second
|
||||
|
||||
// DataURI 由 mime 与原始字节构造一个 base64 data: 图像 URI。
|
||||
func DataURI(mime string, data []byte) string {
|
||||
return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data)
|
||||
@@ -40,7 +46,8 @@ func Resolve(ctx context.Context, ref ImageRef, client *http.Client) (mime strin
|
||||
}
|
||||
}
|
||||
|
||||
// readFile 读取一个图像文件,并从扩展名推断其 MIME 类型。
|
||||
// readFile 读取一个图像文件。扩展名给出声明的 MIME,真实字节嗅探是否决权
|
||||
// (见 imageMIMEFor):改名的 HTML 不会被当成图片读进模型。
|
||||
func readFile(path string) (string, []byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
@@ -51,7 +58,11 @@ func readFile(path string) (string, []byte, error) {
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("vision: %w: %v", ErrBadImage, err)
|
||||
}
|
||||
return mimeByExt(path), data, nil
|
||||
mime, err := imageMIMEFor(mimeByExt(path), data)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return mime, data, nil
|
||||
}
|
||||
|
||||
// decodeBase64 返回 base64/data: 载荷解码后的字节及其 MIME。
|
||||
@@ -72,6 +83,8 @@ func decodeBase64(src string) (string, []byte, error) {
|
||||
}
|
||||
|
||||
// fetchURL 通过 http(s) 下载一张图像;只允许 http/https 协议。
|
||||
// 声明的 Content-Type 只作参考,真实字节嗅探说了算(见 imageMIMEFor):
|
||||
// 网关/PROXY 返回的 HTML 错误页绝不会被当成 png 送进模型。
|
||||
func fetchURL(ctx context.Context, rawURL string, client *http.Client) (string, []byte, error) {
|
||||
if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
|
||||
return "", nil, fmt.Errorf("vision: %w: %q", ErrUnsupportedSource, rawURL)
|
||||
@@ -95,7 +108,35 @@ func fetchURL(ctx context.Context, rawURL string, client *http.Client) (string,
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("vision: %w: %v", ErrFetch, err)
|
||||
}
|
||||
return mimeByContentType(resp.Header.Get("Content-Type")), data, nil
|
||||
mime, err := imageMIMEFor(resp.Header.Get("Content-Type"), data)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return mime, data, nil
|
||||
}
|
||||
|
||||
// inlineImageSrc 把图像来源字符串取回并内联为 data: URI,供主模型
|
||||
// 直接消费。已是 data: URI 的原样返回(客户端现有的 base64 路径因此
|
||||
// 逐字节不变)。URL 取回复用 Resolve→fetchURL 的既有守卫
|
||||
// (协议白名单 + 体积上限),并加取回超时。
|
||||
func inlineImageSrc(ctx context.Context, src string) (string, error) {
|
||||
if strings.HasPrefix(src, "data:") {
|
||||
return src, nil
|
||||
}
|
||||
ref, err := ParseImageRef(src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ref.Source == SourceURL {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, imageFetchTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
mime, data, err := Resolve(ctx, ref, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return DataURI(mime, data), nil
|
||||
}
|
||||
|
||||
// splitDataURI 解析 "data:<mime>;base64,<payload>" 前缀。src 不是
|
||||
@@ -124,46 +165,6 @@ func splitDataURI(src string) (mime, payload string, ok bool) {
|
||||
return mime, payload, true
|
||||
}
|
||||
|
||||
// mimeByExt 从文件路径扩展名推断 MIME 类型。
|
||||
func mimeByExt(path string) string {
|
||||
switch strings.ToLower(pathExtension(path)) {
|
||||
case ".png":
|
||||
return "image/png"
|
||||
case ".jpg", ".jpeg":
|
||||
return "image/jpeg"
|
||||
case ".gif":
|
||||
return "image/gif"
|
||||
case ".webp":
|
||||
return "image/webp"
|
||||
default:
|
||||
return "image/png"
|
||||
}
|
||||
}
|
||||
|
||||
// mimeByContentType 从 HTTP Content-Type 头返回 MIME 类型,
|
||||
// 缺失或非图像时默认为 image/png。
|
||||
func mimeByContentType(ct string) string {
|
||||
if ct == "" || !strings.HasPrefix(ct, "image/") {
|
||||
return "image/png"
|
||||
}
|
||||
return strings.TrimSpace(strings.Split(ct, ";")[0])
|
||||
}
|
||||
|
||||
// pathExtension 返回带点号的文件扩展名,没有则返回空串。
|
||||
func pathExtension(path string) string {
|
||||
base := path
|
||||
if idx := strings.LastIndexByte(path, '/'); idx >= 0 {
|
||||
base = path[idx+1:]
|
||||
} else if idx := strings.LastIndexByte(path, '\\'); idx >= 0 {
|
||||
base = path[idx+1:]
|
||||
}
|
||||
dot := strings.LastIndexByte(base, '.')
|
||||
if dot < 0 {
|
||||
return ""
|
||||
}
|
||||
return base[dot:]
|
||||
}
|
||||
|
||||
// fileExists 报告 path 是否存在于磁盘且为常规文件。
|
||||
func fileExists(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// sniffBytes 是字节嗅探读取的前缀长度:http.DetectContentType 只看前
|
||||
// 512 字节。
|
||||
const sniffBytes = 512
|
||||
|
||||
// knownRasterMIME 把一个 MIME 归一化到允许内联进视觉模型的光栅图
|
||||
// 白名单。白名单与网关上传同口径(internal/services/gateway/upload.go
|
||||
// 的 uploadExtForMIME):SVG/HTML 是可执行内容,绝不进模型。
|
||||
// 第二个返回值为 false 时该 MIME 不是白名单内的图像类型。
|
||||
func knownRasterMIME(mime string) (string, bool) {
|
||||
switch mediaType(mime) {
|
||||
case "image/png":
|
||||
return "image/png", true
|
||||
case "image/jpeg", "image/jpg":
|
||||
// image/jpg 不是标准写法,但确有其事,归一化到 image/jpeg。
|
||||
return "image/jpeg", true
|
||||
case "image/gif":
|
||||
return "image/gif", true
|
||||
case "image/webp":
|
||||
return "image/webp", true
|
||||
case "image/bmp":
|
||||
return "image/bmp", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// imageMediaType 返回 Content-Type 里声明的 image/* 媒体类型(去掉参数、
|
||||
// 转小写)。缺失、非 image/*、或载体可执行的 image/svg+xml 都返回 false。
|
||||
func imageMediaType(ct string) (string, bool) {
|
||||
m := mediaType(ct)
|
||||
if !strings.HasPrefix(m, "image/") || m == "image/svg+xml" {
|
||||
return "", false
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
|
||||
// mediaType 归一化一个 MIME 字符串:截掉 ";charset=..." 参数、去空白、转小写。
|
||||
func mediaType(mime string) string {
|
||||
return strings.ToLower(strings.TrimSpace(strings.Split(mime, ";")[0]))
|
||||
}
|
||||
|
||||
// sniffMIME 按真实字节嗅探类型。它是对声明类型(HTTP Content-Type 或文件
|
||||
// 扩展名)的否决权:两者都可能骗人,字节不会。
|
||||
func sniffMIME(data []byte) string {
|
||||
if len(data) > sniffBytes {
|
||||
data = data[:sniffBytes]
|
||||
}
|
||||
return http.DetectContentType(data)
|
||||
}
|
||||
|
||||
// isTextualMIME 报告 mime 是否是可读文本 —— HTML/PROXY 错误页、SVG、
|
||||
// JSON 等「看起来像图但其实是文本」的载荷都落在这里。
|
||||
func isTextualMIME(mime string) bool {
|
||||
m := mediaType(mime)
|
||||
if strings.HasPrefix(m, "text/") {
|
||||
return true
|
||||
}
|
||||
switch m {
|
||||
case "application/json", "application/xml", "application/xhtml+xml", "application/javascript":
|
||||
return true
|
||||
default:
|
||||
return strings.HasSuffix(m, "+xml")
|
||||
}
|
||||
}
|
||||
|
||||
// imageMIMEFor 决定内联给视觉模型的 MIME。declared 是来源声明的类型
|
||||
// (HTTP Content-Type 或文件扩展名),data 是真实字节:
|
||||
//
|
||||
// - 字节能判出白名单光栅图 → 以字节为准(声明的类型只作参考,
|
||||
// 谎报或错标的类型骗不了模型);
|
||||
// - 字节是文本(HTML/PROXY 错误页、SVG)→ 报错,绝不喂给模型;
|
||||
// - 字节判不定(application/octet-stream 等)且来源明确声明 image/* →
|
||||
// 采信声明(否则 avif 这类嗅探不出的格式会被无谓拒绝);
|
||||
// - 其余(含类型缺失且字节判不定)→ 报错。
|
||||
//
|
||||
// 绝不静默回退成 image/png:把一张 HTML 错误页当 PNG 交给视觉模型,
|
||||
// 会让模型对着乱码编描述。
|
||||
func imageMIMEFor(declared string, data []byte) (string, error) {
|
||||
sniffed := sniffMIME(data)
|
||||
if sniffedMIME, ok := knownRasterMIME(sniffed); ok {
|
||||
return sniffedMIME, nil
|
||||
}
|
||||
declaredMIME, declaredImage := imageMediaType(declared)
|
||||
if isTextualMIME(sniffed) {
|
||||
return "", fmt.Errorf("vision: %w: declared %s but the payload is %s",
|
||||
ErrNotImage, declaredLabel(declared), mediaType(sniffed))
|
||||
}
|
||||
if declaredImage {
|
||||
if known, ok := knownRasterMIME(declaredMIME); ok {
|
||||
return known, nil
|
||||
}
|
||||
return declaredMIME, nil
|
||||
}
|
||||
return "", fmt.Errorf("vision: %w: declared %s, payload is %s",
|
||||
ErrNotImage, declaredLabel(declared), mediaType(sniffed))
|
||||
}
|
||||
|
||||
// declaredLabel 把声明的类型渲染成错误信息里的可读标签;缺失时给出
|
||||
// 明确措辞,避免错误信息里出现空的引号。错误信息只含类型名,
|
||||
// 绝不含图像字节或来源 URL(后者可能带凭据)。
|
||||
func declaredLabel(declared string) string {
|
||||
if m := mediaType(declared); m != "" {
|
||||
return fmt.Sprintf("content type %q", m)
|
||||
}
|
||||
return "no content type"
|
||||
}
|
||||
|
||||
// mimeByExt 给出文件路径扩展名声明的 MIME;未知扩展名返回 "",交给字节
|
||||
// 嗅探裁决(不再静默假定 png)。
|
||||
func mimeByExt(path string) string {
|
||||
switch strings.ToLower(pathExtension(path)) {
|
||||
case ".png":
|
||||
return "image/png"
|
||||
case ".jpg", ".jpeg":
|
||||
return "image/jpeg"
|
||||
case ".gif":
|
||||
return "image/gif"
|
||||
case ".webp":
|
||||
return "image/webp"
|
||||
case ".bmp":
|
||||
return "image/bmp"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// pathExtension 返回带点号的文件扩展名,没有则返回空串。
|
||||
func pathExtension(path string) string {
|
||||
base := path
|
||||
if idx := strings.LastIndexByte(path, '/'); idx >= 0 {
|
||||
base = path[idx+1:]
|
||||
} else if idx := strings.LastIndexByte(path, '\\'); idx >= 0 {
|
||||
base = path[idx+1:]
|
||||
}
|
||||
dot := strings.LastIndexByte(base, '.')
|
||||
if dot < 0 {
|
||||
return ""
|
||||
}
|
||||
return base[dot:]
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// pngBytes 返回一张真实的最小 PNG(1x1),使「字节嗅探」在测试里
|
||||
// 有真实依据,而不是靠手写常量。
|
||||
func pngBytes(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, 1, 1))
|
||||
img.Set(0, 0, color.RGBA{R: 0x11, G: 0x22, B: 0x33, A: 0xff})
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, img); err != nil {
|
||||
t.Fatalf("encode png: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// htmlBytes 模拟网关/PROXY 的错误页:非图像响应体的典型形态。
|
||||
func htmlBytes() []byte {
|
||||
return []byte("<!DOCTYPE html><html><body>502 Bad Gateway</body></html>")
|
||||
}
|
||||
|
||||
// webpBytes 构造一个满足 RIFF/WEBPVP8 嗅探签名的 WebP 头。
|
||||
func webpBytes() []byte {
|
||||
return append([]byte("RIFF\x00\x00\x00\x00WEBPVP8 "), make([]byte, 16)...)
|
||||
}
|
||||
|
||||
// bmpBytes 构造一个满足 "BM" 嗅探签名的 BMP 头。
|
||||
func bmpBytes() []byte {
|
||||
return append([]byte("BM"), make([]byte, 32)...)
|
||||
}
|
||||
|
||||
// TestImageMIMEFor 覆盖「内容类型不能瞎猜」的裁决表:声明的类型只作参考,
|
||||
// 真实字节嗅探是否决权,判不出图像一律报错(绝不静默回退 image/png)。
|
||||
func TestImageMIMEFor(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
declared string
|
||||
data []byte
|
||||
wantMIME string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "png with correct type", declared: "image/png", data: pngBytes(t), wantMIME: "image/png"},
|
||||
{name: "png with charset param", declared: "image/png; charset=binary", data: pngBytes(t), wantMIME: "image/png"},
|
||||
{name: "png with missing type sniffs", declared: "", data: pngBytes(t), wantMIME: "image/png"},
|
||||
{name: "bytes win over a wrong declaration", declared: "image/jpg", data: pngBytes(t), wantMIME: "image/png"},
|
||||
{name: "webp sniffs", declared: "", data: webpBytes(), wantMIME: "image/webp"},
|
||||
{name: "bmp sniffs", declared: "", data: bmpBytes(), wantMIME: "image/bmp"},
|
||||
{name: "html declared as html", declared: "text/html; charset=utf-8", data: htmlBytes(), wantErr: true},
|
||||
{name: "html with missing type", declared: "", data: htmlBytes(), wantErr: true},
|
||||
{name: "html falsely declared png", declared: "image/png", data: htmlBytes(), wantErr: true},
|
||||
{name: "svg rejected", declared: "image/svg+xml", data: []byte(`<svg xmlns="http://www.w3.org/2000/svg"/>`), wantErr: true},
|
||||
{name: "empty payload", declared: "", data: nil, wantErr: true},
|
||||
{name: "unknown binary without type", declared: "", data: []byte{0x00, 0x01, 0x02, 0x03}, wantErr: true},
|
||||
{name: "declared image with unsniffable bytes trusts the declaration",
|
||||
declared: "image/avif", data: []byte{0x00, 0x00, 0x00, 0x20}, wantMIME: "image/avif"},
|
||||
{name: "declared jpg alias normalises in the fallback",
|
||||
declared: "image/jpg", data: []byte{0x00, 0x01, 0x02, 0x03}, wantMIME: "image/jpeg"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := imageMIMEFor(tt.declared, tt.data)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("imageMIMEFor(%q) = %q, want error", tt.declared, got)
|
||||
}
|
||||
if !errors.Is(err, ErrNotImage) {
|
||||
t.Fatalf("error = %v, want ErrNotImage", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("imageMIMEFor(%q) error: %v", tt.declared, err)
|
||||
}
|
||||
if got != tt.wantMIME {
|
||||
t.Fatalf("mime = %q, want %q", got, tt.wantMIME)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMimeByExtLeavesUnknownEmpty 钉住未知扩展名不再被猜成 png —— 交给
|
||||
// 字节嗅探裁决。
|
||||
func TestMimeByExtLeavesUnknownEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := mimeByExt("/tmp/notes.txt"); got != "" {
|
||||
t.Fatalf("mimeByExt(.txt) = %q, want empty (let the bytes decide)", got)
|
||||
}
|
||||
if got := mimeByExt("/tmp/cat.JPEG"); got != "image/jpeg" {
|
||||
t.Fatalf("mimeByExt(.JPEG) = %q, want image/jpeg", got)
|
||||
}
|
||||
if got := mimeByExt("/tmp/shot.bmp"); got != "image/bmp" {
|
||||
t.Fatalf("mimeByExt(.bmp) = %q, want image/bmp", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadFileRejectsRenamedHTML 钉住本地路径也不喂错类型:把 HTML 改名成
|
||||
// .png 不能再被当成 PNG 读进模型。
|
||||
func TestReadFileRejectsRenamedHTML(t *testing.T) {
|
||||
t.Parallel()
|
||||
path := t.TempDir() + "/liar.png"
|
||||
if err := writeFileBytes(path, htmlBytes()); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
mime, data, err := readFile(path)
|
||||
if err == nil {
|
||||
t.Fatalf("readFile = (%q, %d bytes), want error", mime, len(data))
|
||||
}
|
||||
if !errors.Is(err, ErrNotImage) {
|
||||
t.Fatalf("error = %v, want ErrNotImage", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadFileKeepsKnownImage 钉住正常本地图片不退化(含新增的 bmp)。
|
||||
func TestReadFileKeepsKnownImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
path := t.TempDir() + "/cat.png"
|
||||
want := pngBytes(t)
|
||||
if err := writeFileBytes(path, want); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
mime, data, err := readFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("readFile: %v", err)
|
||||
}
|
||||
if mime != "image/png" {
|
||||
t.Fatalf("mime = %q, want image/png", mime)
|
||||
}
|
||||
if !bytes.Equal(data, want) {
|
||||
t.Fatal("bytes changed on read")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDataURIStable 是 base64 路径逐字节不变的底线。
|
||||
func TestDataURIStable(t *testing.T) {
|
||||
t.Parallel()
|
||||
data := pngBytes(t)
|
||||
want := "data:image/png;base64," + base64.StdEncoding.EncodeToString(data)
|
||||
if got := DataURI("image/png", data); got != want {
|
||||
t.Fatalf("DataURI = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -73,8 +73,16 @@ func (r *MessageResolver) resolveImage(ctx context.Context, imageURL string) (mo
|
||||
decision := r.router.Route(target, RouteRequest{})
|
||||
switch decision.Kind {
|
||||
case KindMain:
|
||||
// 支持视觉的主模型直接看到这张图。
|
||||
return model.Content{Type: model.ContentTypeImage, ImageURL: imageURL}, nil
|
||||
// 支持视觉的主模型直接看到这张图;但 URL 与本地路径必须先由服务端
|
||||
// 取回并内联为 data: URI——云 provider 取不到自部署的局域网地址
|
||||
// (如 http://192.168.x.x:10942/api/v1/files/…),Anthropic 协议更是
|
||||
// 只接受 base64 data: URI(model/anthropic_encode.go 会拒绝裸 URL)。
|
||||
// 取回失败时返回错误,由 loop 保留原消息(该轮不因此中断)。
|
||||
src, err := inlineImageSrc(ctx, imageURL)
|
||||
if err != nil {
|
||||
return model.Content{}, err
|
||||
}
|
||||
return model.Content{Type: model.ContentTypeImage, ImageURL: src}, nil
|
||||
case KindVisionTool:
|
||||
return r.invokeTool(ctx, decision.Tool, map[string]any{
|
||||
"image": imageURL,
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// uploadFilePrefix 是本服务提供已存上传的静态路由前缀,与网关
|
||||
// internal/services/gateway/upload.go 的 uploadFilePrefix 一致。
|
||||
// 它位于 /api/v1/ 之下,因此属于版本化客户端契约。
|
||||
const uploadFilePrefix = "/api/v1/files/"
|
||||
|
||||
// uploadNameBytes 是上传文件主名的字节数(16 字节 = 32 位十六进制),
|
||||
// 与网关 randomUploadName 一致。
|
||||
const uploadNameBytes = 16
|
||||
|
||||
// localUploadPath 判断 rawURL 是否指向本服务的已存上传,若是则返回它在
|
||||
// uploadsDir 下的本地路径。命中时直接读盘:比绕一圈 HTTP 回自己更快,
|
||||
// 也不需要网络可达(自部署时同源 URL 用的是局域网地址,服务端自己
|
||||
// 未必路由得到,云 provider 则必然取不到)。
|
||||
//
|
||||
// 认可的形状严格为 <http|https>://<host>/api/v1/files/<name>,且 <name>
|
||||
// 必须通过 validUploadName。查询串、片段、多余路径段、任何分隔符或
|
||||
// 点号段都在触碰文件系统之前被拒绝;最后的包含性检查是
|
||||
// 第二道防线,即使名字校验被改弱,路径也不得逃出 uploadsDir。
|
||||
//
|
||||
// 注意 host 不参与判断:按网关设计,128 位随机文件名本身就是能力
|
||||
// (静态上传路由刻意不认证),因此指向别处的同名 URL 也只解析到
|
||||
// 本机同名上传,不会替调用方去取第三方内容。
|
||||
func localUploadPath(rawURL, uploadsDir string) (string, bool) {
|
||||
if strings.TrimSpace(uploadsDir) == "" {
|
||||
return "", false
|
||||
}
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return "", false
|
||||
}
|
||||
if u.Host == "" || u.RawQuery != "" || u.Fragment != "" {
|
||||
return "", false
|
||||
}
|
||||
name, ok := strings.CutPrefix(u.Path, uploadFilePrefix)
|
||||
if !ok || !validUploadName(name) {
|
||||
return "", false
|
||||
}
|
||||
path := filepath.Join(uploadsDir, name)
|
||||
if filepath.Dir(path) != filepath.Clean(uploadsDir) {
|
||||
return "", false
|
||||
}
|
||||
return path, true
|
||||
}
|
||||
|
||||
// validUploadName 报告 name 是否恰好是 32 位小写十六进制主名加上白名单
|
||||
// 内的图片扩展名。它与网关静态路由的路径穿越防护同口径
|
||||
// (upload.go 的 validUploadName):分隔符、点、双点及任何其他形态都在
|
||||
// 触碰文件系统之前被拒绝。
|
||||
//
|
||||
// 这里不复用网关那份实现是为了不让本包依赖 HTTP 服务层
|
||||
// (runtime → services 是反向依赖);两处规则必须一起改。
|
||||
func validUploadName(name string) bool {
|
||||
if name == "" || name != filepath.Base(name) || strings.ContainsAny(name, "/\\") {
|
||||
return false
|
||||
}
|
||||
stem, ext, ok := strings.Cut(name, ".")
|
||||
if !ok || len(stem) != uploadNameBytes*2 || ext == "" {
|
||||
return false
|
||||
}
|
||||
for _, c := range stem {
|
||||
if !isLowerHex(c) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return knownUploadExt(ext)
|
||||
}
|
||||
|
||||
// knownUploadExt 报告 ext(不含点)是否是白名单内的图片扩展名,与网关
|
||||
// uploadExtForMIME 的可写侧一致:只含光栅图。
|
||||
func knownUploadExt(ext string) bool {
|
||||
_, ok := knownRasterMIME(mimeByExt("upload." + strings.ToLower(ext)))
|
||||
return ok
|
||||
}
|
||||
|
||||
// isLowerHex 报告 c 是否是 ASCII 小写十六进制数字。
|
||||
func isLowerHex(c rune) bool {
|
||||
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||
)
|
||||
|
||||
// uploadTestName 是一个形状合法的上传文件名(32 位小写十六进制 + .png)。
|
||||
const uploadTestName = "0123456789abcdef0123456789abcdef.png"
|
||||
|
||||
// TestLocalUploadPath 是上传 URL → 本地路径的路径穿越防护用例表:
|
||||
// 只有严格形状才被接受,任何分隔符/点号段/编码变体都在触碰文件系统前被拒。
|
||||
func TestLocalUploadPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := filepath.Join("tmp", "uploads")
|
||||
tests := []struct {
|
||||
name string
|
||||
rawURL string
|
||||
dir string
|
||||
want string
|
||||
wantHit bool
|
||||
}{
|
||||
{name: "http upload url", rawURL: "http://192.168.1.5:10942/api/v1/files/" + uploadTestName,
|
||||
dir: dir, want: filepath.Join(dir, uploadTestName), wantHit: true},
|
||||
{name: "https upload url", rawURL: "https://box.local/api/v1/files/" + uploadTestName,
|
||||
dir: dir, want: filepath.Join(dir, uploadTestName), wantHit: true},
|
||||
{name: "traversal", rawURL: "http://h/api/v1/files/../../../data/cyrene.db", dir: dir, wantHit: false},
|
||||
{name: "encoded traversal", rawURL: "http://h/api/v1/files/%2e%2e%2fcyrene.db", dir: dir, wantHit: false},
|
||||
{name: "backslash", rawURL: `http://h/api/v1/files/..\cyrene.db`, dir: dir, wantHit: false},
|
||||
{name: "absolute path name", rawURL: "http://h/api/v1/files//etc/passwd", dir: dir, wantHit: false},
|
||||
{name: "uppercase hex", rawURL: "http://h/api/v1/files/" + strings.ToUpper(uploadTestName), dir: dir, wantHit: false},
|
||||
{name: "short stem", rawURL: "http://h/api/v1/files/abcdef.png", dir: dir, wantHit: false},
|
||||
{name: "non hex stem", rawURL: "http://h/api/v1/files/0123456789abcdef0123456789abcdeg.png", dir: dir, wantHit: false},
|
||||
{name: "html extension", rawURL: "http://h/api/v1/files/" + strings.TrimSuffix(uploadTestName, ".png") + ".html", dir: dir, wantHit: false},
|
||||
{name: "svg extension", rawURL: "http://h/api/v1/files/" + strings.TrimSuffix(uploadTestName, ".png") + ".svg", dir: dir, wantHit: false},
|
||||
{name: "extra segment", rawURL: "http://h/api/v1/files/sub/" + uploadTestName, dir: dir, wantHit: false},
|
||||
{name: "query string", rawURL: "http://h/api/v1/files/" + uploadTestName + "?x=1", dir: dir, wantHit: false},
|
||||
{name: "fragment", rawURL: "http://h/api/v1/files/" + uploadTestName + "#f", dir: dir, wantHit: false},
|
||||
{name: "other route", rawURL: "http://h/api/v1/other/" + uploadTestName, dir: dir, wantHit: false},
|
||||
{name: "ftp scheme", rawURL: "ftp://h/api/v1/files/" + uploadTestName, dir: dir, wantHit: false},
|
||||
{name: "relative url", rawURL: "/api/v1/files/" + uploadTestName, dir: dir, wantHit: false},
|
||||
{name: "remote without uploads dir", rawURL: "http://h/api/v1/files/" + uploadTestName, dir: "", wantHit: false},
|
||||
{name: "empty name", rawURL: "http://h/api/v1/files/", dir: dir, wantHit: false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, ok := localUploadPath(tt.rawURL, tt.dir)
|
||||
if ok != tt.wantHit {
|
||||
t.Fatalf("localUploadPath(%q, %q) = (%q, %v), want hit=%v", tt.rawURL, tt.dir, got, ok, tt.wantHit)
|
||||
}
|
||||
if tt.wantHit && got != tt.want {
|
||||
t.Fatalf("path = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_ReadsOwnUploadLocally 覆盖同源 upload 产物的
|
||||
// 快路径:URL 指向本服务的 /api/v1/files/<name> 时直接读盘,不发起网络
|
||||
// 请求(URL 的 host 在此不可达,若走了 HTTP 必然失败/超时)。
|
||||
func TestOpenAICompatibleAdapter_ReadsOwnUploadLocally(t *testing.T) {
|
||||
t.Parallel()
|
||||
png := pngBytes(t)
|
||||
dataDir := t.TempDir()
|
||||
uploads := filepath.Join(dataDir, "uploads")
|
||||
if err := writeFileBytesAll(uploads, uploadTestName, png); err != nil {
|
||||
t.Fatalf("seed upload: %v", err)
|
||||
}
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL, WithUploadsDir(uploads))
|
||||
|
||||
// 127.0.0.1:1 上是没有服务的:一旦退回网络取回就会立刻失败。
|
||||
srcURL := "http://127.0.0.1:1/api/v1/files/" + uploadTestName
|
||||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: srcURL}, ""); err != nil {
|
||||
t.Fatalf("Describe via the local upload path: %v", err)
|
||||
}
|
||||
_, urls := rec.seen()
|
||||
if len(urls) != 1 || urls[0] != DataURI("image/png", png) {
|
||||
t.Fatalf("image_url = %q, want the local file inlined", urls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_MissingUploadFailsLoudly 钉住「认得的上传 URL
|
||||
// 但文件不在」不会偷偷退回网络:错误如实上报(不静默换成别的内容)。
|
||||
func TestOpenAICompatibleAdapter_MissingUploadFailsLoudly(t *testing.T) {
|
||||
t.Parallel()
|
||||
uploads := filepath.Join(t.TempDir(), "uploads")
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL, WithUploadsDir(uploads))
|
||||
|
||||
srcURL := "http://127.0.0.1:1/api/v1/files/" + uploadTestName
|
||||
_, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: srcURL}, "")
|
||||
if err == nil {
|
||||
t.Fatal("Describe succeeded without the upload on disk, want an error")
|
||||
}
|
||||
if !errors.Is(err, ErrBadImage) {
|
||||
t.Fatalf("error = %v, want ErrBadImage", err)
|
||||
}
|
||||
if calls, _ := rec.seen(); calls != 0 {
|
||||
t.Fatalf("upstream was called %d times, want 0", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_TraversalNotReadLocally 是越界文件名的反证
|
||||
// 用例:本地 secret.png 与远端返回的图内容不同,`../` 名称必须落到远端
|
||||
// 取回(而不是被读成本地越界文件)。
|
||||
func TestOpenAICompatibleAdapter_TraversalNotReadLocally(t *testing.T) {
|
||||
t.Parallel()
|
||||
local := pngBytes(t)
|
||||
remote := append([]byte{}, pngBytes(t)...)
|
||||
remote = append(remote, 0x00) // 内容不同即可,仍是合法 PNG 前缀
|
||||
var hits atomic.Int32
|
||||
|
||||
dataDir := t.TempDir()
|
||||
uploads := filepath.Join(dataDir, "uploads")
|
||||
if err := writeFileBytesAll(uploads, uploadTestName, local); err != nil {
|
||||
t.Fatalf("seed uploads dir: %v", err)
|
||||
}
|
||||
if err := writeFileBytes(filepath.Join(dataDir, "secret.png"), local); err != nil {
|
||||
t.Fatalf("seed secret: %v", err)
|
||||
}
|
||||
|
||||
imgSrv := newImageServer(t, remote, "image/png", &hits)
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL, WithUploadsDir(uploads))
|
||||
|
||||
srcURL := imgSrv.URL + "/api/v1/files/../secret.png"
|
||||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: srcURL}, ""); err != nil {
|
||||
t.Fatalf("Describe: %v", err)
|
||||
}
|
||||
_, urls := rec.seen()
|
||||
if len(urls) != 1 {
|
||||
t.Fatalf("image_url = %v, want one part", urls)
|
||||
}
|
||||
if urls[0] == DataURI("image/png", local) {
|
||||
t.Fatal("the ../ name was resolved to a local file outside uploads")
|
||||
}
|
||||
if urls[0] != DataURI("image/png", remote) {
|
||||
t.Fatalf("image_url = %q, want the remote payload", urls[0])
|
||||
}
|
||||
if hits.Load() != 1 {
|
||||
t.Fatalf("source fetched %d times, want 1", hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// TestWithUploadsDirIsOptional 钉住未配置上传目录时行为不变:上传形状的
|
||||
// URL 仍按普通 URL 取回。
|
||||
func TestWithUploadsDirIsOptional(t *testing.T) {
|
||||
t.Parallel()
|
||||
png := pngBytes(t)
|
||||
var hits atomic.Int32
|
||||
imgSrv := newImageServer(t, png, "image/png", &hits)
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := NewOpenAICompatible(config.LLMConfig{BaseURL: upSrv.URL, Model: "vision-test"})
|
||||
|
||||
_, err := ad.Describe(context.Background(), ImageRef{
|
||||
Source: SourceURL,
|
||||
Data: imgSrv.URL + "/api/v1/files/" + uploadTestName,
|
||||
}, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Describe: %v", err)
|
||||
}
|
||||
_, urls := rec.seen()
|
||||
if len(urls) != 1 || urls[0] != DataURI("image/png", png) {
|
||||
t.Fatalf("image_url = %q, want the fetched payload", urls)
|
||||
}
|
||||
if hits.Load() != 1 {
|
||||
t.Fatalf("source fetched %d times, want 1", hits.Load())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||
)
|
||||
|
||||
// TestOpenAICompatibleAdapter_InlinesFetchedURL 是本缺陷的主用例:来源是
|
||||
// http(s) URL 时,服务端必须自己把图取回并内联成 data: URI 再交给
|
||||
// provider —— 原样透传同源(局域网)URL 时云 provider 根本取不到。
|
||||
func TestOpenAICompatibleAdapter_InlinesFetchedURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
png := pngBytes(t)
|
||||
var hits atomic.Int32
|
||||
imgSrv := newImageServer(t, png, "image/png", &hits)
|
||||
srcURL := imgSrv.URL + "/cat.png"
|
||||
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
text, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: srcURL}, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Describe: %v", err)
|
||||
}
|
||||
if text != "a small square" {
|
||||
t.Fatalf("text = %q", text)
|
||||
}
|
||||
calls, urls := rec.seen()
|
||||
if calls != 1 || len(urls) != 1 {
|
||||
t.Fatalf("upstream calls = %d urls = %v, want one image_url", calls, urls)
|
||||
}
|
||||
if want := DataURI("image/png", png); urls[0] != want {
|
||||
t.Fatalf("image_url = %q, want the inlined %q", urls[0], want)
|
||||
}
|
||||
if strings.Contains(urls[0], srcURL) || strings.HasPrefix(urls[0], "http") {
|
||||
t.Fatalf("image_url still points at the source URL behaviour: %q", urls[0])
|
||||
}
|
||||
if hits.Load() != 1 {
|
||||
t.Fatalf("source fetched %d times, want 1", hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_RejectsHTMLResponse 是非图像响应体的用例:
|
||||
// 网关/PROXY 的 text/html 错误页必须报错,而不是被当成 image/png
|
||||
// 喂给视觉模型;错误信息里不得出现页面内容(不打印图像内容)。
|
||||
func TestOpenAICompatibleAdapter_RejectsHTMLResponse(t *testing.T) {
|
||||
t.Parallel()
|
||||
imgSrv := newImageServer(t, htmlBytes(), "text/html; charset=utf-8", nil)
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
_, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/err"}, "")
|
||||
if err == nil {
|
||||
t.Fatal("Describe succeeded on a text/html payload, want an error")
|
||||
}
|
||||
if !errors.Is(err, ErrNotImage) {
|
||||
t.Fatalf("error = %v, want ErrNotImage", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "502 Bad Gateway") {
|
||||
t.Fatalf("error leaks the response body: %v", err)
|
||||
}
|
||||
if calls, _ := rec.seen(); calls != 0 {
|
||||
t.Fatalf("upstream was called %d times, want 0 (fail before the model)", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_RejectsHTMLMislabelledAsPNG 覆盖缺陷第 2 条:
|
||||
// 服务端谎报 image/png 时,字节嗅探仍必须否决它。
|
||||
func TestOpenAICompatibleAdapter_RejectsHTMLMislabelledAsPNG(t *testing.T) {
|
||||
t.Parallel()
|
||||
imgSrv := newImageServer(t, htmlBytes(), "image/png", nil)
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
_, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/liar.png"}, "")
|
||||
if !errors.Is(err, ErrNotImage) {
|
||||
t.Fatalf("error = %v, want ErrNotImage", err)
|
||||
}
|
||||
if calls, _ := rec.seen(); calls != 0 {
|
||||
t.Fatalf("upstream was called %d times, want 0", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_MissingContentTypeSniffs 覆盖「类型缺失」:
|
||||
// 真实 PNG 字节仍应被字节嗅探救回,且内联的 MIME 正确。
|
||||
func TestOpenAICompatibleAdapter_MissingContentTypeSniffs(t *testing.T) {
|
||||
t.Parallel()
|
||||
png := pngBytes(t)
|
||||
imgSrv := newImageServer(t, png, "", nil)
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/n"}, ""); err != nil {
|
||||
t.Fatalf("Describe: %v", err)
|
||||
}
|
||||
_, urls := rec.seen()
|
||||
if len(urls) != 1 || urls[0] != DataURI("image/png", png) {
|
||||
t.Fatalf("image_url = %v, want the sniffed png inlined", urls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_Base64Unchanged 是行为不退化的底线:客户端
|
||||
// 现在用的 data: URI 路径必须逐字节不变地继续工作。
|
||||
func TestOpenAICompatibleAdapter_Base64Unchanged(t *testing.T) {
|
||||
t.Parallel()
|
||||
dataURI := DataURI("image/png", pngBytes(t))
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: dataURI}, ""); err != nil {
|
||||
t.Fatalf("Describe: %v", err)
|
||||
}
|
||||
_, urls := rec.seen()
|
||||
if len(urls) != 1 || urls[0] != dataURI {
|
||||
t.Fatalf("image_url = %q, want the input data: URI byte for byte", urls[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_LocalPathUnchanged 钉住本地路径来源同样被
|
||||
// 内联且字节不变。
|
||||
func TestOpenAICompatibleAdapter_LocalPathUnchanged(t *testing.T) {
|
||||
t.Parallel()
|
||||
png := pngBytes(t)
|
||||
path := t.TempDir() + "/cat.png"
|
||||
if err := writeFileBytes(path, png); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourcePath, Data: path}, ""); err != nil {
|
||||
t.Fatalf("Describe: %v", err)
|
||||
}
|
||||
_, urls := rec.seen()
|
||||
if len(urls) != 1 || urls[0] != DataURI("image/png", png) {
|
||||
t.Fatalf("image_url = %q, want the file inlined", urls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMessageResolver_InlinesURLForMainModel 覆盖主模型直连那条支路:
|
||||
// 支持视觉的主模型拿到的也必须是内联后的 data: URI(Anthropic 协议更是
|
||||
// 只接受 data: URI),而不是原 URL。
|
||||
func TestMessageResolver_InlinesURLForMainModel(t *testing.T) {
|
||||
t.Parallel()
|
||||
png := pngBytes(t)
|
||||
imgSrv := newImageServer(t, png, "image/png", nil)
|
||||
res := NewMessageResolver(NewRouter(WithVisionTool("vision")), func() ModelInfo {
|
||||
return ModelInfo{Name: "vision-model", SupportsVision: true}
|
||||
}, func(_ context.Context, _ string, _ map[string]any) (any, error) {
|
||||
t.Fatal("tool must not be invoked for a sighted main model")
|
||||
return nil, nil
|
||||
})
|
||||
|
||||
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "x", imgSrv.URL+"/cat.png"))
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
}
|
||||
var got string
|
||||
for _, p := range msg.Content {
|
||||
if p.Type == model.ContentTypeImage {
|
||||
got = p.ImageURL
|
||||
}
|
||||
}
|
||||
if want := DataURI("image/png", png); got != want {
|
||||
t.Fatalf("image part = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMessageResolver_KeepsDataURIByteForByte 钉住解析器对 data: URI 的
|
||||
// 短路:客户端送来的 base64 图不被重编码。
|
||||
func TestMessageResolver_KeepsDataURIByteForByte(t *testing.T) {
|
||||
t.Parallel()
|
||||
in := DataURI("image/png", pngBytes(t))
|
||||
res := NewMessageResolver(NewRouter(WithVisionTool("vision")), func() ModelInfo {
|
||||
return ModelInfo{Name: "vision-model", SupportsVision: true}
|
||||
}, nil)
|
||||
|
||||
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "x", in))
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
}
|
||||
for _, p := range msg.Content {
|
||||
if p.Type == model.ContentTypeImage && p.ImageURL != in {
|
||||
t.Fatalf("image part = %q, want the untouched input", p.ImageURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user