fix(vision): 服务端自己取回图片 URL 并内联为 data URI，修自部署下取不到图（F3，优先级「高」）
- 原缺陷：vision 适配器把 http(s) URL **原样转给上游 provider**，而 upload 返回的是同源局域网地址 ⇒ 云 provider 根本取不到 ⇒ **B1（大图绕开 WS 4096 字节上限）在自部署形态下等于没做** - 修法：① `imageURL` → `localizeRef`：**彻底不再直传 URL**；同源 upload 产物走**本地读取**， 其余 http(s) 由服务端自己取回 → 一律内联为 `data:` URI 再交 provider（OpenAI 与 Anthropic 双协议都能收图） - 顺手修「类型瞎猜」：新增 `vision/mime.go`，**字节嗅探是唯一裁决**——判得出白名单光栅图就用字节结果； 字节是文本（HTML/PROXY 错误页、SVG）→ `ErrNotImage`；判不定但来源明确声明 `image/*` 才采信声明。 错误信息只含类型名，**不含响应体/URL**（不打印图片内容） - 防护：取回复用既有 http/https 白名单 + 50MiB 上限，新增 **30s 取回超时**； 本地读取路径 `localUploadPath` 严格认定 `<host>/api/v1/files/<name>`（无 query/fragment、单段路径）+ 32 位小写十六进制名 + `filepath.Dir` 包含性第二道 ⇒ `../`、`%2e%2e%2f`、反斜杠等编码变体在触碰 FS 前即拒 - 生产接线（主会话补）：`cmd/cyrene/vision.go` 传 `vision.WithUploadsDir(<DataDir>/uploads)` - 复核：build/vet 绿；新增 51 条断言（含 httptest 真取回 → 断言上游收到 data URI 且不含源 URL、 同源 upload 用不可达 host 证明走本地、HTML 谎报 image/png 被字节否决、越界名 19 例）； **3 条变异反证**（恢复 URL 直传 8 红 / 文本静默回退 png 红 / 去掉穿越检查 12 红）均已还原 - 未做（已登记）：历史里 URL 形态图片每轮重新取回（缓解＝持久化时内联/缓存）； Anthropic 只认 png/jpeg/gif/webp 而白名单含 bmp；`fetchURL` 无 IP 过滤（链路本地 169.254.169.254 可达）→ 需收紧
This commit is contained in:
AskaEth committed
1 parent
88d7dd0026
commit
983bc49218
11 files changed
+971
-55
No files matched your search
@@ -0,0 +1,192 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||
)
|
||||
|
||||
// TestOpenAICompatibleAdapter_InlinesFetchedURL 是本缺陷的主用例：来源是
|
||||
// http(s) URL 时，服务端必须自己把图取回并内联成 data: URI 再交给
|
||||
// provider —— 原样透传同源（局域网）URL 时云 provider 根本取不到。
|
||||
func TestOpenAICompatibleAdapter_InlinesFetchedURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
png := pngBytes(t)
|
||||
var hits atomic.Int32
|
||||
imgSrv := newImageServer(t, png, "image/png", &hits)
|
||||
srcURL := imgSrv.URL + "/cat.png"
|
||||
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
text, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: srcURL}, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Describe: %v", err)
|
||||
}
|
||||
if text != "a small square" {
|
||||
t.Fatalf("text = %q", text)
|
||||
}
|
||||
calls, urls := rec.seen()
|
||||
if calls != 1 || len(urls) != 1 {
|
||||
t.Fatalf("upstream calls = %d urls = %v, want one image_url", calls, urls)
|
||||
}
|
||||
if want := DataURI("image/png", png); urls[0] != want {
|
||||
t.Fatalf("image_url = %q, want the inlined %q", urls[0], want)
|
||||
}
|
||||
if strings.Contains(urls[0], srcURL) || strings.HasPrefix(urls[0], "http") {
|
||||
t.Fatalf("image_url still points at the source URL behaviour: %q", urls[0])
|
||||
}
|
||||
if hits.Load() != 1 {
|
||||
t.Fatalf("source fetched %d times, want 1", hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_RejectsHTMLResponse 是非图像响应体的用例：
|
||||
// 网关/PROXY 的 text/html 错误页必须报错，而不是被当成 image/png
|
||||
// 喂给视觉模型；错误信息里不得出现页面内容（不打印图像内容）。
|
||||
func TestOpenAICompatibleAdapter_RejectsHTMLResponse(t *testing.T) {
|
||||
t.Parallel()
|
||||
imgSrv := newImageServer(t, htmlBytes(), "text/html; charset=utf-8", nil)
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
_, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/err"}, "")
|
||||
if err == nil {
|
||||
t.Fatal("Describe succeeded on a text/html payload, want an error")
|
||||
}
|
||||
if !errors.Is(err, ErrNotImage) {
|
||||
t.Fatalf("error = %v, want ErrNotImage", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "502 Bad Gateway") {
|
||||
t.Fatalf("error leaks the response body: %v", err)
|
||||
}
|
||||
if calls, _ := rec.seen(); calls != 0 {
|
||||
t.Fatalf("upstream was called %d times, want 0 (fail before the model)", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_RejectsHTMLMislabelledAsPNG 覆盖缺陷第 2 条：
|
||||
// 服务端谎报 image/png 时，字节嗅探仍必须否决它。
|
||||
func TestOpenAICompatibleAdapter_RejectsHTMLMislabelledAsPNG(t *testing.T) {
|
||||
t.Parallel()
|
||||
imgSrv := newImageServer(t, htmlBytes(), "image/png", nil)
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
_, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/liar.png"}, "")
|
||||
if !errors.Is(err, ErrNotImage) {
|
||||
t.Fatalf("error = %v, want ErrNotImage", err)
|
||||
}
|
||||
if calls, _ := rec.seen(); calls != 0 {
|
||||
t.Fatalf("upstream was called %d times, want 0", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_MissingContentTypeSniffs 覆盖「类型缺失」：
|
||||
// 真实 PNG 字节仍应被字节嗅探救回，且内联的 MIME 正确。
|
||||
func TestOpenAICompatibleAdapter_MissingContentTypeSniffs(t *testing.T) {
|
||||
t.Parallel()
|
||||
png := pngBytes(t)
|
||||
imgSrv := newImageServer(t, png, "", nil)
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/n"}, ""); err != nil {
|
||||
t.Fatalf("Describe: %v", err)
|
||||
}
|
||||
_, urls := rec.seen()
|
||||
if len(urls) != 1 || urls[0] != DataURI("image/png", png) {
|
||||
t.Fatalf("image_url = %v, want the sniffed png inlined", urls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_Base64Unchanged 是行为不退化的底线：客户端
|
||||
// 现在用的 data: URI 路径必须逐字节不变地继续工作。
|
||||
func TestOpenAICompatibleAdapter_Base64Unchanged(t *testing.T) {
|
||||
t.Parallel()
|
||||
dataURI := DataURI("image/png", pngBytes(t))
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: dataURI}, ""); err != nil {
|
||||
t.Fatalf("Describe: %v", err)
|
||||
}
|
||||
_, urls := rec.seen()
|
||||
if len(urls) != 1 || urls[0] != dataURI {
|
||||
t.Fatalf("image_url = %q, want the input data: URI byte for byte", urls[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAICompatibleAdapter_LocalPathUnchanged 钉住本地路径来源同样被
|
||||
// 内联且字节不变。
|
||||
func TestOpenAICompatibleAdapter_LocalPathUnchanged(t *testing.T) {
|
||||
t.Parallel()
|
||||
png := pngBytes(t)
|
||||
path := t.TempDir() + "/cat.png"
|
||||
if err := writeFileBytes(path, png); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
upSrv, rec := newImageURLRecorderServer(t)
|
||||
ad := newAdapter(upSrv.URL)
|
||||
|
||||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourcePath, Data: path}, ""); err != nil {
|
||||
t.Fatalf("Describe: %v", err)
|
||||
}
|
||||
_, urls := rec.seen()
|
||||
if len(urls) != 1 || urls[0] != DataURI("image/png", png) {
|
||||
t.Fatalf("image_url = %q, want the file inlined", urls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMessageResolver_InlinesURLForMainModel 覆盖主模型直连那条支路：
|
||||
// 支持视觉的主模型拿到的也必须是内联后的 data: URI（Anthropic 协议更是
|
||||
// 只接受 data: URI），而不是原 URL。
|
||||
func TestMessageResolver_InlinesURLForMainModel(t *testing.T) {
|
||||
t.Parallel()
|
||||
png := pngBytes(t)
|
||||
imgSrv := newImageServer(t, png, "image/png", nil)
|
||||
res := NewMessageResolver(NewRouter(WithVisionTool("vision")), func() ModelInfo {
|
||||
return ModelInfo{Name: "vision-model", SupportsVision: true}
|
||||
}, func(_ context.Context, _ string, _ map[string]any) (any, error) {
|
||||
t.Fatal("tool must not be invoked for a sighted main model")
|
||||
return nil, nil
|
||||
})
|
||||
|
||||
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "x", imgSrv.URL+"/cat.png"))
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
}
|
||||
var got string
|
||||
for _, p := range msg.Content {
|
||||
if p.Type == model.ContentTypeImage {
|
||||
got = p.ImageURL
|
||||
}
|
||||
}
|
||||
if want := DataURI("image/png", png); got != want {
|
||||
t.Fatalf("image part = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMessageResolver_KeepsDataURIByteForByte 钉住解析器对 data: URI 的
|
||||
// 短路：客户端送来的 base64 图不被重编码。
|
||||
func TestMessageResolver_KeepsDataURIByteForByte(t *testing.T) {
|
||||
t.Parallel()
|
||||
in := DataURI("image/png", pngBytes(t))
|
||||
res := NewMessageResolver(NewRouter(WithVisionTool("vision")), func() ModelInfo {
|
||||
return ModelInfo{Name: "vision-model", SupportsVision: true}
|
||||
}, nil)
|
||||
|
||||
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "x", in))
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
}
|
||||
for _, p := range msg.Content {
|
||||
if p.Type == model.ContentTypeImage && p.ImageURL != in {
|
||||
t.Fatalf("image part = %q, want the untouched input", p.ImageURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user