- 原缺陷:vision 适配器把 http(s) URL **原样转给上游 provider**,而 upload 返回的是同源局域网地址 ⇒ 云 provider 根本取不到 ⇒ **B1(大图绕开 WS 4096 字节上限)在自部署形态下等于没做** - 修法:① `imageURL` → `localizeRef`:**彻底不再直传 URL**;同源 upload 产物走**本地读取**, 其余 http(s) 由服务端自己取回 → 一律内联为 `data:` URI 再交 provider(OpenAI 与 Anthropic 双协议都能收图) - 顺手修「类型瞎猜」:新增 `vision/mime.go`,**字节嗅探是唯一裁决**——判得出白名单光栅图就用字节结果; 字节是文本(HTML/PROXY 错误页、SVG)→ `ErrNotImage`;判不定但来源明确声明 `image/*` 才采信声明。 错误信息只含类型名,**不含响应体/URL**(不打印图片内容) - 防护:取回复用既有 http/https 白名单 + 50MiB 上限,新增 **30s 取回超时**; 本地读取路径 `localUploadPath` 严格认定 `<host>/api/v1/files/<name>`(无 query/fragment、单段路径)+ 32 位小写十六进制名 + `filepath.Dir` 包含性第二道 ⇒ `../`、`%2e%2e%2f`、反斜杠等编码变体在触碰 FS 前即拒 - 生产接线(主会话补):`cmd/cyrene/vision.go` 传 `vision.WithUploadsDir(<DataDir>/uploads)` - 复核:build/vet 绿;新增 51 条断言(含 httptest 真取回 → 断言上游收到 data URI 且不含源 URL、 同源 upload 用不可达 host 证明走本地、HTML 谎报 image/png 被字节否决、越界名 19 例); **3 条变异反证**(恢复 URL 直传 8 红 / 文本静默回退 png 红 / 去掉穿越检查 12 红)均已还原 - 未做(已登记):历史里 URL 形态图片每轮重新取回(缓解=持久化时内联/缓存); Anthropic 只认 png/jpeg/gif/webp 而白名单含 bmp;`fetchURL` 无 IP 过滤(链路本地 169.254.169.254 可达)→ 需收紧
193 lines
6.8 KiB
Go
193 lines
6.8 KiB
Go
package vision
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"strings"
|
||
"sync/atomic"
|
||
"testing"
|
||
|
||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||
)
|
||
|
||
// TestOpenAICompatibleAdapter_InlinesFetchedURL 是本缺陷的主用例:来源是
|
||
// http(s) URL 时,服务端必须自己把图取回并内联成 data: URI 再交给
|
||
// provider —— 原样透传同源(局域网)URL 时云 provider 根本取不到。
|
||
func TestOpenAICompatibleAdapter_InlinesFetchedURL(t *testing.T) {
|
||
t.Parallel()
|
||
png := pngBytes(t)
|
||
var hits atomic.Int32
|
||
imgSrv := newImageServer(t, png, "image/png", &hits)
|
||
srcURL := imgSrv.URL + "/cat.png"
|
||
|
||
upSrv, rec := newImageURLRecorderServer(t)
|
||
ad := newAdapter(upSrv.URL)
|
||
|
||
text, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: srcURL}, "")
|
||
if err != nil {
|
||
t.Fatalf("Describe: %v", err)
|
||
}
|
||
if text != "a small square" {
|
||
t.Fatalf("text = %q", text)
|
||
}
|
||
calls, urls := rec.seen()
|
||
if calls != 1 || len(urls) != 1 {
|
||
t.Fatalf("upstream calls = %d urls = %v, want one image_url", calls, urls)
|
||
}
|
||
if want := DataURI("image/png", png); urls[0] != want {
|
||
t.Fatalf("image_url = %q, want the inlined %q", urls[0], want)
|
||
}
|
||
if strings.Contains(urls[0], srcURL) || strings.HasPrefix(urls[0], "http") {
|
||
t.Fatalf("image_url still points at the source URL behaviour: %q", urls[0])
|
||
}
|
||
if hits.Load() != 1 {
|
||
t.Fatalf("source fetched %d times, want 1", hits.Load())
|
||
}
|
||
}
|
||
|
||
// TestOpenAICompatibleAdapter_RejectsHTMLResponse 是非图像响应体的用例:
|
||
// 网关/PROXY 的 text/html 错误页必须报错,而不是被当成 image/png
|
||
// 喂给视觉模型;错误信息里不得出现页面内容(不打印图像内容)。
|
||
func TestOpenAICompatibleAdapter_RejectsHTMLResponse(t *testing.T) {
|
||
t.Parallel()
|
||
imgSrv := newImageServer(t, htmlBytes(), "text/html; charset=utf-8", nil)
|
||
upSrv, rec := newImageURLRecorderServer(t)
|
||
ad := newAdapter(upSrv.URL)
|
||
|
||
_, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/err"}, "")
|
||
if err == nil {
|
||
t.Fatal("Describe succeeded on a text/html payload, want an error")
|
||
}
|
||
if !errors.Is(err, ErrNotImage) {
|
||
t.Fatalf("error = %v, want ErrNotImage", err)
|
||
}
|
||
if strings.Contains(err.Error(), "502 Bad Gateway") {
|
||
t.Fatalf("error leaks the response body: %v", err)
|
||
}
|
||
if calls, _ := rec.seen(); calls != 0 {
|
||
t.Fatalf("upstream was called %d times, want 0 (fail before the model)", calls)
|
||
}
|
||
}
|
||
|
||
// TestOpenAICompatibleAdapter_RejectsHTMLMislabelledAsPNG 覆盖缺陷第 2 条:
|
||
// 服务端谎报 image/png 时,字节嗅探仍必须否决它。
|
||
func TestOpenAICompatibleAdapter_RejectsHTMLMislabelledAsPNG(t *testing.T) {
|
||
t.Parallel()
|
||
imgSrv := newImageServer(t, htmlBytes(), "image/png", nil)
|
||
upSrv, rec := newImageURLRecorderServer(t)
|
||
ad := newAdapter(upSrv.URL)
|
||
|
||
_, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/liar.png"}, "")
|
||
if !errors.Is(err, ErrNotImage) {
|
||
t.Fatalf("error = %v, want ErrNotImage", err)
|
||
}
|
||
if calls, _ := rec.seen(); calls != 0 {
|
||
t.Fatalf("upstream was called %d times, want 0", calls)
|
||
}
|
||
}
|
||
|
||
// TestOpenAICompatibleAdapter_MissingContentTypeSniffs 覆盖「类型缺失」:
|
||
// 真实 PNG 字节仍应被字节嗅探救回,且内联的 MIME 正确。
|
||
func TestOpenAICompatibleAdapter_MissingContentTypeSniffs(t *testing.T) {
|
||
t.Parallel()
|
||
png := pngBytes(t)
|
||
imgSrv := newImageServer(t, png, "", nil)
|
||
upSrv, rec := newImageURLRecorderServer(t)
|
||
ad := newAdapter(upSrv.URL)
|
||
|
||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/n"}, ""); err != nil {
|
||
t.Fatalf("Describe: %v", err)
|
||
}
|
||
_, urls := rec.seen()
|
||
if len(urls) != 1 || urls[0] != DataURI("image/png", png) {
|
||
t.Fatalf("image_url = %v, want the sniffed png inlined", urls)
|
||
}
|
||
}
|
||
|
||
// TestOpenAICompatibleAdapter_Base64Unchanged 是行为不退化的底线:客户端
|
||
// 现在用的 data: URI 路径必须逐字节不变地继续工作。
|
||
func TestOpenAICompatibleAdapter_Base64Unchanged(t *testing.T) {
|
||
t.Parallel()
|
||
dataURI := DataURI("image/png", pngBytes(t))
|
||
upSrv, rec := newImageURLRecorderServer(t)
|
||
ad := newAdapter(upSrv.URL)
|
||
|
||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: dataURI}, ""); err != nil {
|
||
t.Fatalf("Describe: %v", err)
|
||
}
|
||
_, urls := rec.seen()
|
||
if len(urls) != 1 || urls[0] != dataURI {
|
||
t.Fatalf("image_url = %q, want the input data: URI byte for byte", urls[0])
|
||
}
|
||
}
|
||
|
||
// TestOpenAICompatibleAdapter_LocalPathUnchanged 钉住本地路径来源同样被
|
||
// 内联且字节不变。
|
||
func TestOpenAICompatibleAdapter_LocalPathUnchanged(t *testing.T) {
|
||
t.Parallel()
|
||
png := pngBytes(t)
|
||
path := t.TempDir() + "/cat.png"
|
||
if err := writeFileBytes(path, png); err != nil {
|
||
t.Fatalf("write: %v", err)
|
||
}
|
||
upSrv, rec := newImageURLRecorderServer(t)
|
||
ad := newAdapter(upSrv.URL)
|
||
|
||
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourcePath, Data: path}, ""); err != nil {
|
||
t.Fatalf("Describe: %v", err)
|
||
}
|
||
_, urls := rec.seen()
|
||
if len(urls) != 1 || urls[0] != DataURI("image/png", png) {
|
||
t.Fatalf("image_url = %q, want the file inlined", urls)
|
||
}
|
||
}
|
||
|
||
// TestMessageResolver_InlinesURLForMainModel 覆盖主模型直连那条支路:
|
||
// 支持视觉的主模型拿到的也必须是内联后的 data: URI(Anthropic 协议更是
|
||
// 只接受 data: URI),而不是原 URL。
|
||
func TestMessageResolver_InlinesURLForMainModel(t *testing.T) {
|
||
t.Parallel()
|
||
png := pngBytes(t)
|
||
imgSrv := newImageServer(t, png, "image/png", nil)
|
||
res := NewMessageResolver(NewRouter(WithVisionTool("vision")), func() ModelInfo {
|
||
return ModelInfo{Name: "vision-model", SupportsVision: true}
|
||
}, func(_ context.Context, _ string, _ map[string]any) (any, error) {
|
||
t.Fatal("tool must not be invoked for a sighted main model")
|
||
return nil, nil
|
||
})
|
||
|
||
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "x", imgSrv.URL+"/cat.png"))
|
||
if err != nil {
|
||
t.Fatalf("Resolve: %v", err)
|
||
}
|
||
var got string
|
||
for _, p := range msg.Content {
|
||
if p.Type == model.ContentTypeImage {
|
||
got = p.ImageURL
|
||
}
|
||
}
|
||
if want := DataURI("image/png", png); got != want {
|
||
t.Fatalf("image part = %q, want %q", got, want)
|
||
}
|
||
}
|
||
|
||
// TestMessageResolver_KeepsDataURIByteForByte 钉住解析器对 data: URI 的
|
||
// 短路:客户端送来的 base64 图不被重编码。
|
||
func TestMessageResolver_KeepsDataURIByteForByte(t *testing.T) {
|
||
t.Parallel()
|
||
in := DataURI("image/png", pngBytes(t))
|
||
res := NewMessageResolver(NewRouter(WithVisionTool("vision")), func() ModelInfo {
|
||
return ModelInfo{Name: "vision-model", SupportsVision: true}
|
||
}, nil)
|
||
|
||
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "x", in))
|
||
if err != nil {
|
||
t.Fatalf("Resolve: %v", err)
|
||
}
|
||
for _, p := range msg.Content {
|
||
if p.Type == model.ContentTypeImage && p.ImageURL != in {
|
||
t.Fatalf("image part = %q, want the untouched input", p.ImageURL)
|
||
}
|
||
}
|
||
}
|