Files
AskaEth 983bc49218
CI / Backend (build / vet / test / cross-compile) (push) Has been cancelled
CI / Frontend (tsc + vite build) (push) Has been cancelled
fix(vision): 服务端自己取回图片 URL 并内联为 data URI,修自部署下取不到图(F3,优先级「高」)
- 原缺陷:vision 适配器把 http(s) URL **原样转给上游 provider**,而 upload 返回的是同源局域网地址
  ⇒ 云 provider 根本取不到 ⇒ **B1(大图绕开 WS 4096 字节上限)在自部署形态下等于没做**
- 修法:① `imageURL` → `localizeRef`:**彻底不再直传 URL**;同源 upload 产物走**本地读取**,
  其余 http(s) 由服务端自己取回 → 一律内联为 `data:` URI 再交 provider(OpenAI 与 Anthropic 双协议都能收图)
- 顺手修「类型瞎猜」:新增 `vision/mime.go`,**字节嗅探是唯一裁决**——判得出白名单光栅图就用字节结果;
  字节是文本(HTML/PROXY 错误页、SVG)→ `ErrNotImage`;判不定但来源明确声明 `image/*` 才采信声明。
  错误信息只含类型名,**不含响应体/URL**(不打印图片内容)
- 防护:取回复用既有 http/https 白名单 + 50MiB 上限,新增 **30s 取回超时**;
  本地读取路径 `localUploadPath` 严格认定 `<host>/api/v1/files/<name>`(无 query/fragment、单段路径)+
  32 位小写十六进制名 + `filepath.Dir` 包含性第二道 ⇒ `../`、`%2e%2e%2f`、反斜杠等编码变体在触碰 FS 前即拒
- 生产接线(主会话补):`cmd/cyrene/vision.go` 传 `vision.WithUploadsDir(<DataDir>/uploads)`
- 复核:build/vet 绿;新增 51 条断言(含 httptest 真取回 → 断言上游收到 data URI 且不含源 URL、
  同源 upload 用不可达 host 证明走本地、HTML 谎报 image/png 被字节否决、越界名 19 例);
  **3 条变异反证**(恢复 URL 直传 8 红 / 文本静默回退 png 红 / 去掉穿越检查 12 红)均已还原
- 未做(已登记):历史里 URL 形态图片每轮重新取回(缓解=持久化时内联/缓存);
  Anthropic 只认 png/jpeg/gif/webp 而白名单含 bmp;`fetchURL` 无 IP 过滤(链路本地 169.254.169.254 可达)→ 需收紧
2026-10-04 14:19:28 +08:00

193 lines
6.8 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package vision
import (
"context"
"errors"
"strings"
"sync/atomic"
"testing"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
)
// TestOpenAICompatibleAdapter_InlinesFetchedURL 是本缺陷的主用例:来源是
// http(s) URL 时,服务端必须自己把图取回并内联成 data: URI 再交给
// provider —— 原样透传同源(局域网)URL 时云 provider 根本取不到。
func TestOpenAICompatibleAdapter_InlinesFetchedURL(t *testing.T) {
t.Parallel()
png := pngBytes(t)
var hits atomic.Int32
imgSrv := newImageServer(t, png, "image/png", &hits)
srcURL := imgSrv.URL + "/cat.png"
upSrv, rec := newImageURLRecorderServer(t)
ad := newAdapter(upSrv.URL)
text, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: srcURL}, "")
if err != nil {
t.Fatalf("Describe: %v", err)
}
if text != "a small square" {
t.Fatalf("text = %q", text)
}
calls, urls := rec.seen()
if calls != 1 || len(urls) != 1 {
t.Fatalf("upstream calls = %d urls = %v, want one image_url", calls, urls)
}
if want := DataURI("image/png", png); urls[0] != want {
t.Fatalf("image_url = %q, want the inlined %q", urls[0], want)
}
if strings.Contains(urls[0], srcURL) || strings.HasPrefix(urls[0], "http") {
t.Fatalf("image_url still points at the source URL behaviour: %q", urls[0])
}
if hits.Load() != 1 {
t.Fatalf("source fetched %d times, want 1", hits.Load())
}
}
// TestOpenAICompatibleAdapter_RejectsHTMLResponse 是非图像响应体的用例:
// 网关/PROXY 的 text/html 错误页必须报错,而不是被当成 image/png
// 喂给视觉模型;错误信息里不得出现页面内容(不打印图像内容)。
func TestOpenAICompatibleAdapter_RejectsHTMLResponse(t *testing.T) {
t.Parallel()
imgSrv := newImageServer(t, htmlBytes(), "text/html; charset=utf-8", nil)
upSrv, rec := newImageURLRecorderServer(t)
ad := newAdapter(upSrv.URL)
_, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/err"}, "")
if err == nil {
t.Fatal("Describe succeeded on a text/html payload, want an error")
}
if !errors.Is(err, ErrNotImage) {
t.Fatalf("error = %v, want ErrNotImage", err)
}
if strings.Contains(err.Error(), "502 Bad Gateway") {
t.Fatalf("error leaks the response body: %v", err)
}
if calls, _ := rec.seen(); calls != 0 {
t.Fatalf("upstream was called %d times, want 0 (fail before the model)", calls)
}
}
// TestOpenAICompatibleAdapter_RejectsHTMLMislabelledAsPNG 覆盖缺陷第 2 条:
// 服务端谎报 image/png 时,字节嗅探仍必须否决它。
func TestOpenAICompatibleAdapter_RejectsHTMLMislabelledAsPNG(t *testing.T) {
t.Parallel()
imgSrv := newImageServer(t, htmlBytes(), "image/png", nil)
upSrv, rec := newImageURLRecorderServer(t)
ad := newAdapter(upSrv.URL)
_, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/liar.png"}, "")
if !errors.Is(err, ErrNotImage) {
t.Fatalf("error = %v, want ErrNotImage", err)
}
if calls, _ := rec.seen(); calls != 0 {
t.Fatalf("upstream was called %d times, want 0", calls)
}
}
// TestOpenAICompatibleAdapter_MissingContentTypeSniffs 覆盖「类型缺失」:
// 真实 PNG 字节仍应被字节嗅探救回,且内联的 MIME 正确。
func TestOpenAICompatibleAdapter_MissingContentTypeSniffs(t *testing.T) {
t.Parallel()
png := pngBytes(t)
imgSrv := newImageServer(t, png, "", nil)
upSrv, rec := newImageURLRecorderServer(t)
ad := newAdapter(upSrv.URL)
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceURL, Data: imgSrv.URL + "/n"}, ""); err != nil {
t.Fatalf("Describe: %v", err)
}
_, urls := rec.seen()
if len(urls) != 1 || urls[0] != DataURI("image/png", png) {
t.Fatalf("image_url = %v, want the sniffed png inlined", urls)
}
}
// TestOpenAICompatibleAdapter_Base64Unchanged 是行为不退化的底线:客户端
// 现在用的 data: URI 路径必须逐字节不变地继续工作。
func TestOpenAICompatibleAdapter_Base64Unchanged(t *testing.T) {
t.Parallel()
dataURI := DataURI("image/png", pngBytes(t))
upSrv, rec := newImageURLRecorderServer(t)
ad := newAdapter(upSrv.URL)
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourceBase64, Data: dataURI}, ""); err != nil {
t.Fatalf("Describe: %v", err)
}
_, urls := rec.seen()
if len(urls) != 1 || urls[0] != dataURI {
t.Fatalf("image_url = %q, want the input data: URI byte for byte", urls[0])
}
}
// TestOpenAICompatibleAdapter_LocalPathUnchanged 钉住本地路径来源同样被
// 内联且字节不变。
func TestOpenAICompatibleAdapter_LocalPathUnchanged(t *testing.T) {
t.Parallel()
png := pngBytes(t)
path := t.TempDir() + "/cat.png"
if err := writeFileBytes(path, png); err != nil {
t.Fatalf("write: %v", err)
}
upSrv, rec := newImageURLRecorderServer(t)
ad := newAdapter(upSrv.URL)
if _, err := ad.Describe(context.Background(), ImageRef{Source: SourcePath, Data: path}, ""); err != nil {
t.Fatalf("Describe: %v", err)
}
_, urls := rec.seen()
if len(urls) != 1 || urls[0] != DataURI("image/png", png) {
t.Fatalf("image_url = %q, want the file inlined", urls)
}
}
// TestMessageResolver_InlinesURLForMainModel 覆盖主模型直连那条支路:
// 支持视觉的主模型拿到的也必须是内联后的 data: URI(Anthropic 协议更是
// 只接受 data: URI),而不是原 URL。
func TestMessageResolver_InlinesURLForMainModel(t *testing.T) {
t.Parallel()
png := pngBytes(t)
imgSrv := newImageServer(t, png, "image/png", nil)
res := NewMessageResolver(NewRouter(WithVisionTool("vision")), func() ModelInfo {
return ModelInfo{Name: "vision-model", SupportsVision: true}
}, func(_ context.Context, _ string, _ map[string]any) (any, error) {
t.Fatal("tool must not be invoked for a sighted main model")
return nil, nil
})
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "x", imgSrv.URL+"/cat.png"))
if err != nil {
t.Fatalf("Resolve: %v", err)
}
var got string
for _, p := range msg.Content {
if p.Type == model.ContentTypeImage {
got = p.ImageURL
}
}
if want := DataURI("image/png", png); got != want {
t.Fatalf("image part = %q, want %q", got, want)
}
}
// TestMessageResolver_KeepsDataURIByteForByte 钉住解析器对 data: URI 的
// 短路:客户端送来的 base64 图不被重编码。
func TestMessageResolver_KeepsDataURIByteForByte(t *testing.T) {
t.Parallel()
in := DataURI("image/png", pngBytes(t))
res := NewMessageResolver(NewRouter(WithVisionTool("vision")), func() ModelInfo {
return ModelInfo{Name: "vision-model", SupportsVision: true}
}, nil)
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "x", in))
if err != nil {
t.Fatalf("Resolve: %v", err)
}
for _, p := range msg.Content {
if p.Type == model.ContentTypeImage && p.ImageURL != in {
t.Fatalf("image part = %q, want the untouched input", p.ImageURL)
}
}
}