package documents import ( "context" "encoding/json" "errors" "os" "path/filepath" "strings" "testing" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/agent" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/history" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/loop" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model" ) // 这些测试钉住摄取层的落地事实(design/24 §4.3/§4.4):**只抽一次**(第二次读 // sidecar,原文件被删也照样出正文)、失败可见(unreadable + 诚实原因,且失败也 // 只抽一次)、名字形态非法绝不碰文件系统、抽取器版本变化让缓存失效。 // testLayout 是一份隔离的临时目录布局:数据目录 + S1 的上传目录。 type testLayout struct { dataDir string uploads string } func newTestLayout(t *testing.T) testLayout { t.Helper() data := t.TempDir() uploads := filepath.Join(data, "uploads", "documents") if err := os.MkdirAll(uploads, 0o700); err != nil { t.Fatalf("mkdir uploads: %v", err) } return testLayout{dataDir: data, uploads: uploads} } // writeUpload 把一份文档放进上传目录,返回它的存储名。 func (l testLayout) writeUpload(t *testing.T, name string, data []byte) string { t.Helper() if err := os.WriteFile(filepath.Join(l.uploads, name), data, 0o600); err != nil { t.Fatalf("write upload: %v", err) } return name } // sidecarPath 是某份文档的 sidecar 路径。 func (l testLayout) sidecarPath(name string) string { return filepath.Join(l.dataDir, sidecarDirName, name+sidecarSuffix) } // resolveOne 用给定解析器解析一份文档消息,返回结论与正文(测试专用的窄入口)。 func resolveOne(t *testing.T, r *Resolver, name string) (history.DocumentRef, string) { t.Helper() msg := model.DocumentMessage(model.RoleUser, "", model.Document{Name: name, MIME: "text/plain", Size: 0}) refs, bodies, err := r.Resolve(context.Background(), msg) if err != nil { t.Fatalf("Resolve: %v", err) } return refs[name], bodies[name] } // TestResolver_ExtractsOnceAndThenReadsSidecar 钉住「只抽一次」(design/24 §4.3): // 首次解析写 sidecar,之后**原文件被删**也能从 sidecar 出正文,且 sidecar 不被 // 重写(mtime 不变 —— 「后续回合不再碰原文件」在文件系统上可观测)。 func TestResolver_ExtractsOnceAndThenReadsSidecar(t *testing.T) { t.Parallel() layout := newTestLayout(t) body := "报告正文:三个要点。\n" name := layout.writeUpload(t, txtName, []byte(body)) r := NewResolver(Config{DataDir: layout.dataDir, UploadsDir: layout.uploads}) ref, got := resolveOne(t, r, name) if ref.Status != history.DocumentStatusExtracted || got != body { t.Fatalf("首次解析 = (%+v, %q), want extracted + 原文", ref, got) } first, err := os.Stat(layout.sidecarPath(name)) if err != nil { t.Fatalf("sidecar 未写出: %v", err) } if err := os.Remove(filepath.Join(layout.uploads, name)); err != nil { t.Fatalf("删除原文件: %v", err) } ref2, got2 := resolveOne(t, r, name) if ref2.Status != history.DocumentStatusExtracted || got2 != body { t.Fatalf("第二次解析 = (%+v, %q), want 从 sidecar 得到同一正文", ref2, got2) } second, err := os.Stat(layout.sidecarPath(name)) if err != nil { t.Fatalf("sidecar 消失: %v", err) } if !second.ModTime().Equal(first.ModTime()) { t.Fatalf("sidecar 被重写:mtime %v → %v(「只写一次」被破坏)", first.ModTime(), second.ModTime()) } } // TestResolver_KeepsFullBodyAndRecordsSafetyTruncationInSidecar 钉住 sidecar 的 // 存全文语义(切片 T1 §7.2 第 1 条)与安全上限的如实记账: // // - 正文远超旧的 64K 注入预算、但未撞上安全上限 ⇒ 整份存下,truncated=false, // 因此 document-read 按清单游标续读能真的读到预算之外的正文; // - 撞上安全上限(MaxSidecarBytes)⇒ truncated=true 且字符数随 sidecar 往返。 func TestResolver_KeepsFullBodyAndRecordsSafetyTruncationInSidecar(t *testing.T) { t.Parallel() layout := newTestLayout(t) name := layout.writeUpload(t, txtName, []byte(strings.Repeat("宽", 100*1024))) r := NewResolver(Config{DataDir: layout.dataDir, UploadsDir: layout.uploads}) ref, body := resolveOne(t, r, name) if ref.Truncated || ref.Chars != 100*1024 { t.Fatalf("首次解析 = %+v, want 全文(%d 字符、未截断)", ref, 100*1024) } if err := os.Remove(filepath.Join(layout.uploads, name)); err != nil { t.Fatalf("删除原文件: %v", err) } ref2, body2 := resolveOne(t, r, name) if ref2.Truncated || ref2.Chars != 100*1024 || body2 != body { t.Fatalf("sidecar 命中 = %+v, want 从 sidecar 读回同一份全文", ref2) } capped := layout.writeUpload(t, "0123456789abcdef0123456789abcdee.txt", []byte(strings.Repeat("宽", MaxSidecarBytes))) ref3, body3 := resolveOne(t, r, capped) if !ref3.Truncated { t.Fatalf("撞上安全上限的文档 = %+v, want truncated=true", ref3) } if ref3.Chars != len([]rune(body3)) || len(body3) > MaxSidecarBytes { t.Fatalf("上限命中:Chars=%d / 正文字节=%d,want 一致且不超过 %d", ref3.Chars, len(body3), MaxSidecarBytes) } } // TestResolver_UnreadableIsHonestAndCached 钉住失败可见(design/24 §4.4):图片型/ // 无文本层 PDF 得到 unreadable + **诚实原因**(必须点明「需要 OCR」),清单/回传 // 状态有来源;失败同样只抽一次。 func TestResolver_UnreadableIsHonestAndCached(t *testing.T) { t.Parallel() layout := newTestLayout(t) name := layout.writeUpload(t, pdfName, buildPDF(t, pdfFixture{Contents: []string{imageContent}})) r := NewResolver(Config{DataDir: layout.dataDir, UploadsDir: layout.uploads}) ref, body := resolveOne(t, r, name) if ref.Status != history.DocumentStatusUnreadable { t.Fatalf("Status = %q, want %q", ref.Status, history.DocumentStatusUnreadable) } if body != reasonPDFNoTextLayer { t.Fatalf("body = %q, want %q", body, reasonPDFNoTextLayer) } if !strings.Contains(body, "OCR") { t.Fatalf("原因文案 %q 没点明需要 OCR(用户要知道这份 PDF 得先转文字)", body) } if _, err := os.Stat(layout.sidecarPath(name)); err != nil { t.Fatalf("失败的抽取也应留下 sidecar(只抽一次同样适用于失败): %v", err) } if err := os.Remove(filepath.Join(layout.uploads, name)); err != nil { t.Fatalf("删除原文件: %v", err) } if _, body2 := resolveOne(t, r, name); body2 != reasonPDFNoTextLayer { t.Fatalf("第二次 body = %q, want 从 sidecar 读回同一原因", body2) } } // TestResolver_PDFPagesAndKindReachTheSidecar 钉住切片 B1 的 sidecar 元数据: // PDF 的抽取结论带 `kind=pdf` 与**页数** —— 两者随 sidecar 落盘、可被读侧取回, // 因此「这份 PDF 一共产出多少页」在缓存里是可查的事实(不是只留在日志里)。 func TestResolver_PDFPagesAndKindReachTheSidecar(t *testing.T) { t.Parallel() layout := newTestLayout(t) name := layout.writeUpload(t, pdfName, buildPDF(t, pdfFixture{Contents: []string{ textContent("first page"), textContent("second page"), textContent("third page"), }})) r := NewResolver(Config{DataDir: layout.dataDir, UploadsDir: layout.uploads}) ref, body := resolveOne(t, r, name) if ref.Status != history.DocumentStatusExtracted || !strings.Contains(body, "third page") { t.Fatalf("解析 = (%+v, %q), want extracted 且含第三页正文", ref, body) } side, err := ReadSidecar(layout.dataDir, name) if err != nil { t.Fatalf("ReadSidecar: %v", err) } if side.Kind != kindPDF || side.Pages != 3 { t.Fatalf("sidecar = (kind=%q, pages=%d), want (%q,3)", side.Kind, side.Pages, kindPDF) } raw, err := os.ReadFile(layout.sidecarPath(name)) if err != nil { t.Fatalf("读 sidecar: %v", err) } header, _, ok := strings.Cut(string(raw), sidecarHeaderEnd) if !ok { t.Fatalf("sidecar 缺少元数据行: %q", raw) } var meta sidecarMeta if err := json.Unmarshal([]byte(header), &meta); err != nil { t.Fatalf("元数据行解码失败: %v (%q)", err, header) } if meta.Kind != kindPDF || meta.Pages != 3 || meta.Version != ExtractorVersion { t.Fatalf("落盘元数据 = (kind=%q, pages=%d, v=%q), want (%q,3,%q)", meta.Kind, meta.Pages, meta.Version, kindPDF, ExtractorVersion) } } // TestResolver_MissingFileIsUnreadable 钉住「文件不存在」不是 error、也不是 // 静默:它是一条 unreadable 结论 + 诚实原因(回合照常继续)。 func TestResolver_MissingFileIsUnreadable(t *testing.T) { t.Parallel() layout := newTestLayout(t) r := NewResolver(Config{DataDir: layout.dataDir, UploadsDir: layout.uploads}) ref, body := resolveOne(t, r, txtName) if ref.Status != history.DocumentStatusUnreadable || body != reasonMissing { t.Fatalf("解析不存在的文档 = (%+v, %q), want unreadable + %q", ref, body, reasonMissing) } if ref.Name != txtName { t.Fatalf("Name = %q, want 部件里的存储名(身份来自部件)", ref.Name) } } // TestResolver_RejectsIllegalStorageName 钉住防御性的路径安全:形态非法的名字 // 绝不参与读盘或 sidecar 路径,只得到 unreadable + 诚实原因。 func TestResolver_RejectsIllegalStorageName(t *testing.T) { t.Parallel() layout := newTestLayout(t) r := NewResolver(Config{DataDir: layout.dataDir, UploadsDir: layout.uploads}) for _, name := range []string{"../secret.txt", "..", "a.txt", "0123456789abcdef0123456789abcde.txt", "0123456789abcdef0123456789abcdef./x.txt"} { ref, body := resolveOne(t, r, name) if ref.Status != history.DocumentStatusUnreadable || body != reasonNameInvalid { t.Fatalf("解析 %q = (%+v, %q), want unreadable + %q", name, ref, body, reasonNameInvalid) } } entries, err := os.ReadDir(filepath.Join(layout.dataDir, sidecarDirName)) if err != nil && !errors.Is(err, os.ErrNotExist) { t.Fatalf("read extracted dir: %v", err) } if len(entries) != 0 { t.Fatalf("非法名字竟然写出了 %d 个 sidecar", len(entries)) } } // TestSidecar_SaveRejectsIllegalName 钉住 sidecar 写入的防御层:形态非法的名字 // 绝不落盘,也绝不把路径带出 sidecar 目录。 func TestSidecar_SaveRejectsIllegalName(t *testing.T) { t.Parallel() dir := t.TempDir() store := newSidecarStore(dir, sourcedLog(nil)) err := store.save("../escape.txt", cachedExtract{Status: history.DocumentStatusExtracted}, 1) if !errors.Is(err, ErrInvalidName) { t.Fatalf("save(非法名) = %v, want ErrInvalidName", err) } entries, readErr := os.ReadDir(dir) if readErr != nil { t.Fatalf("read dir: %v", readErr) } if len(entries) != 0 { t.Fatalf("非法名字竟然落盘了 %d 个文件", len(entries)) } } // TestSidecar_StaleVersionIsReextracted 钉住「抽取器版本」的用处(design/24 // §4.3):旧版本的 sidecar 判为失效并重抽一次,新版结论落到盘上。 func TestSidecar_StaleVersionIsReextracted(t *testing.T) { t.Parallel() layout := newTestLayout(t) body := "正文\n" name := layout.writeUpload(t, txtName, []byte(body)) if err := os.MkdirAll(filepath.Join(layout.dataDir, sidecarDirName), sidecarDirMode); err != nil { t.Fatalf("mkdir extracted: %v", err) } header, err := json.Marshal(sidecarMeta{Version: "documents/v0", Status: history.DocumentStatusExtracted, Chars: 2}) if err != nil { t.Fatalf("marshal meta: %v", err) } if err := os.WriteFile(layout.sidecarPath(name), append(header, '\n'), 0o600); err != nil { t.Fatalf("write stale sidecar: %v", err) } r := NewResolver(Config{DataDir: layout.dataDir, UploadsDir: layout.uploads}) ref, got := resolveOne(t, r, name) if ref.Status != history.DocumentStatusExtracted || got != body { t.Fatalf("解析 = (%+v, %q), want 重抽得到 %q", ref, got, body) } raw, err := os.ReadFile(layout.sidecarPath(name)) if err != nil { t.Fatalf("read sidecar: %v", err) } if !strings.Contains(string(raw), ExtractorVersion) { t.Fatalf("sidecar 仍是旧版本: %q", string(raw)) } } // TestSidecar_CorruptIsReextracted 钉住「缓存坏了就重抽」:损坏的 sidecar 绝不 // 让解析失败,也绝不把畸形内容当正文注入。 func TestSidecar_CorruptIsReextracted(t *testing.T) { t.Parallel() layout := newTestLayout(t) body := "好内容\n" name := layout.writeUpload(t, txtName, []byte(body)) if err := os.MkdirAll(filepath.Join(layout.dataDir, sidecarDirName), sidecarDirMode); err != nil { t.Fatalf("mkdir extracted: %v", err) } if err := os.WriteFile(layout.sidecarPath(name), []byte("not-json\n垃圾"), 0o600); err != nil { t.Fatalf("write corrupt sidecar: %v", err) } r := NewResolver(Config{DataDir: layout.dataDir, UploadsDir: layout.uploads}) ref, got := resolveOne(t, r, name) if ref.Status != history.DocumentStatusExtracted || got != body { t.Fatalf("解析 = (%+v, %q), want 重抽得到 %q", ref, got, body) } } // TestResolver_CanceledContextReturnsError 钉住契约里的 error 分支:单份文档的 // 失败是 unreadable 结论,而**整条消息解析不了**(回合已取消)才是 error,由 // 循环降级为诚实兜底。 func TestResolver_CanceledContextReturnsError(t *testing.T) { t.Parallel() layout := newTestLayout(t) name := layout.writeUpload(t, txtName, []byte("正文")) r := NewResolver(Config{DataDir: layout.dataDir, UploadsDir: layout.uploads}) ctx, cancel := context.WithCancel(context.Background()) cancel() msg := model.DocumentMessage(model.RoleUser, "", model.Document{Name: name, MIME: "text/plain"}) if _, _, err := r.Resolve(ctx, msg); !errors.Is(err, context.Canceled) { t.Fatalf("Resolve(已取消的 ctx) = %v, want context.Canceled", err) } } // TestResolver_NoDocumentsIsNoWork 钉住短路:没有文档部件时不做任何工作,也 // 不产生结论(使没有附件的回合与接线前逐字节一致)。 func TestResolver_NoDocumentsIsNoWork(t *testing.T) { t.Parallel() r := NewResolver(Config{}) refs, bodies, err := r.Resolve(context.Background(), model.TextMessage(model.RoleUser, "没有附件")) if err != nil || refs != nil || bodies != nil { t.Fatalf("Resolve(无文档) = (%v, %v, %v), want (nil, nil, nil)", refs, bodies, err) } } // TestResolver_UploadsDirMissingIsUnreadable 钉住配置缺失时的诚实:上传目录 // 没接上时每份文档都是 unreadable(而不是假装读到、也不是 error)。 func TestResolver_UploadsDirMissingIsUnreadable(t *testing.T) { t.Parallel() layout := newTestLayout(t) r := NewResolver(Config{DataDir: layout.dataDir}) ref, body := resolveOne(t, r, txtName) if ref.Status != history.DocumentStatusUnreadable || body != reasonNoUploadDir { t.Fatalf("解析 = (%+v, %q), want unreadable + %q", ref, body, reasonNoUploadDir) } } // recordingAdapter 是集成测试用的最小模型适配器:记录喂给模型的请求并回一句 // 文本,使测试能断言「模型实际看到了什么」。 type recordingAdapter struct{ reqs []model.Request } func (a *recordingAdapter) Name() string { return "recording" } func (a *recordingAdapter) Chat(_ context.Context, req model.Request, _ model.Options) (model.Response, error) { a.reqs = append(a.reqs, req) return model.Response{Model: "stub", Content: "好"}, nil } func (a *recordingAdapter) ChatStream(ctx context.Context, req model.Request, _ func(model.StreamEvent)) (model.Response, error) { return a.Chat(ctx, req, model.Options{}) } // TestResolver_EndToEndWithLoop 是本切片唯一的跨包集成:真实解析器 + 真实循环, // 证明「上传目录里的文档 → sidecar → 装配期注入 → 模型可见正文」这条链闭合, // 且文档部件在模型可见上下文里一个不剩(S2 的编码器对文档 fail-loud)。 func TestResolver_EndToEndWithLoop(t *testing.T) { t.Parallel() layout := newTestLayout(t) body := "季度报告:营收增长 12%。\n" name := layout.writeUpload(t, txtName, []byte(body)) r := NewResolver(Config{DataDir: layout.dataDir, UploadsDir: layout.uploads}) ad := &recordingAdapter{} ag, err := agent.New("你是助手。", nil, nil, agent.WithModel(ad, nil)) if err != nil { t.Fatalf("agent.New: %v", err) } l := loop.New(loop.Options{Agent: ag, DocumentResolver: r, MaxRounds: 1}) msg := model.DocumentMessage(model.RoleUser, "看看附件", model.Document{Name: name, MIME: "text/plain", Size: int64(len(body))}) res, err := l.Run(context.Background(), "sess-doc-e2e", []model.Message{msg}) if err != nil { t.Fatalf("Run: %v", err) } if len(ad.reqs) == 0 { t.Fatal("没有记录到模型请求") } injected := false for _, m := range ad.reqs[0].Messages { for _, part := range m.Content { if part.Type == model.ContentTypeDocument { t.Fatalf("文档部件抵达了模型: %+v", m) } if part.Type == model.ContentTypeText && strings.Contains(part.Text, "营收增长 12%") { injected = true } } } if !injected { t.Fatal("抽取正文没有注入模型可见上下文") } if len(res.Documents) != 1 || res.Documents[0].Status != history.DocumentStatusExtracted { t.Fatalf("Result.Documents = %+v, want 一条 extracted", res.Documents) } if _, err := os.Stat(layout.sidecarPath(name)); err != nil { t.Fatalf("sidecar 未写出: %v", err) } }