package loop import ( "context" "errors" "fmt" "strings" "testing" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/history" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model" ) // 本文件钉住装配期文档解析的四条硬规则(design/24 §2.3-①、§4): // ① 文档部件被确定性替换为「清单行 + 正文」;② 清单恒在(解析器缺席或失败也 // 一样);③ 抽取失败是可见的(诚实标记 + status=unreadable + 回合继续); // ④ 文档部件绝不抵达编码器。另加两处「静默吞掉」的回归(design/24 §9.5): // checkpoint 的用户消息比较键、压缩的 token 估算。 const ( // docA / docB 是合法的 S1 存储名形态(32 位十六进制 + 扩展名)。 docA = "0123456789abcdef0123456789abcdef.txt" docB = "fedcba9876543210fedcba9876543210.log" ) // errDocResolve 是测试用的解析器整条消息失败错误(单份文档失败不走 error)。 var errDocResolve = errors.New("文档解析服务不可用") // fakeDocumentResolver 是测试替身:直接给出结论与正文,不触盘、不依赖真实文件。 type fakeDocumentResolver struct { refs map[string]history.DocumentRef bodies map[string]string err error calls int } func (f *fakeDocumentResolver) Resolve(_ context.Context, _ model.Message) (map[string]history.DocumentRef, map[string]string, error) { f.calls++ if f.err != nil { return nil, nil, f.err } return f.refs, f.bodies, nil } // documentMessage 构造一条只带附件的用户消息。 func documentMessage(name, mime string, size int64) model.Message { return model.DocumentMessage(model.RoleUser, "", model.Document{Name: name, MIME: mime, Size: size}) } // messageTextOf 拼出消息里所有文本部件,供断言注入块的内容。 func messageTextOf(m model.Message) string { var b strings.Builder for _, part := range m.Content { if part.Type == model.ContentTypeText { b.WriteString(part.Text) } } return b.String() } // TestResolveDocuments_InjectsListAndBody 钉住主路径:清单行恒在,正文紧随其后, // 文档部件全部被替换掉(同一位置)。Chars 是**实际注入**的字符数(装配期按注入 // 预算算出,切片 T1 §7.2),不是解析器自称的那个数。 func TestResolveDocuments_InjectsListAndBody(t *testing.T) { t.Parallel() res := &fakeDocumentResolver{ refs: map[string]history.DocumentRef{docA: {Status: history.DocumentStatusExtracted, Chars: 4}}, bodies: map[string]string{docA: "正文内容"}, } l := New(Options{DocumentResolver: res}) in := []model.Message{model.DocumentMessage(model.RoleUser, "看看这个", model.Document{Name: docA, MIME: "text/plain", Size: 2048})} out, refs := l.resolveDocuments(context.Background(), in) if len(out) != 1 || countDocumentParts(out[0]) != 0 { t.Fatalf("解析后仍有文档部件: %+v", out) } text := messageTextOf(out[0]) if !strings.Contains(text, documentListPrefix+docA) || !strings.Contains(text, "text/plain") { t.Fatalf("注入块缺清单行: %q", text) } if !strings.Contains(text, "正文内容") { t.Fatalf("注入块缺正文: %q", text) } if !strings.HasPrefix(text, "看看这个") { t.Fatalf("用户原文顺序被改变: %q", text) } if len(refs) != 1 || refs[0].Status != history.DocumentStatusExtracted || refs[0].Chars != 4 { t.Fatalf("结论 = %+v, want extracted/chars=4", refs) } if refs[0].Name != docA || refs[0].MIME != "text/plain" || refs[0].Size != 2048 { t.Fatalf("结论的身份字段 = %+v, want 取自部件", refs[0]) } } // TestResolveDocuments_KeepsListWithoutResolver 钉住「清单恒在」:没有解析器 // (忘了接线)时文档部件照样被中和为「仅清单」块,绝不出现空上下文。 func TestResolveDocuments_KeepsListWithoutResolver(t *testing.T) { t.Parallel() l := New(Options{}) out, refs := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docA, "text/plain", 10)}) if len(out) != 1 || countDocumentParts(out[0]) != 0 { t.Fatalf("没有解析器时文档部件仍抵达下游: %+v", out) } text := messageTextOf(out[0]) if !strings.Contains(text, documentListPrefix+docA) || !strings.Contains(text, documentNoteListed) { t.Fatalf("注入块 = %q, want 仅清单条目", text) } if len(refs) != 1 || refs[0].Status != history.DocumentStatusListed || refs[0].Chars != 0 { t.Fatalf("结论 = %+v, want listed/chars=0", refs) } } // TestResolveDocuments_TruncatedGivesCursor 钉住截断诚实(design/24 §4.3): // 注入被截断时清单行给出续读游标(已注入字符数 = 继续读的起点),绝不静默截断。 func TestResolveDocuments_TruncatedGivesCursor(t *testing.T) { t.Parallel() res := &fakeDocumentResolver{ refs: map[string]history.DocumentRef{docA: {Status: history.DocumentStatusExtracted, Chars: 700}}, bodies: map[string]string{docA: strings.Repeat("正", 700)}, } l := New(Options{DocumentResolver: res, DocumentMaxInjectChars: 100}) out, _ := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docA, "text/plain", 99)}) text := messageTextOf(out[0]) for _, want := range []string{"已截断", "offset=100"} { if !strings.Contains(text, want) { t.Fatalf("注入块 %q 缺 %q", text, want) } } _, body, _ := strings.Cut(text, "\n") if body != strings.Repeat("正", 100) { t.Fatalf("注入正文不是 100 个字符的 rune 前缀: %q", body) } } // TestResolveDocuments_UnreadableIsHonest 钉住失败可见(design/24 §4.4): // unreadable 结论 + 诚实原因文案 + 清单仍在,且解析不产生 error(回合继续)。 func TestResolveDocuments_UnreadableIsHonest(t *testing.T) { t.Parallel() res := &fakeDocumentResolver{ refs: map[string]history.DocumentRef{docB: {Status: history.DocumentStatusUnreadable}}, bodies: map[string]string{docB: "无法抽取正文:无文本层"}, } l := New(Options{DocumentResolver: res}) out, refs := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docB, "application/pdf", 1024)}) text := messageTextOf(out[0]) if !strings.Contains(text, documentListPrefix+docB) || !strings.Contains(text, documentNoteUnreadable) { t.Fatalf("注入块 = %q, want 清单行标不可读", text) } if !strings.Contains(text, "无文本层") { t.Fatalf("注入块缺诚实原因: %q", text) } if len(refs) != 1 || refs[0].Status != history.DocumentStatusUnreadable { t.Fatalf("结论 = %+v, want unreadable", refs) } } // TestResolveDocuments_ResolverErrorFallsBackHonestly 钉住解析器整体失败时的 // 降级:诚实兜底原因 + 清单仍在 + unreadable 状态,绝不空上下文。 func TestResolveDocuments_ResolverErrorFallsBackHonestly(t *testing.T) { t.Parallel() l := New(Options{DocumentResolver: &fakeDocumentResolver{err: errDocResolve}}) out, refs := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docA, "text/plain", 10)}) text := messageTextOf(out[0]) if !strings.Contains(text, documentListPrefix+docA) || !strings.Contains(text, documentUnreadableBody) { t.Fatalf("注入块 = %q, want 清单行 + 诚实兜底", text) } if len(refs) != 1 || refs[0].Status != history.DocumentStatusUnreadable { t.Fatalf("结论 = %+v, want unreadable", refs) } } // TestResolveDocuments_NeutralizesNonUserParts 钉住红线(design/11 §10.2):文档 // 部件落到非用户回合上是编程错误,但绝不静默丢弃 —— 它照样被中和成清单块。 func TestResolveDocuments_NeutralizesNonUserParts(t *testing.T) { t.Parallel() l := New(Options{}) in := []model.Message{model.DocumentMessage(model.RoleAssistant, "", model.Document{Name: docA, MIME: "text/plain"})} out, refs := l.resolveDocuments(context.Background(), in) if countDocumentParts(out[0]) != 0 || !strings.Contains(messageTextOf(out[0]), documentListPrefix+docA) { t.Fatalf("非用户回合的文档部件未被中和: %+v", out[0]) } if len(refs) != 1 { t.Fatalf("结论数 = %d, want 1", len(refs)) } } // TestRun_DocumentPartsNeverReachEncoder 是同名不变式的负例:无论解析器缺席、 // 解析器报错还是解析成功,喂给模型的每条请求里都不得出现文档部件(S2 的编码器 // 对文档 fail-loud,漏一个就是一次 400)。 func TestRun_DocumentPartsNeverReachEncoder(t *testing.T) { t.Parallel() cases := []struct { name string resolver DocumentResolver }{ {name: "无解析器", resolver: nil}, {name: "解析器报错", resolver: &fakeDocumentResolver{err: errDocResolve}}, {name: "解析成功", resolver: &fakeDocumentResolver{ refs: map[string]history.DocumentRef{docA: {Status: history.DocumentStatusExtracted, Chars: 3}}, bodies: map[string]string{docA: "正文"}, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { t.Parallel() ad := &mockAdapter{chat: []mgResp{{resp: model.Response{Model: "m", Content: "Done."}}}} l := New(Options{Agent: newTestAgent(t, ad, nil), MaxRounds: 4, DocumentResolver: c.resolver}) in := []model.Message{ documentMessage(docA, "text/plain", 12), model.DocumentMessage(model.RoleAssistant, "", model.Document{Name: docB, MIME: "text/plain"}), } if _, err := l.Run(context.Background(), "sess-doc", in); err != nil { t.Fatalf("Run: %v", err) } assertNoDocumentParts(t, ad.reqs) }) } } // assertNoDocumentParts 断言每条模型请求都不含文档部件,并且能通过编码器入口 // 的同一道校验(model.ValidateDocumentParts)。 func assertNoDocumentParts(t *testing.T, reqs []model.Request) { t.Helper() if len(reqs) == 0 { t.Fatal("没有记录到任何模型请求") } for _, req := range reqs { for _, m := range req.Messages { if countDocumentParts(m) != 0 { t.Fatalf("文档部件抵达了模型: %+v", m) } } if err := model.ValidateDocumentParts(req.Messages); err != nil { t.Fatalf("编码器边界校验失败: %v", err) } } } // TestRun_RecordsDocumentOutcomesOnResult 钉住 status 的回传出口:结论随 Result // 返回(按存储名去重、保留首次出现顺序),使客户端能看到每份附件的结果。 func TestRun_RecordsDocumentOutcomesOnResult(t *testing.T) { t.Parallel() res := &fakeDocumentResolver{ refs: map[string]history.DocumentRef{ docA: {Status: history.DocumentStatusExtracted, Chars: 3}, docB: {Status: history.DocumentStatusUnreadable}, }, bodies: map[string]string{docA: "正文", docB: "无法抽取正文:不存在"}, } ad := &mockAdapter{chat: []mgResp{{resp: model.Response{Model: "m", Content: "Done."}}}} l := New(Options{Agent: newTestAgent(t, ad, nil), MaxRounds: 4, DocumentResolver: res}) in := []model.Message{documentMessage(docA, "text/plain", 12), documentMessage(docB, "text/plain", 8)} out, err := l.Run(context.Background(), "sess-doc", in) if err != nil { t.Fatalf("Run: %v", err) } if len(out.Documents) != 2 { t.Fatalf("Documents = %+v, want 2 条(去重后)", out.Documents) } if out.Documents[0].Name != docA || out.Documents[0].Status != history.DocumentStatusExtracted { t.Fatalf("Documents[0] = %+v, want %s/extracted", out.Documents[0], docA) } if out.Documents[1].Name != docB || out.Documents[1].Status != history.DocumentStatusUnreadable { t.Fatalf("Documents[1] = %+v, want %s/unreadable", out.Documents[1], docB) } } // TestUserMessageKey_IncludesDocumentRef 是 design/24 §9.5 移交的回归:纯文档 // 用户消息(客户端「超限走文件」发的就是它)必须在比较键里有内容,否则续接判定 // 会把附件回合当成「没有可比内容」而漏判。 func TestUserMessageKey_IncludesDocumentRef(t *testing.T) { t.Parallel() docOnly := documentMessage(docA, "text/plain", 10) if key := userMessageKey(docOnly); key == "" { t.Fatal("纯文档消息的比较键为空") } if lastUserMessageKey([]model.Message{docOnly}) == "" { t.Fatal("纯文档回合的最后一条用户消息键为空") } if userMessageKey(docOnly) == userMessageKey(documentMessage(docB, "text/plain", 10)) { t.Fatal("两份不同附件得到了同一个比较键") } } // TestResolveDocuments_AppliesInjectionBudget 钉住切片 T1 §7.2 第 2 条:注入预算在 // **装配期**生效 —— ref.Chars 是实际注入的字符数、ref.Truncated 是注入是否被截断, // 清单行的续读游标因此是「继续读的起点」(sidecar 存全文,游标之外还有正文)。 func TestResolveDocuments_AppliesInjectionBudget(t *testing.T) { t.Parallel() const budget = 10 body := "0123456789ABCDEFGHIJ" res := &fakeDocumentResolver{ // 解析器给的是抽取侧的事实(sidecar 全文 20 字符、未撞安全上限)。 refs: map[string]history.DocumentRef{docA: {Status: history.DocumentStatusExtracted, Chars: len(body)}}, bodies: map[string]string{docA: body}, } l := New(Options{DocumentResolver: res, DocumentMaxInjectChars: budget}) out, refs := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docA, "text/plain", 20)}) text := messageTextOf(out[0]) if !strings.Contains(text, "已截断") || !strings.Contains(text, fmt.Sprintf("offset=%d", budget)) { t.Fatalf("注入块缺截断与续读游标: %q", text) } if !strings.Contains(text, body[:budget]) || strings.Contains(text, body[budget:]) { t.Fatalf("注入正文不是预算内的前缀: %q", text) } if len(refs) != 1 || refs[0].Chars != budget || !refs[0].Truncated { t.Fatalf("结论 = %+v, want chars=%d + truncated(注入侧事实)", refs, budget) } // 正文在预算内时:不截断、不给游标,Chars 仍是实际注入的字符数。 small := &fakeDocumentResolver{ refs: map[string]history.DocumentRef{docA: {Status: history.DocumentStatusExtracted, Chars: 3}}, bodies: map[string]string{docA: "短正文"}, } l2 := New(Options{DocumentResolver: small, DocumentMaxInjectChars: budget}) out2, refs2 := l2.resolveDocuments(context.Background(), []model.Message{documentMessage(docA, "text/plain", 9)}) if text2 := messageTextOf(out2[0]); strings.Contains(text2, "已截断") { t.Fatalf("预算内却被标成截断: %q", text2) } if len(refs2) != 1 || refs2[0].Chars != 3 || refs2[0].Truncated { t.Fatalf("结论 = %+v, want chars=3 + 未截断", refs2) } } // TestUnexportedInjectBudgetDefaultsOnConstruction 钉住零值语义:Options 不设 // DocumentMaxInjectChars 时用缺省 64K 字符(小窗口部署行为不退化),而不是 0 // (0 会把正文整个截掉)。 func TestUnexportedInjectBudgetDefaultsOnConstruction(t *testing.T) { t.Parallel() if got := New(Options{}).documentInjectChars; got != defaultDocumentInjectChars { t.Fatalf("缺省注入预算 = %d, want %d", got, defaultDocumentInjectChars) } if got := New(Options{DocumentMaxInjectChars: -5}).documentInjectChars; got != defaultDocumentInjectChars { t.Fatalf("非正注入预算 = %d, want 回落 %d", got, defaultDocumentInjectChars) } if got := New(Options{DocumentMaxInjectChars: 1234}).documentInjectChars; got != 1234 { t.Fatalf("显式注入预算 = %d, want 1234", got) } } // TestDocumentInjectCharsFor 钉住「注入预算随上下文预算推导」这条口径(表驱动): // 1M 窗口推导出的 699050 token ⇒ 单份附件可注入 300K+ 字符(远超旧的 64K), // 但小预算下不跌破历史默认,且绝不超过预算本身。 func TestDocumentInjectCharsFor(t *testing.T) { t.Parallel() cases := []struct { name string budget int want int }{ {"1M 窗口推导出的预算", 699050, 699050 * charsPerToken / documentInjectBudgetShare}, {"fallback 窗口推导出的预算(推导值跌破历史默认 ⇒ 取默认)", 87381, defaultDocumentInjectChars}, {"小预算不超过预算本身", 32768, 32768}, {"预算关闭", 0, defaultDocumentInjectChars}, {"负预算", -1, defaultDocumentInjectChars}, } for _, c := range cases { if got := DocumentInjectCharsFor(c.budget); got != c.want { t.Fatalf("%s: DocumentInjectCharsFor(%d) = %d, want %d", c.name, c.budget, got, c.want) } } if big := DocumentInjectCharsFor(699050); big <= defaultDocumentInjectChars { t.Fatalf("大窗口下的注入预算 = %d, want > 旧的 64K 卡死值", big) } } // TestEstimateContextTokens_CountsDocuments 是 design/24 §9.5 移交的回归:文档 // 部件必须在 token 估算里占分量,否则纯文档回合被估成 0 token,在压缩里等同于 // 空内容。 func TestEstimateContextTokens_CountsDocuments(t *testing.T) { t.Parallel() small := []model.Message{documentMessage(docA, "text/plain", 1024)} big := []model.Message{documentMessage(docA, "text/plain", 4<<20)} if got := estimateContextTokens(small); got <= messageOverheadTokens { t.Fatalf("纯文档回合估算 = %d, want > %d(文档部件不是免费的)", got, messageOverheadTokens) } if estimateContextTokens(big) <= estimateContextTokens(small) { t.Fatalf("大附件估算未变大:%d vs %d", estimateContextTokens(big), estimateContextTokens(small)) } if got := documentPartTokens(model.Content{Type: model.ContentTypeDocument}, 0); got != documentMinTokens { t.Fatalf("documentPartTokens(未知字节数) = %d, want %d", got, documentMinTokens) } if got := documentPartTokens(model.Content{Type: model.ContentTypeDocument, DocumentSize: defaultDocumentInjectChars * 4}, defaultDocumentInjectChars); got != defaultDocumentInjectChars/charsPerToken { t.Fatalf("documentPartTokens(超大附件) = %d, want 按注入预算封顶 %d", got, defaultDocumentInjectChars/charsPerToken) } // 注入预算变大(1M 窗口部署)时,同一个文档部件必须按新预算计入 —— 否则 // 注入 683K 字符的附件在估算里只按 64K 计,压缩永远迟一步。 bigger := DocumentInjectCharsFor(699050) if bigger <= defaultDocumentInjectChars { t.Fatalf("DocumentInjectCharsFor(1M 窗口推导出的预算) = %d, want > %d", bigger, defaultDocumentInjectChars) } if got, want := documentPartTokens(model.Content{Type: model.ContentTypeDocument, DocumentSize: int64(bigger)}, bigger), bigger/charsPerToken; got != want { t.Fatalf("按更大注入预算的估算 = %d, want %d", got, want) } } // TestFormatDocumentSize 钉住清单行里的展示口径(只用于人读,不做任何判定)。 func TestFormatDocumentSize(t *testing.T) { t.Parallel() cases := []struct { in int64 want string }{ {0, "0 B"}, {512, "512 B"}, {2048, "2.0 KiB"}, {3 << 20, "3.0 MiB"}, {-1, "0 B"}, } for _, c := range cases { if got := formatDocumentSize(c.in); got != c.want { t.Fatalf("formatDocumentSize(%d) = %q, want %q", c.in, got, c.want) } } } // TestDocumentBlockShape 钉住注入块的稳定形状(清单行在前、正文换行在后), // 因为客户端渲染与「清单恒在」都依赖它。 func TestDocumentBlockShape(t *testing.T) { t.Parallel() part := model.Content{Type: model.ContentTypeDocument, DocumentName: docA, DocumentMIME: "text/plain", DocumentSize: 2048} block := documentBlock(part, history.DocumentRef{Name: docA, Status: history.DocumentStatusExtracted, Chars: 5}, "正文") want := fmt.Sprintf("%s%s · text/plain · 2.0 KiB · 已注入正文 5 字符]\n正文", documentListPrefix, docA) if block != want { t.Fatalf("documentBlock = %q, want %q", block, want) } empty := documentBlock(model.Content{Type: model.ContentTypeDocument, DocumentName: docA}, history.DocumentRef{Status: history.DocumentStatusListed}, "") if strings.Contains(empty, "\n") || !strings.Contains(empty, documentNoteListed) { t.Fatalf("无正文时不该有换行: %q", empty) } }