package loop import ( "context" "errors" "fmt" "strings" "testing" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/history" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model" ) // 本文件钉住装配期文档解析的四条硬规则(design/24 §2.3-①、§4): // ① 文档部件被确定性替换为「清单行 + 正文」;② 清单恒在(解析器缺席或失败也 // 一样);③ 抽取失败是可见的(诚实标记 + status=unreadable + 回合继续); // ④ 文档部件绝不抵达编码器。另加两处「静默吞掉」的回归(design/24 §9.5): // checkpoint 的用户消息比较键、压缩的 token 估算。 const ( // docA / docB 是合法的 S1 存储名形态(32 位十六进制 + 扩展名)。 docA = "0123456789abcdef0123456789abcdef.txt" docB = "fedcba9876543210fedcba9876543210.log" ) // errDocResolve 是测试用的解析器整条消息失败错误(单份文档失败不走 error)。 var errDocResolve = errors.New("文档解析服务不可用") // fakeDocumentResolver 是测试替身:直接给出结论与正文,不触盘、不依赖真实文件。 type fakeDocumentResolver struct { refs map[string]history.DocumentRef bodies map[string]string err error calls int } func (f *fakeDocumentResolver) Resolve(_ context.Context, _ model.Message) (map[string]history.DocumentRef, map[string]string, error) { f.calls++ if f.err != nil { return nil, nil, f.err } return f.refs, f.bodies, nil } // documentMessage 构造一条只带附件的用户消息。 func documentMessage(name, mime string, size int64) model.Message { return model.DocumentMessage(model.RoleUser, "", model.Document{Name: name, MIME: mime, Size: size}) } // messageTextOf 拼出消息里所有文本部件,供断言注入块的内容。 func messageTextOf(m model.Message) string { var b strings.Builder for _, part := range m.Content { if part.Type == model.ContentTypeText { b.WriteString(part.Text) } } return b.String() } // TestResolveDocuments_InjectsListAndBody 钉住主路径:清单行恒在,正文紧随其后, // 文档部件全部被替换掉(同一位置)。 func TestResolveDocuments_InjectsListAndBody(t *testing.T) { t.Parallel() res := &fakeDocumentResolver{ refs: map[string]history.DocumentRef{docA: {Status: history.DocumentStatusExtracted, Chars: 5}}, bodies: map[string]string{docA: "正文内容"}, } l := New(Options{DocumentResolver: res}) in := []model.Message{model.DocumentMessage(model.RoleUser, "看看这个", model.Document{Name: docA, MIME: "text/plain", Size: 2048})} out, refs := l.resolveDocuments(context.Background(), in) if len(out) != 1 || countDocumentParts(out[0]) != 0 { t.Fatalf("解析后仍有文档部件: %+v", out) } text := messageTextOf(out[0]) if !strings.Contains(text, documentListPrefix+docA) || !strings.Contains(text, "text/plain") { t.Fatalf("注入块缺清单行: %q", text) } if !strings.Contains(text, "正文内容") { t.Fatalf("注入块缺正文: %q", text) } if !strings.HasPrefix(text, "看看这个") { t.Fatalf("用户原文顺序被改变: %q", text) } if len(refs) != 1 || refs[0].Status != history.DocumentStatusExtracted || refs[0].Chars != 5 { t.Fatalf("结论 = %+v, want extracted/chars=5", refs) } if refs[0].Name != docA || refs[0].MIME != "text/plain" || refs[0].Size != 2048 { t.Fatalf("结论的身份字段 = %+v, want 取自部件", refs[0]) } } // TestResolveDocuments_KeepsListWithoutResolver 钉住「清单恒在」:没有解析器 // (忘了接线)时文档部件照样被中和为「仅清单」块,绝不出现空上下文。 func TestResolveDocuments_KeepsListWithoutResolver(t *testing.T) { t.Parallel() l := New(Options{}) out, refs := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docA, "text/plain", 10)}) if len(out) != 1 || countDocumentParts(out[0]) != 0 { t.Fatalf("没有解析器时文档部件仍抵达下游: %+v", out) } text := messageTextOf(out[0]) if !strings.Contains(text, documentListPrefix+docA) || !strings.Contains(text, documentNoteListed) { t.Fatalf("注入块 = %q, want 仅清单条目", text) } if len(refs) != 1 || refs[0].Status != history.DocumentStatusListed || refs[0].Chars != 0 { t.Fatalf("结论 = %+v, want listed/chars=0", refs) } } // TestResolveDocuments_TruncatedGivesCursor 钉住截断诚实(design/24 §4.3): // 截断时清单行给出续读游标(已注入字符数),绝不静默截断。 func TestResolveDocuments_TruncatedGivesCursor(t *testing.T) { t.Parallel() res := &fakeDocumentResolver{ refs: map[string]history.DocumentRef{docA: {Status: history.DocumentStatusExtracted, Chars: 700, Truncated: true}}, bodies: map[string]string{docA: "前 700 字符"}, } l := New(Options{DocumentResolver: res}) out, _ := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docA, "text/plain", 99)}) text := messageTextOf(out[0]) for _, want := range []string{"已截断", "offset=700"} { if !strings.Contains(text, want) { t.Fatalf("注入块 %q 缺 %q", text, want) } } } // TestResolveDocuments_UnreadableIsHonest 钉住失败可见(design/24 §4.4): // unreadable 结论 + 诚实原因文案 + 清单仍在,且解析不产生 error(回合继续)。 func TestResolveDocuments_UnreadableIsHonest(t *testing.T) { t.Parallel() res := &fakeDocumentResolver{ refs: map[string]history.DocumentRef{docB: {Status: history.DocumentStatusUnreadable}}, bodies: map[string]string{docB: "无法抽取正文:无文本层"}, } l := New(Options{DocumentResolver: res}) out, refs := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docB, "application/pdf", 1024)}) text := messageTextOf(out[0]) if !strings.Contains(text, documentListPrefix+docB) || !strings.Contains(text, documentNoteUnreadable) { t.Fatalf("注入块 = %q, want 清单行标不可读", text) } if !strings.Contains(text, "无文本层") { t.Fatalf("注入块缺诚实原因: %q", text) } if len(refs) != 1 || refs[0].Status != history.DocumentStatusUnreadable { t.Fatalf("结论 = %+v, want unreadable", refs) } } // TestResolveDocuments_ResolverErrorFallsBackHonestly 钉住解析器整体失败时的 // 降级:诚实兜底原因 + 清单仍在 + unreadable 状态,绝不空上下文。 func TestResolveDocuments_ResolverErrorFallsBackHonestly(t *testing.T) { t.Parallel() l := New(Options{DocumentResolver: &fakeDocumentResolver{err: errDocResolve}}) out, refs := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docA, "text/plain", 10)}) text := messageTextOf(out[0]) if !strings.Contains(text, documentListPrefix+docA) || !strings.Contains(text, documentUnreadableBody) { t.Fatalf("注入块 = %q, want 清单行 + 诚实兜底", text) } if len(refs) != 1 || refs[0].Status != history.DocumentStatusUnreadable { t.Fatalf("结论 = %+v, want unreadable", refs) } } // TestResolveDocuments_NeutralizesNonUserParts 钉住红线(design/11 §10.2):文档 // 部件落到非用户回合上是编程错误,但绝不静默丢弃 —— 它照样被中和成清单块。 func TestResolveDocuments_NeutralizesNonUserParts(t *testing.T) { t.Parallel() l := New(Options{}) in := []model.Message{model.DocumentMessage(model.RoleAssistant, "", model.Document{Name: docA, MIME: "text/plain"})} out, refs := l.resolveDocuments(context.Background(), in) if countDocumentParts(out[0]) != 0 || !strings.Contains(messageTextOf(out[0]), documentListPrefix+docA) { t.Fatalf("非用户回合的文档部件未被中和: %+v", out[0]) } if len(refs) != 1 { t.Fatalf("结论数 = %d, want 1", len(refs)) } } // TestRun_DocumentPartsNeverReachEncoder 是同名不变式的负例:无论解析器缺席、 // 解析器报错还是解析成功,喂给模型的每条请求里都不得出现文档部件(S2 的编码器 // 对文档 fail-loud,漏一个就是一次 400)。 func TestRun_DocumentPartsNeverReachEncoder(t *testing.T) { t.Parallel() cases := []struct { name string resolver DocumentResolver }{ {name: "无解析器", resolver: nil}, {name: "解析器报错", resolver: &fakeDocumentResolver{err: errDocResolve}}, {name: "解析成功", resolver: &fakeDocumentResolver{ refs: map[string]history.DocumentRef{docA: {Status: history.DocumentStatusExtracted, Chars: 3}}, bodies: map[string]string{docA: "正文"}, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { t.Parallel() ad := &mockAdapter{chat: []mgResp{{resp: model.Response{Model: "m", Content: "Done."}}}} l := New(Options{Agent: newTestAgent(t, ad, nil), MaxRounds: 4, DocumentResolver: c.resolver}) in := []model.Message{ documentMessage(docA, "text/plain", 12), model.DocumentMessage(model.RoleAssistant, "", model.Document{Name: docB, MIME: "text/plain"}), } if _, err := l.Run(context.Background(), "sess-doc", in); err != nil { t.Fatalf("Run: %v", err) } assertNoDocumentParts(t, ad.reqs) }) } } // assertNoDocumentParts 断言每条模型请求都不含文档部件,并且能通过编码器入口 // 的同一道校验(model.ValidateDocumentParts)。 func assertNoDocumentParts(t *testing.T, reqs []model.Request) { t.Helper() if len(reqs) == 0 { t.Fatal("没有记录到任何模型请求") } for _, req := range reqs { for _, m := range req.Messages { if countDocumentParts(m) != 0 { t.Fatalf("文档部件抵达了模型: %+v", m) } } if err := model.ValidateDocumentParts(req.Messages); err != nil { t.Fatalf("编码器边界校验失败: %v", err) } } } // TestRun_RecordsDocumentOutcomesOnResult 钉住 status 的回传出口:结论随 Result // 返回(按存储名去重、保留首次出现顺序),使客户端能看到每份附件的结果。 func TestRun_RecordsDocumentOutcomesOnResult(t *testing.T) { t.Parallel() res := &fakeDocumentResolver{ refs: map[string]history.DocumentRef{ docA: {Status: history.DocumentStatusExtracted, Chars: 3}, docB: {Status: history.DocumentStatusUnreadable}, }, bodies: map[string]string{docA: "正文", docB: "无法抽取正文:不存在"}, } ad := &mockAdapter{chat: []mgResp{{resp: model.Response{Model: "m", Content: "Done."}}}} l := New(Options{Agent: newTestAgent(t, ad, nil), MaxRounds: 4, DocumentResolver: res}) in := []model.Message{documentMessage(docA, "text/plain", 12), documentMessage(docB, "text/plain", 8)} out, err := l.Run(context.Background(), "sess-doc", in) if err != nil { t.Fatalf("Run: %v", err) } if len(out.Documents) != 2 { t.Fatalf("Documents = %+v, want 2 条(去重后)", out.Documents) } if out.Documents[0].Name != docA || out.Documents[0].Status != history.DocumentStatusExtracted { t.Fatalf("Documents[0] = %+v, want %s/extracted", out.Documents[0], docA) } if out.Documents[1].Name != docB || out.Documents[1].Status != history.DocumentStatusUnreadable { t.Fatalf("Documents[1] = %+v, want %s/unreadable", out.Documents[1], docB) } } // TestUserMessageKey_IncludesDocumentRef 是 design/24 §9.5 移交的回归:纯文档 // 用户消息(客户端「超限走文件」发的就是它)必须在比较键里有内容,否则续接判定 // 会把附件回合当成「没有可比内容」而漏判。 func TestUserMessageKey_IncludesDocumentRef(t *testing.T) { t.Parallel() docOnly := documentMessage(docA, "text/plain", 10) if key := userMessageKey(docOnly); key == "" { t.Fatal("纯文档消息的比较键为空") } if lastUserMessageKey([]model.Message{docOnly}) == "" { t.Fatal("纯文档回合的最后一条用户消息键为空") } if userMessageKey(docOnly) == userMessageKey(documentMessage(docB, "text/plain", 10)) { t.Fatal("两份不同附件得到了同一个比较键") } } // TestEstimateContextTokens_CountsDocuments 是 design/24 §9.5 移交的回归:文档 // 部件必须在 token 估算里占分量,否则纯文档回合被估成 0 token,在压缩里等同于 // 空内容。 func TestEstimateContextTokens_CountsDocuments(t *testing.T) { t.Parallel() small := []model.Message{documentMessage(docA, "text/plain", 1024)} big := []model.Message{documentMessage(docA, "text/plain", 4<<20)} if got := estimateContextTokens(small); got <= messageOverheadTokens { t.Fatalf("纯文档回合估算 = %d, want > %d(文档部件不是免费的)", got, messageOverheadTokens) } if estimateContextTokens(big) <= estimateContextTokens(small) { t.Fatalf("大附件估算未变大:%d vs %d", estimateContextTokens(big), estimateContextTokens(small)) } if got := documentPartTokens(model.Content{Type: model.ContentTypeDocument}); got != documentMinTokens { t.Fatalf("documentPartTokens(未知字节数) = %d, want %d", got, documentMinTokens) } if got := documentPartTokens(model.Content{Type: model.ContentTypeDocument, DocumentSize: documentInjectedCharsCap * 4}) / 1; got != documentInjectedCharsCap/charsPerToken { t.Fatalf("documentPartTokens(超大附件) = %d, want 按注入预算封顶 %d", got, documentInjectedCharsCap/charsPerToken) } } // TestFormatDocumentSize 钉住清单行里的展示口径(只用于人读,不做任何判定)。 func TestFormatDocumentSize(t *testing.T) { t.Parallel() cases := []struct { in int64 want string }{ {0, "0 B"}, {512, "512 B"}, {2048, "2.0 KiB"}, {3 << 20, "3.0 MiB"}, {-1, "0 B"}, } for _, c := range cases { if got := formatDocumentSize(c.in); got != c.want { t.Fatalf("formatDocumentSize(%d) = %q, want %q", c.in, got, c.want) } } } // TestDocumentBlockShape 钉住注入块的稳定形状(清单行在前、正文换行在后), // 因为客户端渲染与「清单恒在」都依赖它。 func TestDocumentBlockShape(t *testing.T) { t.Parallel() part := model.Content{Type: model.ContentTypeDocument, DocumentName: docA, DocumentMIME: "text/plain", DocumentSize: 2048} block := documentBlock(part, history.DocumentRef{Name: docA, Status: history.DocumentStatusExtracted, Chars: 5}, "正文") want := fmt.Sprintf("%s%s · text/plain · 2.0 KiB · 已注入正文 5 字符]\n正文", documentListPrefix, docA) if block != want { t.Fatalf("documentBlock = %q, want %q", block, want) } empty := documentBlock(model.Content{Type: model.ContentTypeDocument, DocumentName: docA}, history.DocumentRef{Status: history.DocumentStatusListed}, "") if strings.Contains(empty, "\n") || !strings.Contains(empty, documentNoteListed) { t.Fatalf("无正文时不该有换行: %q", empty) } }