AskaEth
a8851dc0a2
- 抽取器零新依赖纯函数：UTF-8 清洗、CSV/TSV 行数列名摘要、64K rune 安全截断；PDF/Office/二进制走哨兵错误诚实拒绝（不引解析库）。 - sidecar：<DataDir>/extracted/<存储名>.txt，首行紧凑 JSON 元数据 + 正文；临时文件 + rename 原子落定；只抽一次（删原文件仍出正文）；版本不匹配/坏缓存重抽。 - 装配期 resolveDocuments（resolveImages 同位，每步对模型可见视图注入）：清单恒在（解析器缺席/报错也在）、截断带续读游标、失败诚实标记 + status=unreadable、回合继续；文档部件永不抵达编码器（负例钉住）。 - 修 S2 移交的两处静默吞掉：checkpoint 比较键纳入文档引用、compactor token 估算计入文档部件（不是免费的）。 - 主会话独立验收：隔离 worktree（HEAD + 本切片）build/vet 绿、全树 59 包 0 FAIL、linux/amd64 + darwin/arm64 交叉编译绿、documents+loop 两包 167 条 PASS；变异独立复现（resolveDocuments 短路 ⇒ 3 条红，含「文档部件仍抵达下游」）。
339 lines
14 KiB
Go
339 lines
14 KiB
Go
package loop
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/history"
|
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
|
)
|
|
|
|
// 本文件钉住装配期文档解析的四条硬规则（design/24 §2.3-①、§4）：
|
|
// ① 文档部件被确定性替换为「清单行 + 正文」；② 清单恒在（解析器缺席或失败也
|
|
// 一样）；③ 抽取失败是可见的（诚实标记 + status=unreadable + 回合继续）；
|
|
// ④ 文档部件绝不抵达编码器。另加两处「静默吞掉」的回归（design/24 §9.5）：
|
|
// checkpoint 的用户消息比较键、压缩的 token 估算。
|
|
|
|
const (
|
|
// docA / docB 是合法的 S1 存储名形态（32 位十六进制 + 扩展名）。
|
|
docA = "0123456789abcdef0123456789abcdef.txt"
|
|
docB = "fedcba9876543210fedcba9876543210.log"
|
|
)
|
|
|
|
// errDocResolve 是测试用的解析器整条消息失败错误（单份文档失败不走 error）。
|
|
var errDocResolve = errors.New("文档解析服务不可用")
|
|
|
|
// fakeDocumentResolver 是测试替身：直接给出结论与正文，不触盘、不依赖真实文件。
|
|
type fakeDocumentResolver struct {
|
|
refs map[string]history.DocumentRef
|
|
bodies map[string]string
|
|
err error
|
|
calls int
|
|
}
|
|
|
|
func (f *fakeDocumentResolver) Resolve(_ context.Context, _ model.Message) (map[string]history.DocumentRef, map[string]string, error) {
|
|
f.calls++
|
|
if f.err != nil {
|
|
return nil, nil, f.err
|
|
}
|
|
return f.refs, f.bodies, nil
|
|
}
|
|
|
|
// documentMessage 构造一条只带附件的用户消息。
|
|
func documentMessage(name, mime string, size int64) model.Message {
|
|
return model.DocumentMessage(model.RoleUser, "", model.Document{Name: name, MIME: mime, Size: size})
|
|
}
|
|
|
|
// messageTextOf 拼出消息里所有文本部件，供断言注入块的内容。
|
|
func messageTextOf(m model.Message) string {
|
|
var b strings.Builder
|
|
for _, part := range m.Content {
|
|
if part.Type == model.ContentTypeText {
|
|
b.WriteString(part.Text)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// TestResolveDocuments_InjectsListAndBody 钉住主路径：清单行恒在，正文紧随其后，
|
|
// 文档部件全部被替换掉（同一位置）。
|
|
func TestResolveDocuments_InjectsListAndBody(t *testing.T) {
|
|
t.Parallel()
|
|
res := &fakeDocumentResolver{
|
|
refs: map[string]history.DocumentRef{docA: {Status: history.DocumentStatusExtracted, Chars: 5}},
|
|
bodies: map[string]string{docA: "正文内容"},
|
|
}
|
|
l := New(Options{DocumentResolver: res})
|
|
in := []model.Message{model.DocumentMessage(model.RoleUser, "看看这个", model.Document{Name: docA, MIME: "text/plain", Size: 2048})}
|
|
|
|
out, refs := l.resolveDocuments(context.Background(), in)
|
|
if len(out) != 1 || countDocumentParts(out[0]) != 0 {
|
|
t.Fatalf("解析后仍有文档部件: %+v", out)
|
|
}
|
|
text := messageTextOf(out[0])
|
|
if !strings.Contains(text, documentListPrefix+docA) || !strings.Contains(text, "text/plain") {
|
|
t.Fatalf("注入块缺清单行: %q", text)
|
|
}
|
|
if !strings.Contains(text, "正文内容") {
|
|
t.Fatalf("注入块缺正文: %q", text)
|
|
}
|
|
if !strings.HasPrefix(text, "看看这个") {
|
|
t.Fatalf("用户原文顺序被改变: %q", text)
|
|
}
|
|
if len(refs) != 1 || refs[0].Status != history.DocumentStatusExtracted || refs[0].Chars != 5 {
|
|
t.Fatalf("结论 = %+v, want extracted/chars=5", refs)
|
|
}
|
|
if refs[0].Name != docA || refs[0].MIME != "text/plain" || refs[0].Size != 2048 {
|
|
t.Fatalf("结论的身份字段 = %+v, want 取自部件", refs[0])
|
|
}
|
|
}
|
|
|
|
// TestResolveDocuments_KeepsListWithoutResolver 钉住「清单恒在」：没有解析器
|
|
// （忘了接线）时文档部件照样被中和为「仅清单」块，绝不出现空上下文。
|
|
func TestResolveDocuments_KeepsListWithoutResolver(t *testing.T) {
|
|
t.Parallel()
|
|
l := New(Options{})
|
|
out, refs := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docA, "text/plain", 10)})
|
|
if len(out) != 1 || countDocumentParts(out[0]) != 0 {
|
|
t.Fatalf("没有解析器时文档部件仍抵达下游: %+v", out)
|
|
}
|
|
text := messageTextOf(out[0])
|
|
if !strings.Contains(text, documentListPrefix+docA) || !strings.Contains(text, documentNoteListed) {
|
|
t.Fatalf("注入块 = %q, want 仅清单条目", text)
|
|
}
|
|
if len(refs) != 1 || refs[0].Status != history.DocumentStatusListed || refs[0].Chars != 0 {
|
|
t.Fatalf("结论 = %+v, want listed/chars=0", refs)
|
|
}
|
|
}
|
|
|
|
// TestResolveDocuments_TruncatedGivesCursor 钉住截断诚实（design/24 §4.3）：
|
|
// 截断时清单行给出续读游标（已注入字符数），绝不静默截断。
|
|
func TestResolveDocuments_TruncatedGivesCursor(t *testing.T) {
|
|
t.Parallel()
|
|
res := &fakeDocumentResolver{
|
|
refs: map[string]history.DocumentRef{docA: {Status: history.DocumentStatusExtracted, Chars: 700, Truncated: true}},
|
|
bodies: map[string]string{docA: "前 700 字符"},
|
|
}
|
|
l := New(Options{DocumentResolver: res})
|
|
out, _ := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docA, "text/plain", 99)})
|
|
text := messageTextOf(out[0])
|
|
for _, want := range []string{"已截断", "offset=700"} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("注入块 %q 缺 %q", text, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestResolveDocuments_UnreadableIsHonest 钉住失败可见（design/24 §4.4）：
|
|
// unreadable 结论 + 诚实原因文案 + 清单仍在，且解析不产生 error（回合继续）。
|
|
func TestResolveDocuments_UnreadableIsHonest(t *testing.T) {
|
|
t.Parallel()
|
|
res := &fakeDocumentResolver{
|
|
refs: map[string]history.DocumentRef{docB: {Status: history.DocumentStatusUnreadable}},
|
|
bodies: map[string]string{docB: "无法抽取正文：无文本层"},
|
|
}
|
|
l := New(Options{DocumentResolver: res})
|
|
out, refs := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docB, "application/pdf", 1024)})
|
|
text := messageTextOf(out[0])
|
|
if !strings.Contains(text, documentListPrefix+docB) || !strings.Contains(text, documentNoteUnreadable) {
|
|
t.Fatalf("注入块 = %q, want 清单行标不可读", text)
|
|
}
|
|
if !strings.Contains(text, "无文本层") {
|
|
t.Fatalf("注入块缺诚实原因: %q", text)
|
|
}
|
|
if len(refs) != 1 || refs[0].Status != history.DocumentStatusUnreadable {
|
|
t.Fatalf("结论 = %+v, want unreadable", refs)
|
|
}
|
|
}
|
|
|
|
// TestResolveDocuments_ResolverErrorFallsBackHonestly 钉住解析器整体失败时的
|
|
// 降级：诚实兜底原因 + 清单仍在 + unreadable 状态，绝不空上下文。
|
|
func TestResolveDocuments_ResolverErrorFallsBackHonestly(t *testing.T) {
|
|
t.Parallel()
|
|
l := New(Options{DocumentResolver: &fakeDocumentResolver{err: errDocResolve}})
|
|
out, refs := l.resolveDocuments(context.Background(), []model.Message{documentMessage(docA, "text/plain", 10)})
|
|
text := messageTextOf(out[0])
|
|
if !strings.Contains(text, documentListPrefix+docA) || !strings.Contains(text, documentUnreadableBody) {
|
|
t.Fatalf("注入块 = %q, want 清单行 + 诚实兜底", text)
|
|
}
|
|
if len(refs) != 1 || refs[0].Status != history.DocumentStatusUnreadable {
|
|
t.Fatalf("结论 = %+v, want unreadable", refs)
|
|
}
|
|
}
|
|
|
|
// TestResolveDocuments_NeutralizesNonUserParts 钉住红线（design/11 §10.2）：文档
|
|
// 部件落到非用户回合上是编程错误，但绝不静默丢弃 —— 它照样被中和成清单块。
|
|
func TestResolveDocuments_NeutralizesNonUserParts(t *testing.T) {
|
|
t.Parallel()
|
|
l := New(Options{})
|
|
in := []model.Message{model.DocumentMessage(model.RoleAssistant, "", model.Document{Name: docA, MIME: "text/plain"})}
|
|
out, refs := l.resolveDocuments(context.Background(), in)
|
|
if countDocumentParts(out[0]) != 0 || !strings.Contains(messageTextOf(out[0]), documentListPrefix+docA) {
|
|
t.Fatalf("非用户回合的文档部件未被中和: %+v", out[0])
|
|
}
|
|
if len(refs) != 1 {
|
|
t.Fatalf("结论数 = %d, want 1", len(refs))
|
|
}
|
|
}
|
|
|
|
// TestRun_DocumentPartsNeverReachEncoder 是同名不变式的负例：无论解析器缺席、
|
|
// 解析器报错还是解析成功，喂给模型的每条请求里都不得出现文档部件（S2 的编码器
|
|
// 对文档 fail-loud，漏一个就是一次 400）。
|
|
func TestRun_DocumentPartsNeverReachEncoder(t *testing.T) {
|
|
t.Parallel()
|
|
cases := []struct {
|
|
name string
|
|
resolver DocumentResolver
|
|
}{
|
|
{name: "无解析器", resolver: nil},
|
|
{name: "解析器报错", resolver: &fakeDocumentResolver{err: errDocResolve}},
|
|
{name: "解析成功", resolver: &fakeDocumentResolver{
|
|
refs: map[string]history.DocumentRef{docA: {Status: history.DocumentStatusExtracted, Chars: 3}},
|
|
bodies: map[string]string{docA: "正文"},
|
|
}},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
ad := &mockAdapter{chat: []mgResp{{resp: model.Response{Model: "m", Content: "Done."}}}}
|
|
l := New(Options{Agent: newTestAgent(t, ad, nil), MaxRounds: 4, DocumentResolver: c.resolver})
|
|
|
|
in := []model.Message{
|
|
documentMessage(docA, "text/plain", 12),
|
|
model.DocumentMessage(model.RoleAssistant, "", model.Document{Name: docB, MIME: "text/plain"}),
|
|
}
|
|
if _, err := l.Run(context.Background(), "sess-doc", in); err != nil {
|
|
t.Fatalf("Run: %v", err)
|
|
}
|
|
assertNoDocumentParts(t, ad.reqs)
|
|
})
|
|
}
|
|
}
|
|
|
|
// assertNoDocumentParts 断言每条模型请求都不含文档部件，并且能通过编码器入口
|
|
// 的同一道校验（model.ValidateDocumentParts）。
|
|
func assertNoDocumentParts(t *testing.T, reqs []model.Request) {
|
|
t.Helper()
|
|
if len(reqs) == 0 {
|
|
t.Fatal("没有记录到任何模型请求")
|
|
}
|
|
for _, req := range reqs {
|
|
for _, m := range req.Messages {
|
|
if countDocumentParts(m) != 0 {
|
|
t.Fatalf("文档部件抵达了模型: %+v", m)
|
|
}
|
|
}
|
|
if err := model.ValidateDocumentParts(req.Messages); err != nil {
|
|
t.Fatalf("编码器边界校验失败: %v", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestRun_RecordsDocumentOutcomesOnResult 钉住 status 的回传出口：结论随 Result
|
|
// 返回（按存储名去重、保留首次出现顺序），使客户端能看到每份附件的结果。
|
|
func TestRun_RecordsDocumentOutcomesOnResult(t *testing.T) {
|
|
t.Parallel()
|
|
res := &fakeDocumentResolver{
|
|
refs: map[string]history.DocumentRef{
|
|
docA: {Status: history.DocumentStatusExtracted, Chars: 3},
|
|
docB: {Status: history.DocumentStatusUnreadable},
|
|
},
|
|
bodies: map[string]string{docA: "正文", docB: "无法抽取正文：不存在"},
|
|
}
|
|
ad := &mockAdapter{chat: []mgResp{{resp: model.Response{Model: "m", Content: "Done."}}}}
|
|
l := New(Options{Agent: newTestAgent(t, ad, nil), MaxRounds: 4, DocumentResolver: res})
|
|
|
|
in := []model.Message{documentMessage(docA, "text/plain", 12), documentMessage(docB, "text/plain", 8)}
|
|
out, err := l.Run(context.Background(), "sess-doc", in)
|
|
if err != nil {
|
|
t.Fatalf("Run: %v", err)
|
|
}
|
|
if len(out.Documents) != 2 {
|
|
t.Fatalf("Documents = %+v, want 2 条（去重后）", out.Documents)
|
|
}
|
|
if out.Documents[0].Name != docA || out.Documents[0].Status != history.DocumentStatusExtracted {
|
|
t.Fatalf("Documents[0] = %+v, want %s/extracted", out.Documents[0], docA)
|
|
}
|
|
if out.Documents[1].Name != docB || out.Documents[1].Status != history.DocumentStatusUnreadable {
|
|
t.Fatalf("Documents[1] = %+v, want %s/unreadable", out.Documents[1], docB)
|
|
}
|
|
}
|
|
|
|
// TestUserMessageKey_IncludesDocumentRef 是 design/24 §9.5 移交的回归：纯文档
|
|
// 用户消息（客户端「超限走文件」发的就是它）必须在比较键里有内容，否则续接判定
|
|
// 会把附件回合当成「没有可比内容」而漏判。
|
|
func TestUserMessageKey_IncludesDocumentRef(t *testing.T) {
|
|
t.Parallel()
|
|
docOnly := documentMessage(docA, "text/plain", 10)
|
|
if key := userMessageKey(docOnly); key == "" {
|
|
t.Fatal("纯文档消息的比较键为空")
|
|
}
|
|
if lastUserMessageKey([]model.Message{docOnly}) == "" {
|
|
t.Fatal("纯文档回合的最后一条用户消息键为空")
|
|
}
|
|
if userMessageKey(docOnly) == userMessageKey(documentMessage(docB, "text/plain", 10)) {
|
|
t.Fatal("两份不同附件得到了同一个比较键")
|
|
}
|
|
}
|
|
|
|
// TestEstimateContextTokens_CountsDocuments 是 design/24 §9.5 移交的回归：文档
|
|
// 部件必须在 token 估算里占分量，否则纯文档回合被估成 0 token，在压缩里等同于
|
|
// 空内容。
|
|
func TestEstimateContextTokens_CountsDocuments(t *testing.T) {
|
|
t.Parallel()
|
|
small := []model.Message{documentMessage(docA, "text/plain", 1024)}
|
|
big := []model.Message{documentMessage(docA, "text/plain", 4<<20)}
|
|
if got := estimateContextTokens(small); got <= messageOverheadTokens {
|
|
t.Fatalf("纯文档回合估算 = %d, want > %d（文档部件不是免费的）", got, messageOverheadTokens)
|
|
}
|
|
if estimateContextTokens(big) <= estimateContextTokens(small) {
|
|
t.Fatalf("大附件估算未变大：%d vs %d", estimateContextTokens(big), estimateContextTokens(small))
|
|
}
|
|
if got := documentPartTokens(model.Content{Type: model.ContentTypeDocument}); got != documentMinTokens {
|
|
t.Fatalf("documentPartTokens(未知字节数) = %d, want %d", got, documentMinTokens)
|
|
}
|
|
if got := documentPartTokens(model.Content{Type: model.ContentTypeDocument, DocumentSize: documentInjectedCharsCap * 4}) /
|
|
1; got != documentInjectedCharsCap/charsPerToken {
|
|
t.Fatalf("documentPartTokens(超大附件) = %d, want 按注入预算封顶 %d", got, documentInjectedCharsCap/charsPerToken)
|
|
}
|
|
}
|
|
|
|
// TestFormatDocumentSize 钉住清单行里的展示口径（只用于人读，不做任何判定）。
|
|
func TestFormatDocumentSize(t *testing.T) {
|
|
t.Parallel()
|
|
cases := []struct {
|
|
in int64
|
|
want string
|
|
}{
|
|
{0, "0 B"},
|
|
{512, "512 B"},
|
|
{2048, "2.0 KiB"},
|
|
{3 << 20, "3.0 MiB"},
|
|
{-1, "0 B"},
|
|
}
|
|
for _, c := range cases {
|
|
if got := formatDocumentSize(c.in); got != c.want {
|
|
t.Fatalf("formatDocumentSize(%d) = %q, want %q", c.in, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestDocumentBlockShape 钉住注入块的稳定形状（清单行在前、正文换行在后），
|
|
// 因为客户端渲染与「清单恒在」都依赖它。
|
|
func TestDocumentBlockShape(t *testing.T) {
|
|
t.Parallel()
|
|
part := model.Content{Type: model.ContentTypeDocument, DocumentName: docA, DocumentMIME: "text/plain", DocumentSize: 2048}
|
|
block := documentBlock(part, history.DocumentRef{Name: docA, Status: history.DocumentStatusExtracted, Chars: 5}, "正文")
|
|
want := fmt.Sprintf("%s%s · text/plain · 2.0 KiB · 已注入正文 5 字符]\n正文", documentListPrefix, docA)
|
|
if block != want {
|
|
t.Fatalf("documentBlock = %q, want %q", block, want)
|
|
}
|
|
empty := documentBlock(model.Content{Type: model.ContentTypeDocument, DocumentName: docA}, history.DocumentRef{Status: history.DocumentStatusListed}, "")
|
|
if strings.Contains(empty, "\n") || !strings.Contains(empty, documentNoteListed) {
|
|
t.Fatalf("无正文时不该有换行: %q", empty)
|
|
}
|
|
}
|