AskaEth
2a43035f5d
- 抽共享存储核心：internal/runtime/documents/store.go（22 项白名单 + 字节嗅探唯一裁决 + 内容寻址 sha256[:32]+ext + 去重 + 限额 + 路径双检）+ store_meta.go（documents 表唯一写入实现）。
- gateway 改为调用 store，**对外行为零变化**（机器证明：既有 documents_test/chat_documents_test/chat_documents_bounds_test 三个文件 0 行改动且全绿；状态码/响应字段/错误码/审计动作名/限额文档口径全部逐字保留；maxChatDocuments 改为引用 documents.MaxPerMessage）。
- IM 入站附件摄取（channelbridge）：文件类媒体 → store.Save → 挂 model.DocumentMessage(RoleUser) 部件 ⇒ 后续 resolveDocuments 自动抽取/注入/清单；挂载点在「会话解析 + 黑白名单 + 最小打扰」三重闸门之后（被拦消息绝不在盘上留文件）；**不支持的格式不静默**（诚实提示行 + 带来源日志，只记 name/mime/size，名字做展示净化防伪造行）；图片/语音保持既有链路。
- 主会话独立验收：隔离 worktree（M1 的 12 文件）build/vet 绿、全树 59 包 0 FAIL、双平台交叉编译绿、12 条 Inbound 用例 PASS；变异独立复现（关闭 IM 摄取 ⇒ 5 条红，含「落盘文件数 = [] want 1」）。
- ⚠️ 真机边界（需裁决）：现网四个适配器上报的文件媒体**只带 URL**（telegram FileID / qq-obv11 file|url / weixin·kook CDN URL），Media.Data 恒空 ⇒ 本切片只摄**内联字节**，URL 一律「诚实提示，不代取远程」（避免第三份 SSRF 实现）。真机要让「她读到 IM 文件」，需各适配器入站时下载并内联（凭据在各自渠道里，推荐）或新增共享受守卫抓取器。
- 另登记：IM 附件暂未写 chat_messages.document 列（二手回合看不到引用，建议并入 S7）；channelbridge 既有测试夹具在 -race 下有数据竞争（stubSessions.GetOrCreate，与本次无关）。
396 lines
14 KiB
Go
396 lines
14 KiB
Go
package channelbridge
|
||
|
||
// 本文件锁定 **IM 入站附件摄取**（design/24 §3 / §4.4；切片 M1）：
|
||
//
|
||
// - 文件类媒体 → 共享文档存储落库 → 装配期抽取 → **模型可见视图里出现清单行与
|
||
// 正文**（真 store + 真抽取器 + 真循环 + 假模型，端到端）；
|
||
// - 不支持的格式**不落库**，但模型收到一行诚实提示、日志留一条带来源的记录，
|
||
// 且日志**绝不记正文**；
|
||
// - 渠道只给远程引用（无内联字节）时同样诚实，不静默；
|
||
// - 摄取在**闸门之后**：被拉黑/被隔离的消息绝不在盘上留下文件，其附件也只进
|
||
// 隔离会话，不镜像到主人主会话；
|
||
// - 只有附件的消息（无文本）不再被判成「没有内容」而丢弃；
|
||
// - 图片类媒体保持现状（不走摄取，也不落库）。
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"log/slog"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
|
||
"git.yeij.top/AskaEth/Cyrene_v2/internal/channel/adapter"
|
||
"git.yeij.top/AskaEth/Cyrene_v2/internal/channel/session"
|
||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/agent"
|
||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/documents"
|
||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/loop"
|
||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||
"git.yeij.top/AskaEth/Cyrene_v2/internal/user"
|
||
)
|
||
|
||
// stubModelAdapter 是最小的 model.Adapter：记下每次补全请求的**模型可见视图**
|
||
// （即装配期 resolveDocuments 替换之后的形态），并回一条固定文本。
|
||
type stubModelAdapter struct {
|
||
mu sync.Mutex
|
||
seen [][]model.Message
|
||
}
|
||
|
||
func (a *stubModelAdapter) Name() string { return "stub" }
|
||
|
||
func (a *stubModelAdapter) Chat(_ context.Context, req model.Request, _ model.Options) (model.Response, error) {
|
||
a.mu.Lock()
|
||
a.seen = append(a.seen, req.Messages)
|
||
a.mu.Unlock()
|
||
return model.Response{Model: "stub", Content: "读完了", StopReason: "stop"}, nil
|
||
}
|
||
|
||
func (a *stubModelAdapter) ChatStream(ctx context.Context, req model.Request, _ func(model.StreamEvent)) (model.Response, error) {
|
||
return a.Chat(ctx, req, model.Options{})
|
||
}
|
||
|
||
// visibleText 返回模型最后看到的全部消息文本（含系统与用户回合）。
|
||
func (a *stubModelAdapter) visibleText() string {
|
||
a.mu.Lock()
|
||
defer a.mu.Unlock()
|
||
if len(a.seen) == 0 {
|
||
return ""
|
||
}
|
||
var b strings.Builder
|
||
for _, m := range a.seen[len(a.seen)-1] {
|
||
for _, c := range m.Content {
|
||
b.WriteString(c.Text)
|
||
b.WriteString("\n")
|
||
}
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// calls 返回模型被调用的次数（0 = 循环根本没跑）。
|
||
func (a *stubModelAdapter) calls() int {
|
||
a.mu.Lock()
|
||
defer a.mu.Unlock()
|
||
return len(a.seen)
|
||
}
|
||
|
||
// safeBuffer 是并发安全的日志缓冲（桥接器的入站路径可能在 goroutine 里写日志）。
|
||
type safeBuffer struct {
|
||
mu sync.Mutex
|
||
buf bytes.Buffer
|
||
}
|
||
|
||
func (b *safeBuffer) Write(p []byte) (int, error) {
|
||
b.mu.Lock()
|
||
defer b.mu.Unlock()
|
||
return b.buf.Write(p)
|
||
}
|
||
|
||
func (b *safeBuffer) String() string {
|
||
b.mu.Lock()
|
||
defer b.mu.Unlock()
|
||
return b.buf.String()
|
||
}
|
||
|
||
// ingestEnv 是入站附件端到端用例的装配：**真 store + 真抽取器 + 真循环 + 假模型**，
|
||
// 因此模型看到的清单行与正文就是生产路径装配出来的那一份。
|
||
type ingestEnv struct {
|
||
bridge *Bridge
|
||
model *stubModelAdapter
|
||
adapter *fakeAdapter
|
||
sessions *stubSessions
|
||
uploads string
|
||
logs *safeBuffer
|
||
}
|
||
|
||
// newIngestEnv 装配上述端到端环境。cfg 给出入站附件的落盘配置（未给 UploadsDir 时
|
||
// 落到临时目录）；cfg 为零值时即「未接线」用例（跳过 SetDocumentStore）。
|
||
func newIngestEnv(t *testing.T, cfg documents.StoreConfig, wireStore bool) *ingestEnv {
|
||
t.Helper()
|
||
dataDir := t.TempDir()
|
||
uploads := cfg.UploadsDir
|
||
if uploads == "" {
|
||
uploads = filepath.Join(dataDir, "uploads", "documents")
|
||
}
|
||
cfg.UploadsDir = uploads
|
||
logs := &safeBuffer{}
|
||
log := slog.New(slog.NewTextHandler(logs, nil))
|
||
|
||
modelAdapter := &stubModelAdapter{}
|
||
ag, err := agent.New("测试系统提示词", nil, nil, agent.WithModel(modelAdapter, nil))
|
||
if err != nil {
|
||
t.Fatalf("构造 agent: %v", err)
|
||
}
|
||
resolver := documents.NewResolver(documents.Config{DataDir: dataDir, UploadsDir: uploads, Log: log})
|
||
runner := loop.New(loop.Options{
|
||
Agent: ag, DocumentResolver: resolver, MaxRounds: 4, Log: log,
|
||
})
|
||
sess := &stubSessions{sess: session.Session{SessionID: "sess-1", Owner: true, MemWritable: true}}
|
||
opts := Options{
|
||
Log: log, Loop: runner, Sessions: sess, Owner: testOwnerProfile("telegram", "100"),
|
||
}
|
||
if wireStore {
|
||
opts.DocumentStore = cfg
|
||
}
|
||
b, err := New(opts)
|
||
if err != nil {
|
||
t.Fatalf("New: %v", err)
|
||
}
|
||
platform := &fakeAdapter{name: "telegram"}
|
||
b.Add(platform)
|
||
return &ingestEnv{
|
||
bridge: b, model: modelAdapter, adapter: platform,
|
||
sessions: sess, uploads: uploads, logs: logs,
|
||
}
|
||
}
|
||
|
||
// fileMessage 构造一条主人私聊消息，附带一个文件类媒体。
|
||
func fileMessage(text, name string, data []byte, mime string) adapter.UnifiedMessage {
|
||
msg := directMessage()
|
||
msg.Content = text
|
||
msg.ContentType = adapter.ContentText
|
||
if text == "" {
|
||
msg.ContentType = adapter.ContentFile
|
||
}
|
||
msg.Media = []adapter.Media{{
|
||
Kind: adapter.MediaFile, Name: name, Size: int64(len(data)), MIMEType: mime, Data: data,
|
||
}}
|
||
return msg
|
||
}
|
||
|
||
// docDirNames 返回文档落盘目录里的条目名（目录不存在时返回 nil）。
|
||
func docDirNames(t *testing.T, dir string) []string {
|
||
t.Helper()
|
||
entries, err := os.ReadDir(dir)
|
||
if err != nil {
|
||
if os.IsNotExist(err) {
|
||
return nil
|
||
}
|
||
t.Fatalf("读文档目录: %v", err)
|
||
}
|
||
out := make([]string, 0, len(entries))
|
||
for _, e := range entries {
|
||
out = append(out, e.Name())
|
||
}
|
||
return out
|
||
}
|
||
|
||
// TestInboundFileMediaIngestedAndVisibleToModel 覆盖验收第 3 条：一条带文件媒体的
|
||
// 入站消息 → 落库 → 抽取 → **模型可见视图里出现该附件的清单行与正文**。
|
||
func TestInboundFileMediaIngestedAndVisibleToModel(t *testing.T) {
|
||
t.Parallel()
|
||
env := newIngestEnv(t, documents.StoreConfig{}, true)
|
||
body := "第一行：昔涟的测试附件\n第二行：正文必须真的进上下文\n"
|
||
msg := fileMessage("看看这个", "notes.txt", []byte(body), "text/plain")
|
||
|
||
env.bridge.process(context.Background(), msg)
|
||
|
||
names := docDirNames(t, env.uploads)
|
||
if len(names) != 1 {
|
||
t.Fatalf("落盘文件数 = %v, want 1（附件必须落库）", names)
|
||
}
|
||
stored, err := os.ReadFile(filepath.Join(env.uploads, names[0]))
|
||
if err != nil {
|
||
t.Fatalf("读落盘文件: %v", err)
|
||
}
|
||
if string(stored) != body {
|
||
t.Fatalf("落盘内容 = %q, want %q", stored, body)
|
||
}
|
||
if !strings.HasSuffix(names[0], ".txt") || len(names[0]) != documents.HashHexLen+len(".txt") {
|
||
t.Fatalf("落盘名 = %q, want <32 位十六进制>.txt（内容寻址）", names[0])
|
||
}
|
||
|
||
visible := env.model.visibleText()
|
||
if !strings.Contains(visible, "[附件 "+names[0]+" · ") {
|
||
t.Fatalf("模型可见视图里没有附件清单行：%q", visible)
|
||
}
|
||
if !strings.Contains(visible, "第二行：正文必须真的进上下文") {
|
||
t.Fatalf("模型可见视图里没有附件正文：%q", visible)
|
||
}
|
||
if !strings.Contains(visible, "看看这个") {
|
||
t.Fatalf("模型可见视图里丢了用户文本：%q", visible)
|
||
}
|
||
if log := env.logs.String(); !strings.Contains(log, "inbound attachment stored") ||
|
||
!strings.Contains(log, "source=services/channelbridge") {
|
||
t.Fatalf("落库日志缺少来源或内容：%q", log)
|
||
}
|
||
}
|
||
|
||
// TestInboundUnsupportedFileIsHonestAndNotStored 覆盖验收第 4 条：不支持的格式
|
||
// 不落库，但模型知道「有附件读不到」，且日志带来源、只记 name/mime/size（绝不记正文）。
|
||
func TestInboundUnsupportedFileIsHonestAndNotStored(t *testing.T) {
|
||
t.Parallel()
|
||
cases := []struct {
|
||
label string
|
||
name string
|
||
mime string
|
||
data []byte
|
||
}{
|
||
{"扩展名不在白名单", "tool.py", "text/x-python", []byte("print('秘密内容')\n")},
|
||
{"内容被嗅探否决", "lie.txt", "text/plain", append([]byte("\x89PNG\r\n\x1a\n"), bytes.Repeat([]byte{0x00}, 32)...)},
|
||
}
|
||
for _, tc := range cases {
|
||
env := newIngestEnv(t, documents.StoreConfig{}, true)
|
||
env.bridge.process(context.Background(), fileMessage("跑一下", tc.name, tc.data, tc.mime))
|
||
|
||
if names := docDirNames(t, env.uploads); len(names) != 0 {
|
||
t.Fatalf("%s: 被拒的附件仍落库 %v", tc.label, names)
|
||
}
|
||
visible := env.model.visibleText()
|
||
if !strings.Contains(visible, tc.name) || !strings.Contains(visible, "无法读取") {
|
||
t.Fatalf("%s: 模型可见视图里没有诚实提示行：%q", tc.label, visible)
|
||
}
|
||
if env.model.calls() == 0 {
|
||
t.Fatalf("%s: 回合没有跑到模型", tc.label)
|
||
}
|
||
log := env.logs.String()
|
||
if !strings.Contains(log, "inbound attachment rejected") || !strings.Contains(log, tc.name) {
|
||
t.Fatalf("%s: 缺少带来源的拒绝日志：%q", tc.label, log)
|
||
}
|
||
if strings.Contains(log, "秘密内容") {
|
||
t.Fatalf("%s: 日志里出现了附件正文（红线：只记 name/mime/size）", tc.label)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestInboundFileMediaBlacklistedNotIngested 覆盖验收第 5 条的一半：被黑白名单拉黑的
|
||
// 群/陌生人消息，附件**不落库**、也不驱动循环（摄取在闸门之后）。
|
||
func TestInboundFileMediaBlacklistedNotIngested(t *testing.T) {
|
||
t.Parallel()
|
||
uploads := filepath.Join(t.TempDir(), "uploads", "documents")
|
||
chat := &stubChatter{res: &loop.Result{Text: "不该回复"}}
|
||
sess := &stubSessions{sess: session.Session{SessionID: "sess-g", IsGroup: true, Owner: false, MemWritable: false}}
|
||
var rejected []string
|
||
b, err := New(Options{
|
||
Loop: chat, Sessions: sess, Owner: &user.OwnerProfile{}, History: &captureHistory{},
|
||
DocumentStore: documents.StoreConfig{UploadsDir: uploads},
|
||
OnRejected: func(_ adapter.UnifiedMessage, reason string) { rejected = append(rejected, reason) },
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("New: %v", err)
|
||
}
|
||
ad := &fakeAdapter{name: "telegram", cfg: map[string]any{
|
||
"bot_token": "configured", "denyGroups": []string{"-100"},
|
||
}}
|
||
b.Add(ad)
|
||
|
||
msg := adapter.UnifiedMessage{
|
||
Platform: "telegram", ChannelID: "-100", ChannelType: adapter.ChannelGroup,
|
||
GroupID: "-100", SenderID: "9", SenderName: "陌生人",
|
||
Sender: adapter.Actor{Role: adapter.RoleMember}, Content: "给你个文件",
|
||
ContentType: adapter.ContentText,
|
||
Media: []adapter.Media{{
|
||
Kind: adapter.MediaFile, Name: "stranger.txt", Size: 6, MIMEType: "text/plain", Data: []byte("你好\n"),
|
||
}},
|
||
}
|
||
b.process(context.Background(), msg)
|
||
|
||
if names := docDirNames(t, uploads); len(names) != 0 {
|
||
t.Fatalf("被拉黑的消息把附件落库了：%v", names)
|
||
}
|
||
if chat.sessionID != "" || chat.messages != nil {
|
||
t.Fatalf("被拉黑的消息仍驱动了循环：session=%q", chat.sessionID)
|
||
}
|
||
if len(rejected) != 1 || rejected[0] != "group_denied" {
|
||
t.Fatalf("拒绝原因 = %v, want [group_denied]", rejected)
|
||
}
|
||
}
|
||
|
||
// TestInboundGroupFileStaysInIsolatedSession 覆盖验收第 5 条的另一半：第三方（群聊）
|
||
// 消息的附件只随该消息进入**隔离会话**，绝不镜像到主人主会话（沿用既有策略测试写法）。
|
||
func TestInboundGroupFileStaysInIsolatedSession(t *testing.T) {
|
||
t.Parallel()
|
||
uploads := filepath.Join(t.TempDir(), "uploads", "documents")
|
||
chat := &stubChatter{res: &loop.Result{Text: "收到"}}
|
||
sess := &stubSessions{sess: session.Session{SessionID: "sess-g", IsGroup: true, Owner: false, MemWritable: false}}
|
||
hist := &captureHistory{}
|
||
b, err := New(Options{
|
||
Loop: chat, Sessions: sess, Owner: &user.OwnerProfile{}, History: hist,
|
||
DocumentStore: documents.StoreConfig{UploadsDir: uploads},
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("New: %v", err)
|
||
}
|
||
|
||
// 被点名的群消息 → 回复（design/12 §6 策略二），因此会真的走完流水线。
|
||
msg := adapter.UnifiedMessage{
|
||
Platform: "telegram", ChannelID: "-20", ChannelType: adapter.ChannelGroup,
|
||
GroupID: "-20", GroupName: "Gang", SenderID: "9", SenderName: "张三",
|
||
Sender: adapter.Actor{Role: adapter.RoleMember}, Mentions: []adapter.Mention{{UserID: "9"}},
|
||
Content: "看下这个", ContentType: adapter.ContentText,
|
||
Media: []adapter.Media{{
|
||
Kind: adapter.MediaFile, Name: "third.txt", Size: 4, MIMEType: "text/plain", Data: []byte("hi\n"),
|
||
}},
|
||
}
|
||
b.process(context.Background(), msg)
|
||
|
||
if names := docDirNames(t, uploads); len(names) != 1 {
|
||
t.Fatalf("群聊附件落盘文件数 = %v, want 1（第三方附件同样走共享存储）", names)
|
||
}
|
||
if chat.sessionID != "sess-g" {
|
||
t.Fatalf("循环会话 = %q, want 隔离会话 sess-g", chat.sessionID)
|
||
}
|
||
recs := hist.records()
|
||
if len(recs) != 2 {
|
||
t.Fatalf("隔离会话落库条数 = %d, want 2", len(recs))
|
||
}
|
||
for i, rec := range recs {
|
||
if rec.SessionID != "sess-g" {
|
||
t.Fatalf("record[%d] SessionID = %q, want 隔离会话（绝不镜像到主人主会话）", i, rec.SessionID)
|
||
}
|
||
}
|
||
// 用户消息在循环里带着文档部件（图片/文档只进用户回合）。
|
||
var docParts int
|
||
for _, m := range chat.messages {
|
||
if m.Role == model.RoleSystem {
|
||
continue
|
||
}
|
||
for _, c := range m.Content {
|
||
if c.Type == model.ContentTypeDocument {
|
||
docParts++
|
||
}
|
||
}
|
||
}
|
||
if docParts != 1 {
|
||
t.Fatalf("用户消息里的文档部件数 = %d, want 1", docParts)
|
||
}
|
||
}
|
||
|
||
// TestInboundFileOnlyMessageReachesLoop 钉住「只有附件、没有文本」的消息不再被判成
|
||
// 没有内容而丢弃，且那条消息**不塞空文本部件**（provider 会拒空文本块）。
|
||
func TestInboundFileOnlyMessageReachesLoop(t *testing.T) {
|
||
t.Parallel()
|
||
env := newIngestEnv(t, documents.StoreConfig{}, true)
|
||
msg := fileMessage("", "only.md", []byte("# 只有附件\n正文\n"), "text/markdown")
|
||
|
||
env.bridge.process(context.Background(), msg)
|
||
|
||
if env.model.calls() == 0 {
|
||
t.Fatal("纯附件消息没有驱动循环")
|
||
}
|
||
names := docDirNames(t, env.uploads)
|
||
if len(names) != 1 {
|
||
t.Fatalf("落盘文件数 = %v, want 1", names)
|
||
}
|
||
visible := env.model.visibleText()
|
||
if !strings.Contains(visible, "[附件 "+names[0]+" · ") || !strings.Contains(visible, "# 只有附件") {
|
||
t.Fatalf("模型可见视图里没有清单行或正文：%q", visible)
|
||
}
|
||
env.model.mu.Lock()
|
||
defer env.model.mu.Unlock()
|
||
if len(env.model.seen) == 0 {
|
||
t.Fatal("没有记录到模型请求")
|
||
}
|
||
for _, m := range env.model.seen[0] {
|
||
if m.Role != model.RoleUser {
|
||
continue
|
||
}
|
||
for _, c := range m.Content {
|
||
if c.Type == model.ContentTypeText && strings.TrimSpace(c.Text) == "" {
|
||
t.Fatalf("纯附件消息里塞了空文本部件：%+v", m.Content)
|
||
}
|
||
}
|
||
}
|
||
}
|