From d947ec41a9a360d1f8f8059a0535f34dce54181f Mon Sep 17 00:00:00 2001 From: Aska Date: Sun, 6 Sep 2026 06:15:18 +0800 Subject: [PATCH] =?UTF-8?q?feat(memory):=20M4=20=E8=AE=B0=E5=BF=86?= =?UTF-8?q?=E5=AE=8C=E5=96=84=EF=BC=88=E4=B8=89=E7=B1=BB=E4=BD=9C=E7=94=A8?= =?UTF-8?q?=E5=9F=9F=E8=AE=BF=E9=97=AE=E8=A7=84=E5=88=99+=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E7=B4=A2=E5=BC=95=E5=88=86=E5=B1=82+LLM=E5=BC=82?= =?UTF-8?q?=E6=AD=A5=E6=8F=90=E5=8F=96+=E8=A1=B0=E5=87=8F=E6=BA=AF?= =?UTF-8?q?=E6=BA=90+=E5=90=91=E9=87=8F=E6=8E=A5=E5=85=A5=EF=BC=8C?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E6=97=A2=E6=9C=8957=E7=94=A8=E4=BE=8B?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/runtime/memory/decay.go | 23 ++ internal/runtime/memory/decay_test.go | 79 ++++++ internal/runtime/memory/extract.go | 18 +- internal/runtime/memory/llmextractor.go | 227 +++++++++++++++++ internal/runtime/memory/llmextractor_test.go | 240 ++++++++++++++++++ internal/runtime/memory/retrieve.go | 135 ++++++++-- internal/runtime/memory/retrieve_m4_test.go | 249 +++++++++++++++++++ internal/runtime/memory/rules.go | 148 +++++++++++ internal/runtime/memory/rules_test.go | 217 ++++++++++++++++ internal/runtime/memory/store.go | 66 +++-- 10 files changed, 1349 insertions(+), 53 deletions(-) create mode 100644 internal/runtime/memory/decay.go create mode 100644 internal/runtime/memory/decay_test.go create mode 100644 internal/runtime/memory/llmextractor.go create mode 100644 internal/runtime/memory/llmextractor_test.go create mode 100644 internal/runtime/memory/retrieve_m4_test.go create mode 100644 internal/runtime/memory/rules.go create mode 100644 internal/runtime/memory/rules_test.go diff --git a/internal/runtime/memory/decay.go b/internal/runtime/memory/decay.go new file mode 100644 index 0000000..d782283 --- /dev/null +++ b/internal/runtime/memory/decay.go @@ -0,0 +1,23 @@ +package memory + +import ( + "math" + "time" +) + +// decayFactor computes the decay multiplier for a single item: the stored +// decay_weight scaled by a time-decay that halves every recencyHalfLife days +// (design §2 衰减: 久未使用的记忆降权). A freshly touched item keeps its full +// decay_weight; an item unused for N half-lives is scaled to +// decay_weight * 0.5^N, so it ranks progressively lower unless it stays relevant. +// +// age is measured from UpdatedAt; a future timestamp (clock skew) is clamped to +// zero, so a just-recorded item is never scored above its base weight. +func (r *Retriever) decayFactor(it MemoryItem, now time.Time) float64 { + age := now.Sub(it.UpdatedAt) + if age < 0 { + age = 0 + } + halfLives := float64(age) / float64(r.recencyHalfLife) + return it.DecayWeight * math.Pow(0.5, halfLives) +} diff --git a/internal/runtime/memory/decay_test.go b/internal/runtime/memory/decay_test.go new file mode 100644 index 0000000..8d8f95f --- /dev/null +++ b/internal/runtime/memory/decay_test.go @@ -0,0 +1,79 @@ +package memory + +import ( + "context" + "math" + "testing" + "time" +) + +func TestRetriever_DecayFactor_久未使用降权(t *testing.T) { + t.Parallel() + r := NewRetriever(newTestStore(t)) + now := time.Now().UTC() + + tests := []struct { + name string + updatedAt time.Time + decayWeight float64 + want float64 + }{ + {name: "fresh keeps full weight", updatedAt: now, decayWeight: 1.0, want: 1.0}, + // Default half-life is 30 days: 60 days ago = 2 half-lives = 2^-2. + {name: "two half-lives halves twice", updatedAt: now.Add(-60 * 24 * time.Hour), decayWeight: 1.0, want: 0.25}, + // Lower base decay_weight scales the whole curve down. + {name: "low decay_weight scales down", updatedAt: now, decayWeight: 0.5, want: 0.5}, + {name: "low weight plus age compounds", updatedAt: now.Add(-60 * 24 * time.Hour), decayWeight: 0.5, want: 0.125}, + // A future timestamp must not boost above the base weight. + {name: "future timestamp clamps to zero", updatedAt: now.Add(time.Hour), decayWeight: 1.0, want: 1.0}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := r.decayFactor(MemoryItem{ + DecayWeight: tt.decayWeight, + UpdatedAt: tt.updatedAt, + }, now) + if math.Abs(got-tt.want) > 1e-9 { + t.Fatalf("decayFactor = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRetriever_Retrieve_DecayWeight_降权(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + now := time.Now().UTC() + // A fresh item but with a near-zero decay_weight (design: 用户可削弱某记忆). + addItem(t, s, MemoryItem{ + PersonaID: "persona.a", Content: "低权重但新鲜", UpdatedAt: now, + DecayWeight: 0.1, Scope: ScopePersona, Kind: KindWorldEvent, + }) + // A slightly older item with a full decay_weight. + addItem(t, s, MemoryItem{ + PersonaID: "persona.a", Content: "权重高稍旧", UpdatedAt: now.Add(-time.Hour), + DecayWeight: 1.0, Scope: ScopePersona, Kind: KindWorldEvent, + }) + + r := NewRetriever(s) + // A blank query matches nothing, so ordering is driven by the decay factor + // (decay_weight × time decay) alone: the low-weight fresh item must fall + // below the full-weight slightly-older item. + results, err := r.Retrieve(ctx, "persona.a", "", 10) + if err != nil { + t.Fatalf("Retrieve: %v", err) + } + if len(results) != 2 { + t.Fatalf("got %d results, want 2", len(results)) + } + if got := results[0].Item.Content; got != "权重高稍旧" { + t.Fatalf("top result = %q, want the full-decay-weight item", got) + } + if got := results[1].Item.Content; got != "低权重但新鲜" { + t.Fatalf("second result = %q, want the low-decay-weight item", got) + } +} diff --git a/internal/runtime/memory/extract.go b/internal/runtime/memory/extract.go index 7677dba..4efeda0 100644 --- a/internal/runtime/memory/extract.go +++ b/internal/runtime/memory/extract.go @@ -2,17 +2,27 @@ package memory import ( "context" + "errors" "strings" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model" ) +var ( + // ErrNilAdapter is returned when an LLMExtractor is built or used without a + // model.Adapter to drive extraction. + ErrNilAdapter = errors.New("memory: nil model adapter") +) + // Extractor derives candidate memory items from a conversation (design §1). // -// The production path is an LLM-driven asynchronous extractor that denoises and -// deduplicates each turn; that is deferred to milestone M4, and its integration -// point is this interface. The M2 implementation is the rule-based -// ManualExtractor, which only records what the user explicitly asks to remember. +// Two implementations exist: +// +// - LLMExtractor: asks a model.Adapter to denoise and deduplicate each turn, +// running synchronously (Extract) or on a background goroutine +// (ExtractAsync). +// - ManualExtractor: deterministic, rule-based — it only records what the user +// explicitly asks to remember ("记住 X"). type Extractor interface { // Extract returns memory items worth persisting from conversation. ctx // controls cancellation; a canceled context returns an error. diff --git a/internal/runtime/memory/llmextractor.go b/internal/runtime/memory/llmextractor.go new file mode 100644 index 0000000..8b75c6f --- /dev/null +++ b/internal/runtime/memory/llmextractor.go @@ -0,0 +1,227 @@ +package memory + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model" +) + +// extractSystemPrompt tells the model.Adapter how to produce candidate memories: +// a JSON array of objects, one per memory worth persisting. Format is locked so +// parseCandidates can unmarshal it deterministically. +const extractSystemPrompt = `你是记忆提取器。从对话中提取值得长期记住的记忆项（用户偏好、习惯、约定、重要事件）。 +只输出一个 JSON 数组，不要任何其他文字。每个元素形如： +{"content":"记忆内容","scope":"user|persona|project","kind":"relationship|world_event","tags":["标签"],"confidence":0.9,"project_id":"可选项目id","source_conversation":"溯源"} +去噪去重：只保留值得长期记住的，忽略寒暄与临时信息。` + +// llmCandidate is the wire shape the extractor asks the LLM to emit: one JSON +// object per candidate memory. Scope/Kind default to the extractor's configured +// values when the model does not provide a valid one. +type llmCandidate struct { + Content string `json:"content"` + Scope string `json:"scope"` + Kind string `json:"kind"` + Tags []string `json:"tags"` + Confidence float64 `json:"confidence"` + ProjectID string `json:"project_id"` + Source string `json:"source_conversation"` +} + +// LLMExtractor implements Extractor by asking a model.Adapter to derive candidate +// memory items from a conversation (design §1 LLM 异步自动提取). It runs the LLM +// call synchronously in Extract and on a background goroutine in ExtractAsync. +// +// It is idempotent and safe for concurrent use: candidate content is normalized +// and deduplicated against a mutex-guarded seen set, so re-extracting the same +// memory never yields a second copy, and the same content appearing twice inside +// one conversation collapses to a single item. +type LLMExtractor struct { + adapter model.Adapter + personaID string + roleID string + scope Scope + kind Kind + + mu sync.Mutex + seen map[string]struct{} +} + +// NewLLMExtractor builds an LLMExtractor whose extracted items are authored by +// personaID/roleID and default to ScopePersona + KindRelationship when the model +// does not specify otherwise. A nil adapter is rejected at use time via +// ErrNilAdapter, matching the lazy construction pattern of the rest of the +// runtime; callers may pass nil to build the struct and wire the adapter later. +func NewLLMExtractor(adapter model.Adapter, personaID, roleID string) *LLMExtractor { + return &LLMExtractor{ + adapter: adapter, + personaID: personaID, + roleID: roleID, + scope: ScopePersona, + kind: KindRelationship, + seen: make(map[string]struct{}), + } +} + +// Extract returns the candidate memory items the adapter derives from +// conversation. A nil adapter returns ErrNilAdapter; a canceled context aborts +// before the LLM call. The response is parsed as a JSON array and then +// deduplicated, so items already extracted by this extractor are dropped. +func (e *LLMExtractor) Extract(ctx context.Context, conversation []model.Message) ([]MemoryItem, error) { + if e.adapter == nil { + return nil, ErrNilAdapter + } + if err := ctx.Err(); err != nil { + return nil, err + } + resp, err := e.adapter.Chat(ctx, model.Request{ + Messages: buildExtractionPrompt(conversation), + Purpose: "memory_extract", + }, model.Options{}) + if err != nil { + return nil, fmt.Errorf("memory: llm extract: %w", err) + } + cands, err := parseCandidates(resp.Content) + if err != nil { + return nil, err + } + return e.dedupeAndBuild(cands), nil +} + +// AsyncResult is the outcome of an ExtractAsync call: either candidate items or +// the error that aborted extraction. +type AsyncResult struct { + Items []MemoryItem + Err error +} + +// ExtractAsync submits the conversation to a background goroutine and returns a +// channel that delivers exactly one AsyncResult when extraction finishes, closed +// by the goroutine. A nil adapter returns ErrNilAdapter immediately. If ctx is +// canceled before the goroutine runs, the result carries the cancellation error. +// Because Extract is idempotent, repeated async extraction of the same memory is +// safe and never double-records. +func (e *LLMExtractor) ExtractAsync(ctx context.Context, conversation []model.Message) (<-chan AsyncResult, error) { + if e.adapter == nil { + return nil, ErrNilAdapter + } + out := make(chan AsyncResult, 1) + go func() { + defer close(out) + if err := ctx.Err(); err != nil { + out <- AsyncResult{Err: err} + return + } + items, err := e.Extract(ctx, conversation) + out <- AsyncResult{Items: items, Err: err} + }() + return out, nil +} + +// buildExtractionPrompt turns the conversation into the model request messages: +// the system instruction plus one user message carrying the transcript. +func buildExtractionPrompt(conversation []model.Message) []model.Message { + var b strings.Builder + for _, m := range conversation { + b.WriteString(string(m.Role)) + b.WriteString(": ") + b.WriteString(messageText(m)) + b.WriteByte('\n') + } + return []model.Message{ + model.TextMessage(model.RoleSystem, extractSystemPrompt), + model.TextMessage(model.RoleUser, b.String()), + } +} + +// parseCandidates decodes the adapter's text response into candidates. A fenced +// ```json block is unwrapped so the model may wrap its array in markdown. Blank +// or non-array content yields no candidates; malformed JSON is an error so the +// caller can decide whether to retry or fall back. +func parseCandidates(s string) ([]llmCandidate, error) { + s = strings.TrimSpace(s) + if strings.HasPrefix(s, "```") { + s = strings.TrimPrefix(s, "```") + s = strings.TrimSuffix(s, "```") + s = strings.TrimSpace(s) + s = strings.TrimPrefix(s, "json") + s = strings.TrimSpace(s) + } + if s == "" { + return nil, nil + } + if strings.HasPrefix(s, "{") { + return nil, fmt.Errorf("memory: llm candidates: expected a JSON array") + } + var cands []llmCandidate + if err := json.Unmarshal([]byte(s), &cands); err != nil { + return nil, fmt.Errorf("memory: decode llm candidates: %w", err) + } + if cands == nil { + return nil, nil + } + return cands, nil +} + +// dedupeAndBuild converts valid candidates to MemoryItems, dropping blank content +// and any content already emitted by this extractor (idempotent dedup). It +// requires e.mu; only valid items mark their dedup key so a malformed candidate +// does not suppress a later valid one with the same text. +func (e *LLMExtractor) dedupeAndBuild(cands []llmCandidate) []MemoryItem { + e.mu.Lock() + defer e.mu.Unlock() + + out := make([]MemoryItem, 0, len(cands)) + for _, c := range cands { + content := strings.TrimSpace(c.Content) + if content == "" { + continue + } + key := dedupeKey(content) + if _, ok := e.seen[key]; ok { + continue + } + it := MemoryItem{ + PersonaID: e.personaID, + RoleID: e.roleID, + ProjectID: strings.TrimSpace(c.ProjectID), + Scope: Scope(normalizeScope(c.Scope, string(e.scope))), + Kind: Kind(normalizeKind(c.Kind, string(e.kind))), + Content: content, + Tags: c.Tags, + Confidence: c.Confidence, + SourceConversation: strings.TrimSpace(c.Source), + } + if err := it.validate(); err != nil { + continue + } + e.seen[key] = struct{}{} + out = append(out, it) + } + return out +} + +// dedupeKey normalizes content for idempotent dedup: case-folded and whitespace- +// collapsed, so "记住 A" and "记住 a" collide into one memory. +func dedupeKey(content string) string { + return strings.Join(strings.Fields(strings.ToLower(content)), " ") +} + +// normalizeScope returns s when it names a defined scope, otherwise fallback. +func normalizeScope(s, fallback string) string { + if Scope(s).valid() { + return s + } + return fallback +} + +// normalizeKind returns s when it names a defined kind, otherwise fallback. +func normalizeKind(s, fallback string) string { + if Kind(s).valid() { + return s + } + return fallback +} diff --git a/internal/runtime/memory/llmextractor_test.go b/internal/runtime/memory/llmextractor_test.go new file mode 100644 index 0000000..a2ec07d --- /dev/null +++ b/internal/runtime/memory/llmextractor_test.go @@ -0,0 +1,240 @@ +package memory + +import ( + "context" + "errors" + "testing" + "time" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model" +) + +// mockAdapter is a scriptable model.Adapter for tests. With no fn it returns an +// empty candidate array, so a test that only needs a successful call can use it +// directly. +type mockAdapter struct { + fn func(ctx context.Context, req model.Request, opts model.Options) (model.Response, error) +} + +func (m *mockAdapter) Name() string { return "mock" } + +func (m *mockAdapter) Chat(ctx context.Context, req model.Request, opts model.Options) (model.Response, error) { + if m.fn != nil { + return m.fn(ctx, req, opts) + } + return model.Response{Content: "[]"}, nil +} + +func (m *mockAdapter) ChatStream(ctx context.Context, req model.Request, onChunk func(delta string)) (model.Response, error) { + return model.Response{}, model.ErrStream +} + +func TestLLMExtractor_Extract_Parse_正常(t *testing.T) { + t.Parallel() + adapter := &mockAdapter{fn: func(_ context.Context, _ model.Request, _ model.Options) (model.Response, error) { + return model.Response{Content: `[ + {"content":"用户喜欢冰美式","scope":"user","kind":"relationship","tags":["咖啡"],"confidence":0.9,"source_conversation":"conv.1"}, + {"content":"项目Alpha周三开会","scope":"project","project_id":"project.alpha","source_conversation":"conv.1"} + ]`}, nil + }} + ex := NewLLMExtractor(adapter, "persona.a", "role.1") + + items, err := ex.Extract(context.Background(), []model.Message{ + model.TextMessage(model.RoleUser, "你好"), + }) + if err != nil { + t.Fatalf("Extract: %v", err) + } + if len(items) != 2 { + t.Fatalf("got %d items, want 2", len(items)) + } + if items[0].Content != "用户喜欢冰美式" || items[0].Scope != ScopeUser || items[0].Confidence != 0.9 { + t.Errorf("items[0] = %+v, want parsed user item", items[0]) + } + if items[0].SourceConversation != "conv.1" { + t.Errorf("items[0].SourceConversation = %q, want conv.1", items[0].SourceConversation) + } + if items[1].ProjectID != "project.alpha" || items[1].Scope != ScopeProject { + t.Errorf("items[1] = %+v, want project-scoped item", items[1]) + } + if items[0].PersonaID != "persona.a" || items[0].RoleID != "role.1" { + t.Errorf("items[0] author = %q/%q, want persona.a/role.1", items[0].PersonaID, items[0].RoleID) + } +} + +func TestLLMExtractor_Extract_InvalidScopeKind_Fallback_正常(t *testing.T) { + t.Parallel() + adapter := &mockAdapter{fn: func(_ context.Context, _ model.Request, _ model.Options) (model.Response, error) { + return model.Response{Content: `[{"content":"一个事实","scope":"bogus","kind":"bogus"}]`}, nil + }} + ex := NewLLMExtractor(adapter, "persona.a", "role.1") + + items, err := ex.Extract(context.Background(), []model.Message{ + model.TextMessage(model.RoleUser, "hi"), + }) + if err != nil { + t.Fatalf("Extract: %v", err) + } + if len(items) != 1 { + t.Fatalf("got %d items, want 1", len(items)) + } + // Bogus scope/kind fall back to the extractor's configured defaults. + if items[0].Scope != ScopePersona || items[0].Kind != KindRelationship { + t.Errorf("fallback scope/kind = %q/%q, want persona/relationship", items[0].Scope, items[0].Kind) + } +} + +func TestLLMExtractor_Extract_Dedup_正常(t *testing.T) { + t.Parallel() + // The model emits the same memory three times, differing only in case and + // surrounding whitespace; they must collapse into one item. + adapter := &mockAdapter{fn: func(_ context.Context, _ model.Request, _ model.Options) (model.Response, error) { + return model.Response{Content: `[ + {"content":"记得喝咖啡"}, + {"content":" 记得喝咖啡 "}, + {"content":"记得喝咖啡"} + ]`}, nil + }} + ex := NewLLMExtractor(adapter, "persona.a", "role.1") + + items, err := ex.Extract(context.Background(), []model.Message{ + model.TextMessage(model.RoleUser, "hi"), + }) + if err != nil { + t.Fatalf("Extract: %v", err) + } + if len(items) != 1 { + t.Fatalf("got %d items, want 1 after dedup", len(items)) + } +} + +func TestLLMExtractor_Extract_Empty_正常(t *testing.T) { + t.Parallel() + ex := NewLLMExtractor(&mockAdapter{}, "persona.a", "role.1") + + items, err := ex.Extract(context.Background(), []model.Message{ + model.TextMessage(model.RoleUser, "hi"), + }) + if err != nil { + t.Fatalf("Extract: %v", err) + } + if len(items) != 0 { + t.Fatalf("got %d items, want 0 for empty candidate response", len(items)) + } +} + +func TestLLMExtractor_Extract_InvalidJSON_异常(t *testing.T) { + t.Parallel() + adapter := &mockAdapter{fn: func(_ context.Context, _ model.Request, _ model.Options) (model.Response, error) { + return model.Response{Content: "{not-json}"}, nil + }} + ex := NewLLMExtractor(adapter, "persona.a", "role.1") + + if _, err := ex.Extract(context.Background(), []model.Message{ + model.TextMessage(model.RoleUser, "hi"), + }); err == nil { + t.Fatal("Extract on malformed JSON: expected error, got nil") + } +} + +func TestLLMExtractor_Extract_NilAdapter_异常(t *testing.T) { + t.Parallel() + ex := NewLLMExtractor(nil, "persona.a", "role.1") + + if _, err := ex.Extract(context.Background(), []model.Message{ + model.TextMessage(model.RoleUser, "hi"), + }); !errors.Is(err, ErrNilAdapter) { + t.Fatalf("Extract with nil adapter err = %v, want ErrNilAdapter", err) + } +} + +func TestLLMExtractor_Extract_Canceled_异常(t *testing.T) { + t.Parallel() + adapter := &mockAdapter{fn: func(ctx context.Context, _ model.Request, _ model.Options) (model.Response, error) { + <-ctx.Done() + return model.Response{}, ctx.Err() + }} + ex := NewLLMExtractor(adapter, "persona.a", "role.1") + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if _, err := ex.Extract(ctx, []model.Message{ + model.TextMessage(model.RoleUser, "hi"), + }); !errors.Is(err, context.Canceled) { + t.Fatalf("Extract with canceled ctx err = %v, want context.Canceled", err) + } +} + +func TestLLMExtractor_ExtractAsync_异步_正常(t *testing.T) { + t.Parallel() + started := make(chan struct{}) + release := make(chan struct{}) + adapter := &mockAdapter{fn: func(_ context.Context, _ model.Request, _ model.Options) (model.Response, error) { + close(started) + <-release + return model.Response{Content: `[{"content":"异步提取的记忆"}]`}, nil + }} + ex := NewLLMExtractor(adapter, "persona.a", "role.1") + + ch, err := ex.ExtractAsync(context.Background(), []model.Message{ + model.TextMessage(model.RoleUser, "hi"), + }) + if err != nil { + t.Fatalf("ExtractAsync: %v", err) + } + // ExtractAsync returned while the adapter is still blocked on release, + // proving the extraction runs on a background goroutine. + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("extraction goroutine did not start in time") + } + close(release) + + res := <-ch + if res.Err != nil { + t.Fatalf("async result err = %v", res.Err) + } + if len(res.Items) != 1 || res.Items[0].Content != "异步提取的记忆" { + t.Fatalf("async items = %+v, want one extracted item", res.Items) + } +} + +func TestLLMExtractor_ExtractAsync_Idempotent_正常(t *testing.T) { + t.Parallel() + adapter := &mockAdapter{fn: func(_ context.Context, _ model.Request, _ model.Options) (model.Response, error) { + return model.Response{Content: `[{"content":"习惯：早起"}]`}, nil + }} + ex := NewLLMExtractor(adapter, "persona.a", "role.1") + + conv := []model.Message{model.TextMessage(model.RoleUser, "hi")} + + first, err := ex.ExtractAsync(context.Background(), conv) + if err != nil { + t.Fatalf("ExtractAsync first: %v", err) + } + if res := <-first; res.Err != nil || len(res.Items) != 1 { + t.Fatalf("first async result = %+v, want 1 item", res) + } + + // A second async extraction of the same memory deduplicates to zero. + second, err := ex.ExtractAsync(context.Background(), conv) + if err != nil { + t.Fatalf("ExtractAsync second: %v", err) + } + if res := <-second; res.Err != nil || len(res.Items) != 0 { + t.Fatalf("second async result = %+v, want 0 items (idempotent)", res) + } +} + +func TestLLMExtractor_ExtractAsync_NilAdapter_异常(t *testing.T) { + t.Parallel() + ex := NewLLMExtractor(nil, "persona.a", "role.1") + + if _, err := ex.ExtractAsync(context.Background(), []model.Message{ + model.TextMessage(model.RoleUser, "hi"), + }); !errors.Is(err, ErrNilAdapter) { + t.Fatalf("ExtractAsync with nil adapter err = %v, want ErrNilAdapter", err) + } +} diff --git a/internal/runtime/memory/retrieve.go b/internal/runtime/memory/retrieve.go index 3511b41..bd7ddae 100644 --- a/internal/runtime/memory/retrieve.go +++ b/internal/runtime/memory/retrieve.go @@ -9,6 +9,8 @@ import ( "time" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/embedding" + "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/memory/projectindex" + "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/memory/vector" ) // Retriever defaults and tuning constants. @@ -35,20 +37,31 @@ const ( ) // Result is a single retrieval hit: the memory item plus its combined -// relevance × recency score explaining its ranking. Higher is more relevant. +// relevance × decay score explaining its ranking. Higher is more relevant. type Result struct { Item MemoryItem Score float64 + // SourceConversation is the conversation the item was extracted from, for + // 溯源 (design §2). It mirrors Item.SourceConversation so callers can trace a + // hit without reaching into Item. + SourceConversation string + // Project is the linked project-index entry (设计 §3 项目分层), populated only + // when the retriever is wired with a project-index store and the item is a + // project-scoped memory carrying a ProjectID. It lets callers surface the + // 项目索引 → 项目描述 → 章节内容 chain for a retrieved item. + Project *projectindex.Project } // Retriever ranks a persona's long-term memories by relevance (keyword and, -// when an embedder is configured, semantic vectors) weighted by recency, then -// returns the top K. It is stateless and safe for concurrent use; the ranking is -// recomputed from the store on every call, so results always reflect persisted -// state. +// when an embedder or vector index is configured, semantic vectors) weighted by +// decay (decay_weight × time decay), then returns the top K. It is stateless and +// safe for concurrent use; the ranking is recomputed from the store on every +// call, so results always reflect persisted state. type Retriever struct { store *Store embedder embedding.Embedder + vindex vector.Index + projectStore *projectindex.Store keywordWeight float64 semanticWeight float64 recencyHalfLife time.Duration @@ -57,14 +70,38 @@ type Retriever struct { // RetrieverOption customizes a Retriever built by NewRetriever. type RetrieverOption func(*Retriever) -// WithEmbedder sets the semantic embedder used for vector similarity. When nil -// (the default), retrieval falls back to keyword matching plus recency only. +// WithEmbedder sets the semantic embedder used for vector similarity computed +// in-memory. When nil (the default), retrieval falls back to keyword matching +// plus decay only. When a vector index is also wired via WithVectorIndex, the +// index takes precedence: it is the persistent semantic path (design §3 +// 可选增强), and WithEmbedder computes similarity on the fly instead. func WithEmbedder(e embedding.Embedder) RetrieverOption { return func(r *Retriever) { r.embedder = e } } +// WithVectorIndex wires a persistent vector.Index into retrieval. When set, +// semantic relevance is answered by idx.Search over the stored vectors rather +// than by in-memory embedding, so a configured pgvector/SQLite vector store is +// genuinely used (design §3 "真正启用向量检索"). When nil (the default), the +// semantic path is skipped unless WithEmbedder is set. The caller is responsible +// for keeping the index in sync with stored items (embed + add on write). +func WithVectorIndex(idx vector.Index) RetrieverOption { + return func(r *Retriever) { + r.vindex = idx + } +} + +// WithProjectIndex wires a projectindex.Store so retrieval can attach the linked +// project-index entry (设计 §3 项目分层) to each project-scoped hit. When nil +// (the default), Result.Project is left unset. +func WithProjectIndex(s *projectindex.Store) RetrieverOption { + return func(r *Retriever) { + r.projectStore = s + } +} + // WithKeywordWeight sets the relative weight of keyword relevance. Non-positive // values are ignored and the default kept. func WithKeywordWeight(w float64) RetrieverOption { @@ -111,9 +148,10 @@ func NewRetriever(store *Store, opts ...RetrieverOption) *Retriever { } // Retrieve returns up to topK items for personaID that best match query, ranked -// by relevance (keyword + optional semantic) weighted by recency. Query and -// candidate contents are both read under ctx; a canceled context aborts the -// work. A blank query returns the most recently used items (recency only). +// by relevance (keyword + optional semantic) weighted by decay (decay_weight × +// time decay). Query and candidate contents are both read under ctx; a canceled +// context aborts the work. A blank query returns the most recently used items +// (decay order only). func (r *Retriever) Retrieve(ctx context.Context, personaID, query string, topK int) ([]Result, error) { if topK <= 0 { topK = defaultTopK @@ -146,12 +184,18 @@ func (r *Retriever) Retrieve(ctx context.Context, personaID, query string, topK results := make([]Result, 0, len(items)) for _, it := range items { - recency := r.recencyFactor(it, now) - score := recency + decay := r.decayFactor(it, now) + score := decay if matched { - score = relevance[it.ID] * recency + score = relevance[it.ID] * decay } - results = append(results, Result{Item: it, Score: score}) + res := Result{ + Item: it, + Score: score, + SourceConversation: it.SourceConversation, + Project: r.projectFor(ctx, it), + } + results = append(results, res) } sort.SliceStable(results, func(i, j int) bool { @@ -194,11 +238,17 @@ func (r *Retriever) keywordScores(items []MemoryItem, query string) map[string]f return scores } -// semanticScores embeds the query and each candidate content with the configured -// embedder and returns the clamped cosine similarity per item. With no embedder -// it returns an empty map, so semantic relevance contributes nothing. +// semanticScores computes per-item semantic relevance for query. When a vector +// index is wired (WithVectorIndex) it answers from the persistent index via +// idx.Search — the real vector path (design §3). Otherwise, with an embedder +// configured, it embeds query and candidates in-memory and uses clamped cosine +// similarity. With neither it returns an empty map, so semantic relevance +// contributes nothing. func (r *Retriever) semanticScores(ctx context.Context, items []MemoryItem, query string) map[string]float64 { scores := make(map[string]float64, len(items)) + if r.vindex != nil { + return r.vectorIndexScores(ctx, items, query) + } if r.embedder == nil { return scores } @@ -228,15 +278,48 @@ func (r *Retriever) semanticScores(ctx context.Context, items []MemoryItem, quer return scores } -// recencyFactor combines the stored decay_weight with the time since the item -// was last touched, halving every recencyHalfLife days. -func (r *Retriever) recencyFactor(it MemoryItem, now time.Time) float64 { - age := now.Sub(it.UpdatedAt) - if age < 0 { - age = 0 +// vectorIndexScores answers semantic relevance from the persistent vector index: +// it embeds the query with the index, searches for the top similar ids, and +// returns their cosine similarity, normalized to [0,1] by the top hit. Because +// idx.Search ranks over all stored ids, an item absent from the index carries no +// semantic score. A blank query or an index error degrades to an empty map. +func (r *Retriever) vectorIndexScores(ctx context.Context, items []MemoryItem, query string) map[string]float64 { + scores := make(map[string]float64, len(items)) + if strings.TrimSpace(query) == "" { + return scores } - halfLives := float64(age) / float64(r.recencyHalfLife) - return it.DecayWeight * math.Pow(0.5, halfLives) + qvecs, err := r.vindex.Embed(ctx, []string{query}) + if err != nil || len(qvecs) == 0 { + return scores + } + hits, err := r.vindex.Search(ctx, qvecs[0], len(items)) + if err != nil || len(hits) == 0 { + return scores + } + max := hits[0].Score + for _, h := range hits { + if max > 0 { + scores[h.ID] = h.Score / max + } else { + scores[h.ID] = h.Score + } + } + return scores +} + +// projectFor returns the project-index entry linked to a project-scoped item, or +// nil when the retriever has no project-index store, the item is not +// project-scoped, or it carries no ProjectID. A load failure is treated as "no +// project" rather than failing the whole retrieval (design §3 项目索引各角色可查). +func (r *Retriever) projectFor(ctx context.Context, it MemoryItem) *projectindex.Project { + if r.projectStore == nil || it.Scope != ScopeProject || strings.TrimSpace(it.ProjectID) == "" { + return nil + } + p, err := r.projectStore.GetProject(ctx, it.ProjectID) + if err != nil { + return nil + } + return p } // keywordText is the text indexed for keyword matching: content plus tags. @@ -279,7 +362,7 @@ func (r *Retriever) Inject(ctx context.Context, personaID, query string, topK in } var b strings.Builder - b.WriteString("相关记忆（按相关度 × 最近度排序）：") + b.WriteString("相关记忆（按相关度 × 衰减排序）：") for _, res := range results { b.WriteByte('\n') b.WriteString("- [") diff --git a/internal/runtime/memory/retrieve_m4_test.go b/internal/runtime/memory/retrieve_m4_test.go new file mode 100644 index 0000000..4fa100a --- /dev/null +++ b/internal/runtime/memory/retrieve_m4_test.go @@ -0,0 +1,249 @@ +package memory + +import ( + "context" + "testing" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/db" + "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/memory/projectindex" + "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/memory/vector" +) + +// mockVectorIndex is a scriptable vector.Index for tests. With no fns it emits +// one-hot vectors and returns no search hits. +type mockVectorIndex struct { + dim int + embedFn func(ctx context.Context, texts []string) ([]vector.Vector, error) + searchFn func(ctx context.Context, q vector.Vector, topK int) ([]vector.Hit, error) +} + +func (m *mockVectorIndex) Dim() int { + if m.dim > 0 { + return m.dim + } + return 4 +} + +func (m *mockVectorIndex) Add(_ context.Context, _ string, _ vector.Vector) error { return nil } + +func (m *mockVectorIndex) Close() error { return nil } + +func (m *mockVectorIndex) Embed(ctx context.Context, texts []string) ([]vector.Vector, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if m.embedFn != nil { + return m.embedFn(ctx, texts) + } + out := make([]vector.Vector, len(texts)) + for i := range texts { + v := make(vector.Vector, m.Dim()) + v[i%m.Dim()] = 1 + out[i] = v + } + return out, nil +} + +func (m *mockVectorIndex) Search(ctx context.Context, _ vector.Vector, topK int) ([]vector.Hit, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if m.searchFn != nil { + return m.searchFn(ctx, nil, topK) + } + return []vector.Hit{}, nil +} + +func TestRetriever_Retrieve_WithVectorIndex_正常(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + // Both items are in English while the query is Chinese, so the keyword path + // shares no token and contributes nothing. The ranking is therefore driven + // entirely by the vector index, proving the persistent vector path is used. + item1 := addItem(t, s, MemoryItem{ + PersonaID: "persona.a", Content: "she is a morning person", + Scope: ScopePersona, Kind: KindRelationship, + }) + item2 := addItem(t, s, MemoryItem{ + PersonaID: "persona.a", Content: "she likes jazz", + Scope: ScopePersona, Kind: KindRelationship, + }) + + idx := &mockVectorIndex{ + searchFn: func(_ context.Context, _ vector.Vector, _ int) ([]vector.Hit, error) { + return []vector.Hit{ + {ID: item1.ID, Score: 0.9}, + {ID: item2.ID, Score: 0.2}, + }, nil + }, + } + r := NewRetriever(s, WithVectorIndex(idx)) + + results, err := r.Retrieve(ctx, "persona.a", "咖啡", 10) + if err != nil { + t.Fatalf("Retrieve: %v", err) + } + if len(results) == 0 { + t.Fatal("expected results with a vector index wired") + } + if got := results[0].Item.Content; got != "she is a morning person" { + t.Fatalf("top result = %q, want the item the vector index ranked highest", got) + } + if results[0].Score <= 0 { + t.Fatalf("top result score = %v, want > 0", results[0].Score) + } +} + +func TestRetriever_Retrieve_WithVectorIndex_PartialHits_正常(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + item1 := addItem(t, s, MemoryItem{ + PersonaID: "persona.a", Content: "she is a morning person", + Scope: ScopePersona, Kind: KindRelationship, + }) + item2 := addItem(t, s, MemoryItem{ + PersonaID: "persona.a", Content: "she likes jazz", + Scope: ScopePersona, Kind: KindRelationship, + }) + + // The vector index only knows about item1; item2 is absent from it, so it + // carries no semantic score. With a Chinese query that the keyword path + // cannot match, only item1 is relevant: it must rank at the top with a + // positive score while item2 scores 0, proving the semantic contribution + // comes strictly from the vector index. + idx := &mockVectorIndex{ + searchFn: func(_ context.Context, _ vector.Vector, _ int) ([]vector.Hit, error) { + return []vector.Hit{{ID: item1.ID, Score: 0.9}}, nil + }, + } + r := NewRetriever(s, WithVectorIndex(idx)) + + results, err := r.Retrieve(ctx, "persona.a", "咖啡", 10) + if err != nil { + t.Fatalf("Retrieve: %v", err) + } + if len(results) != 2 { + t.Fatalf("got %d results, want 2", len(results)) + } + if got := results[0].Item.Content; got != "she is a morning person" { + t.Fatalf("top result = %q, want the vector-indexed item", got) + } + if results[0].Score <= 0 { + t.Fatalf("top result score = %v, want > 0", results[0].Score) + } + // item2 has no vector hit and no keyword overlap, so it scores 0 (matched + // mode) and sorts below. + if results[1].Item.ID != item2.ID { + t.Fatalf("second result id = %q, want item2", results[1].Item.ID) + } + if results[1].Score != 0 { + t.Fatalf("second result score = %v, want 0 (no vector hit)", results[1].Score) + } +} + +func TestRetriever_Retrieve_Source_正常(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + addItem(t, s, MemoryItem{ + PersonaID: "persona.a", + Content: "她喜欢冰美式", + Tags: []string{"coffee"}, + SourceConversation: "conv.42", + Scope: ScopePersona, + Kind: KindRelationship, + }) + + r := NewRetriever(s) + results, err := r.Retrieve(ctx, "persona.a", "冰美式", 10) + if err != nil { + t.Fatalf("Retrieve: %v", err) + } + if len(results) == 0 { + t.Fatal("expected a result") + } + if results[0].SourceConversation != "conv.42" { + t.Fatalf("Result.SourceConversation = %q, want conv.42 (溯源)", results[0].SourceConversation) + } + if results[0].Item.SourceConversation != "conv.42" { + t.Fatalf("Item.SourceConversation = %q, want conv.42", results[0].Item.SourceConversation) + } +} + +func TestRetriever_Retrieve_ProjectLink_正常(t *testing.T) { + t.Parallel() + ctx := context.Background() + + // Separate project-index DB, so it shares no connection with the memory store. + projDB, err := db.Open(ctx, ":memory:", testLogger()) + if err != nil { + t.Fatalf("open project db: %v", err) + } + t.Cleanup(func() { _ = projDB.Close() }) + ps, err := projectindex.New(ctx, projDB, testLogger()) + if err != nil { + t.Fatalf("projectindex.New: %v", err) + } + if err := ps.PutProject(ctx, projectindex.Project{ + ID: "project.alpha", Title: "Alpha 项目", Desc: "Alpha 项目描述", + }); err != nil { + t.Fatalf("PutProject: %v", err) + } + + ms := newTestStore(t) + addItem(t, ms, MemoryItem{ + PersonaID: "persona.a", + Content: "Alpha 项目的一个事实", + Scope: ScopeProject, + ProjectID: "project.alpha", + Kind: KindWorldEvent, + }) + + r := NewRetriever(ms, WithProjectIndex(ps)) + results, err := r.Retrieve(ctx, "persona.a", "Alpha", 10) + if err != nil { + t.Fatalf("Retrieve: %v", err) + } + if len(results) == 0 { + t.Fatal("expected a result") + } + if results[0].Item.Scope != ScopeProject { + t.Fatalf("result scope = %q, want project", results[0].Item.Scope) + } + if results[0].Project == nil { + t.Fatal("Result.Project is nil, want the linked project-index entry") + } + if results[0].Project.ID != "project.alpha" { + t.Fatalf("Result.Project.ID = %q, want project.alpha", results[0].Project.ID) + } +} + +func TestRetriever_Retrieve_ProjectLink_NoStore_正常(t *testing.T) { + t.Parallel() + ctx := context.Background() + ms := newTestStore(t) + + addItem(t, ms, MemoryItem{ + PersonaID: "persona.a", Content: "Alpha 项目的一个事实", + Scope: ScopeProject, ProjectID: "project.alpha", Kind: KindWorldEvent, + }) + + // Without a project-index store wired, the hit still returns and Project is + // simply left unset (no error). + r := NewRetriever(ms) + results, err := r.Retrieve(ctx, "persona.a", "Alpha", 10) + if err != nil { + t.Fatalf("Retrieve: %v", err) + } + if len(results) == 0 { + t.Fatal("expected a result") + } + if results[0].Project != nil { + t.Fatalf("Result.Project = %+v, want nil without a project-index store", results[0].Project) + } +} diff --git a/internal/runtime/memory/rules.go b/internal/runtime/memory/rules.go new file mode 100644 index 0000000..a6a2e40 --- /dev/null +++ b/internal/runtime/memory/rules.go @@ -0,0 +1,148 @@ +package memory + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" +) + +// Scope access control sentinels. These surface when a persona attempts a write +// it is not permitted to make under the design §3 rules. +var ( + // ErrReadOnly is returned when a persona tries to write or delete a + // persona-scope memory item it does not own. Under design §3 other personas + // are read-only for 角色日常记忆. + ErrReadOnly = errors.New("memory: read-only: persona does not own this item") + + // ErrMissingProjectID is returned when a project-scoped query has a blank id. + ErrMissingProjectID = errors.New("memory: project_id is required") +) + +// canWrite reports whether actorID may write item under the design §3 scope rules: +// +// - ScopeUser: any persona may write (user habits are globally shared). +// - ScopePersona: only the owning persona may write (other personas read-only). +// - ScopeProject: any persona may write (project memory is 人人可写). +// +// The caller must have already validated item.Scope; an invalid scope is not +// writable by anyone. +func canWrite(actorID string, it MemoryItem) bool { + switch it.Scope { + case ScopeUser, ScopeProject: + return true + case ScopePersona: + return it.PersonaID == actorID + default: + return false + } +} + +// AddScoped persists item under actorID's authority, enforcing the design §3 +// write rules by persona_id + scope. A blank actorID returns ErrMissingPersona. +// When item.PersonaID is blank it is filled with actorID, so a persona always +// records its own provenance; a persona-scope item owned by a different persona +// returns ErrReadOnly. User- and project-scope items are writable by any persona. +func (s *Store) AddScoped(ctx context.Context, actorID string, item MemoryItem) (MemoryItem, error) { + if strings.TrimSpace(actorID) == "" { + return MemoryItem{}, ErrMissingPersona + } + if item.PersonaID == "" { + item.PersonaID = actorID + } + if !item.Scope.valid() { + return MemoryItem{}, ErrInvalidScope + } + if !canWrite(actorID, item) { + return MemoryItem{}, ErrReadOnly + } + return s.Add(ctx, item) +} + +// ListScope returns every item of scope visible to personaID, newest first, +// applying the design §3 read rules: +// +// - ScopeUser: every user-scope item (globally shared across personas). +// - ScopePersona: every persona-scope item. Other personas are read-only but +// still readable, so a persona can read another persona's 角色日常记忆 here. +// - ScopeProject: every project-scope item (人人可读). +// +// personaID names the reading persona and is required so callers always declare +// the identity boundary even for a shared scope. The higher policy layer still +// decides whether a restricted context (group chat / stranger, design §5) may +// receive these; this method only enforces the base read rule. +func (s *Store) ListScope(ctx context.Context, scope Scope, personaID string) ([]MemoryItem, error) { + if strings.TrimSpace(personaID) == "" { + return nil, ErrMissingPersona + } + if !scope.valid() { + return nil, ErrInvalidScope + } + rows, err := s.db.QueryContext(ctx, `SELECT id, persona_id, role_id, project_id, scope, kind, + content, tags, confidence, source_conversation, created_at, updated_at, + decay_weight FROM memory_items WHERE scope = ? + ORDER BY created_at DESC`, string(scope)) + if err != nil { + return nil, fmt.Errorf("memory: list scope items: %w", err) + } + return scanItems(rows, "scope item") +} + +// ListProject returns every memory item linked to projectID, newest first. A +// project-scoped memory carries a ProjectID that links to the projectindex +// hierarchy (design §3); this is the deep-search entry point — callers combine +// it with the projectindex store to walk 项目索引 → 项目描述 → 章节内容. Project +// memory is readable by all personas, so no persona filter is applied. A blank +// projectID returns ErrMissingProjectID. +func (s *Store) ListProject(ctx context.Context, projectID string) ([]MemoryItem, error) { + if strings.TrimSpace(projectID) == "" { + return nil, ErrMissingProjectID + } + rows, err := s.db.QueryContext(ctx, `SELECT id, persona_id, role_id, project_id, scope, kind, + content, tags, confidence, source_conversation, created_at, updated_at, + decay_weight FROM memory_items WHERE project_id = ? + ORDER BY created_at DESC`, projectID) + if err != nil { + return nil, fmt.Errorf("memory: list project items: %w", err) + } + return scanItems(rows, "project item") +} + +// DeleteScoped removes the item with the given id, but only if actorID may write +// it under the design §3 rules. This enforces the persona read-only boundary on +// deletion: a persona may delete its own persona-scope items and any user- or +// project-scope item, but not another persona's 角色日常记忆. A blank id returns +// ErrNotFound; writing a foreign persona-scope item returns ErrReadOnly. +func (s *Store) DeleteScoped(ctx context.Context, actorID, id string) error { + if strings.TrimSpace(actorID) == "" { + return ErrMissingPersona + } + item, err := s.Get(ctx, id) + if err != nil { + return fmt.Errorf("memory: delete scoped load: %w", err) + } + if !canWrite(actorID, item) { + return ErrReadOnly + } + return s.Delete(ctx, id) +} + +// ensureProjectIDColumn rolls the memory_items table forward when it was created +// before the M4 project index (design §3) existed: it adds the project_id column +// if it is missing, and ignores the duplicate-column result when present. It is +// SQLite-specific (the memory store is SQLite-backed) and idempotent, so NewStore +// may be called on a fresh schema or a pre-existing one. +func ensureProjectIDColumn(ctx context.Context, db *sql.DB) error { + _, err := db.ExecContext(ctx, `ALTER TABLE memory_items + ADD COLUMN project_id TEXT NOT NULL DEFAULT ''`) + if err == nil { + return nil + } + // SQLite reports "duplicate column name: project_id" when the column already + // exists (e.g. created by the CREATE TABLE in NewStore, or by a previous run). + if strings.Contains(err.Error(), "duplicate column name") { + return nil + } + return fmt.Errorf("memory: roll forward project_id column: %w", err) +} diff --git a/internal/runtime/memory/rules_test.go b/internal/runtime/memory/rules_test.go new file mode 100644 index 0000000..a6e3b6d --- /dev/null +++ b/internal/runtime/memory/rules_test.go @@ -0,0 +1,217 @@ +package memory + +import ( + "context" + "errors" + "testing" +) + +func TestStore_ListScope_User_全局共享(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + for _, p := range []string{"persona.a", "persona.b"} { + if _, err := s.AddScoped(ctx, p, MemoryItem{ + PersonaID: p, Scope: ScopeUser, Kind: KindRelationship, + Content: "用户习惯-" + p, + }); err != nil { + t.Fatalf("AddScoped user for %s: %v", p, err) + } + } + + // persona.a sees both personas' user-scope items (globally shared). + items, err := s.ListScope(ctx, ScopeUser, "persona.a") + if err != nil { + t.Fatalf("ListScope user: %v", err) + } + if len(items) != 2 { + t.Fatalf("ListScope(user) = %d items, want 2 (globally shared)", len(items)) + } +} + +func TestStore_ListScope_Persona_其他角色只读(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + if _, err := s.AddScoped(ctx, "persona.a", MemoryItem{ + Scope: ScopePersona, Kind: KindWorldEvent, Content: "a的日常", + }); err != nil { + t.Fatalf("AddScoped persona.a: %v", err) + } + + // persona.b may READ persona.a's persona-scope item (read-only rule). + items, err := s.ListScope(ctx, ScopePersona, "persona.b") + if err != nil { + t.Fatalf("ListScope persona for b: %v", err) + } + if len(items) != 1 || items[0].PersonaID != "persona.a" { + t.Fatalf("ListScope(persona) for persona.b = %+v, want persona.a's item", items) + } + // AddScoped auto-filled PersonaID from the acting actor. + if items[0].PersonaID != "persona.a" { + t.Errorf("owner persona = %q, want persona.a", items[0].PersonaID) + } +} + +func TestStore_AddScoped_Persona_跨角色写_异常(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + if _, err := s.AddScoped(ctx, "persona.a", MemoryItem{ + Scope: ScopePersona, Kind: KindRelationship, Content: "a的日常", + }); err != nil { + t.Fatalf("AddScoped persona.a: %v", err) + } + + // persona.b cannot WRITE persona.a's persona-scope item. + _, err := s.AddScoped(ctx, "persona.b", MemoryItem{ + PersonaID: "persona.a", Scope: ScopePersona, Kind: KindRelationship, + Content: "b想改a的日常", + }) + if !errors.Is(err, ErrReadOnly) { + t.Fatalf("AddScoped cross-persona err = %v, want wrap of ErrReadOnly", err) + } +} + +func TestStore_AddScoped_UserProject_人人写(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + // Any persona may write a user- or project-scope item. + for _, sc := range []Scope{ScopeUser, ScopeProject} { + item, err := s.AddScoped(ctx, "persona.b", MemoryItem{ + Scope: sc, Kind: KindWorldEvent, Content: "共享-" + string(sc), + }) + if err != nil { + t.Fatalf("AddScoped %s: %v", sc, err) + } + // PersonaID was auto-filled with the acting actor. + if item.PersonaID != "persona.b" { + t.Errorf("AddScoped %s owner = %q, want persona.b", sc, item.PersonaID) + } + } +} + +func TestStore_AddScoped_MissingActor_异常(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + if _, err := s.AddScoped(ctx, "", MemoryItem{ + Content: "x", Scope: ScopeUser, + }); !errors.Is(err, ErrMissingPersona) { + t.Fatalf("AddScoped(empty actor) err = %v, want ErrMissingPersona", err) + } +} + +func TestStore_AddScoped_InvalidScope_异常(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + if _, err := s.AddScoped(ctx, "persona.a", MemoryItem{ + Content: "x", Scope: "bogus", + }); !errors.Is(err, ErrInvalidScope) { + t.Fatalf("AddScoped(bogus scope) err = %v, want ErrInvalidScope", err) + } +} + +func TestStore_ListScope_前置校验_异常(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + if _, err := s.ListScope(ctx, ScopeUser, ""); !errors.Is(err, ErrMissingPersona) { + t.Fatalf("ListScope(empty persona) err = %v, want ErrMissingPersona", err) + } + if _, err := s.ListScope(ctx, "bogus", "persona.a"); !errors.Is(err, ErrInvalidScope) { + t.Fatalf("ListScope(bogus scope) err = %v, want ErrInvalidScope", err) + } +} + +func TestStore_DeleteScoped_跨角色_异常(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + stored, err := s.AddScoped(ctx, "persona.a", MemoryItem{ + Scope: ScopePersona, Kind: KindRelationship, Content: "a的日常", + }) + if err != nil { + t.Fatalf("AddScoped persona.a: %v", err) + } + + // persona.a deletes its own item. + if err := s.DeleteScoped(ctx, "persona.a", stored.ID); err != nil { + t.Fatalf("DeleteScoped owner: %v", err) + } + + // Re-add, then persona.b cannot delete persona.a's item. + stored2, err := s.AddScoped(ctx, "persona.a", MemoryItem{ + Scope: ScopePersona, Kind: KindRelationship, Content: "a的日常2", + }) + if err != nil { + t.Fatalf("AddScoped persona.a: %v", err) + } + if err := s.DeleteScoped(ctx, "persona.b", stored2.ID); !errors.Is(err, ErrReadOnly) { + t.Fatalf("DeleteScoped cross-persona err = %v, want ErrReadOnly", err) + } + // The foreign item survives. + if _, err := s.Get(ctx, stored2.ID); err != nil { + t.Fatalf("foreign item removed: %v", err) + } +} + +func TestStore_DeleteScoped_BlankActor_异常(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + if err := s.DeleteScoped(ctx, "", "some-id"); !errors.Is(err, ErrMissingPersona) { + t.Fatalf("DeleteScoped(empty actor) err = %v, want ErrMissingPersona", err) + } +} + +func TestStore_ListProject_项目记忆(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + if _, err := s.AddScoped(ctx, "persona.a", MemoryItem{ + Scope: ScopeProject, Kind: KindWorldEvent, Content: "项目Alpha事实", + ProjectID: "project.alpha", + }); err != nil { + t.Fatalf("AddScoped project: %v", err) + } + if _, err := s.AddScoped(ctx, "persona.b", MemoryItem{ + Scope: ScopeProject, Kind: KindWorldEvent, Content: "项目Beta事实", + ProjectID: "project.beta", + }); err != nil { + t.Fatalf("AddScoped project: %v", err) + } + + items, err := s.ListProject(ctx, "project.alpha") + if err != nil { + t.Fatalf("ListProject: %v", err) + } + if len(items) != 1 || items[0].ProjectID != "project.alpha" { + t.Fatalf("ListProject(alpha) = %+v, want only alpha item", items) + } + if items[0].Content != "项目Alpha事实" { + t.Errorf("item content = %q, want the alpha item", items[0].Content) + } +} + +func TestStore_ListProject_BlankID_异常(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t) + + if _, err := s.ListProject(ctx, ""); !errors.Is(err, ErrMissingProjectID) { + t.Fatalf("ListProject(blank) err = %v, want ErrMissingProjectID", err) + } +} diff --git a/internal/runtime/memory/store.go b/internal/runtime/memory/store.go index fb510b8..7803b61 100644 --- a/internal/runtime/memory/store.go +++ b/internal/runtime/memory/store.go @@ -4,23 +4,27 @@ // It exposes three layers: // // - Store: a SQLite-backed long-term memory store. It persists memory items -// with persona/role isolation (design §0 layer ②, §4) and offers -// Add/Get/ListByPersona/Delete plus a manual "记住" path (design §1). +// with persona/role isolation (design §0 layer ②, §4), applies the +// three-scope access rules (design §3) via AddScoped/ListScope/ListProject/ +// DeleteScoped, and offers Add/Get/ListByPersona/Delete plus a manual +// "记住" path (design §1). Each item may carry a project_id linking the +// project-index hierarchy. // - WorkingMemory: a per-session sliding window of messages plus a summary // (design §0 layer ①). -// - Retriever: relevance + recency retrieval and context injection -// (design §2). +// - Retriever: relevance (keyword + optional semantic) × decay retrieval with +// source provenance and context injection (design §2). It accepts an optional +// vector.Index for the persistent semantic path and a projectindex.Store so +// project-scoped hits surface their linked project entry. // -// Extraction (design §1) is exposed through the Extractor interface; the -// concrete M2 implementation is the rule-based ManualExtractor, which records -// what the user explicitly asks to remember. LLM-driven automatic extraction is -// deferred to milestone M4 and is not implemented here. +// Extraction (design §1) is exposed through the Extractor interface with two +// implementations: ManualExtractor (rule-based, records an explicit "记住 X" +// directive) and LLMExtractor (drives a model.Adapter to denoise and deduplicate +// a turn, running synchronously via Extract or asynchronously via ExtractAsync). // -// Milestone M4 additions that this package deliberately leaves out but keeps -// extension points for: persistent vector storage, decay scheduling, and the -// project-index hierarchy (design §3). The schema already carries decay_weight -// and the retriever already plugs an optional embedding.Embedder, so those -// features land without a schema migration. +// Milestone M4 is implemented here: the schema carries decay_weight and +// project_id, the retriever plugs an optional embedding.Embedder / vector.Index +// and a project-index store, and the store enforces the user/persona/project +// scope rules. No external dependency is needed beyond the existing packages. package memory import ( @@ -35,6 +39,8 @@ import ( "log/slog" "strings" "time" + + "git.yeij.top/AskaEth/Cyrene_v2/internal/logger" ) // Scope classifies who a long-term memory belongs to. It is orthogonal to Kind @@ -89,6 +95,7 @@ type MemoryItem struct { ID string PersonaID string RoleID string + ProjectID string Scope Scope Kind Kind Content string @@ -156,7 +163,7 @@ func NewStore(ctx context.Context, db *sql.DB, log *slog.Logger) (*Store, error) if db == nil { return nil, ErrNilDB } - log = discardLogger(log) + log = logger.WithSource(discardLogger(log), "memory") for _, ddl := range []string{ createMemoryItemsTable, createMemoryItemsPersonaIndex, @@ -166,6 +173,9 @@ func NewStore(ctx context.Context, db *sql.DB, log *slog.Logger) (*Store, error) return nil, fmt.Errorf("memory: apply schema: %w", err) } } + if err := ensureProjectIDColumn(ctx, db); err != nil { + return nil, err + } return &Store{db: db, log: log}, nil } @@ -186,10 +196,10 @@ func (s *Store) Add(ctx context.Context, item MemoryItem) (MemoryItem, error) { } _, err = s.db.ExecContext(ctx, `INSERT INTO memory_items - (id, persona_id, role_id, scope, kind, content, tags, confidence, + (id, persona_id, role_id, project_id, scope, kind, content, tags, confidence, source_conversation, created_at, updated_at, decay_weight) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - item.ID, item.PersonaID, item.RoleID, string(item.Scope), string(item.Kind), + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + item.ID, item.PersonaID, item.RoleID, item.ProjectID, string(item.Scope), string(item.Kind), item.Content, string(tagsJSON), item.Confidence, item.SourceConversation, formatTime(item.CreatedAt), formatTime(item.UpdatedAt), item.DecayWeight) if err != nil { @@ -204,7 +214,7 @@ func (s *Store) Get(ctx context.Context, id string) (MemoryItem, error) { if strings.TrimSpace(id) == "" { return MemoryItem{}, ErrNotFound } - row := s.db.QueryRowContext(ctx, `SELECT id, persona_id, role_id, scope, kind, + row := s.db.QueryRowContext(ctx, `SELECT id, persona_id, role_id, project_id, scope, kind, content, tags, confidence, source_conversation, created_at, updated_at, decay_weight FROM memory_items WHERE id = ?`, id) item, err := scanItem(row) @@ -219,30 +229,39 @@ func (s *Store) Get(ctx context.Context, id string) (MemoryItem, error) { // ListByPersona returns every item for personaID, newest first. It is the // persona-isolation boundary: items belonging to other personas are never -// returned, so personas never share memory (design §4). +// returned, so personas never share memory (design §4). Use ListScope when a +// caller needs to see shared user/project scopes or another persona's read-only +// items. func (s *Store) ListByPersona(ctx context.Context, personaID string) ([]MemoryItem, error) { if strings.TrimSpace(personaID) == "" { return nil, ErrMissingPersona } - rows, err := s.db.QueryContext(ctx, `SELECT id, persona_id, role_id, scope, kind, + rows, err := s.db.QueryContext(ctx, `SELECT id, persona_id, role_id, project_id, scope, kind, content, tags, confidence, source_conversation, created_at, updated_at, decay_weight FROM memory_items WHERE persona_id = ? ORDER BY created_at DESC`, personaID) if err != nil { return nil, fmt.Errorf("memory: list persona items: %w", err) } + return scanItems(rows, "persona item") +} + +// scanItems reads every row into a MemoryItem slice, closing rows and wrapping +// scan/iterate failures with the operation name. A non-nil, possibly empty slice +// is always returned on success. +func scanItems(rows *sql.Rows, op string) ([]MemoryItem, error) { defer func() { _ = rows.Close() }() items := make([]MemoryItem, 0) for rows.Next() { item, err := scanItem(rows) if err != nil { - return nil, fmt.Errorf("memory: scan persona item: %w", err) + return nil, fmt.Errorf("memory: scan %s: %w", op, err) } items = append(items, item) } if err := rows.Err(); err != nil { - return nil, fmt.Errorf("memory: iterate persona items: %w", err) + return nil, fmt.Errorf("memory: iterate %s: %w", op, err) } return items, nil } @@ -352,7 +371,7 @@ func scanItem(sc rowScanner) (MemoryItem, error) { tagsJSON string createdAt, updatedAt string ) - if err := sc.Scan(&it.ID, &it.PersonaID, &it.RoleID, &scope, &kind, + if err := sc.Scan(&it.ID, &it.PersonaID, &it.RoleID, &it.ProjectID, &scope, &kind, &it.Content, &tagsJSON, &it.Confidence, &it.SourceConversation, &createdAt, &updatedAt, &it.DecayWeight); err != nil { return MemoryItem{}, err @@ -423,6 +442,7 @@ CREATE TABLE IF NOT EXISTS memory_items ( id TEXT PRIMARY KEY, persona_id TEXT NOT NULL, role_id TEXT NOT NULL DEFAULT '', + project_id TEXT NOT NULL DEFAULT '', scope TEXT NOT NULL, kind TEXT NOT NULL, content TEXT NOT NULL,