feat(memory): M4 记忆完善（三类作用域访问规则+项目索引分层+LLM异步提取+衰减溯源+向量接入，兼容既有57用例）
This commit is contained in:
Aska committed
1 parent
5b37f5a551
commit
d947ec41a9
10 files changed
+1349
-53
No files matched your search
@@ -9,6 +9,8 @@ import (
|
||||
"time"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/embedding"
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/memory/projectindex"
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/memory/vector"
|
||||
)
|
||||
|
||||
// Retriever defaults and tuning constants.
|
||||
@@ -35,20 +37,31 @@ const (
|
||||
)
|
||||
|
||||
// Result is a single retrieval hit: the memory item plus its combined
|
||||
// relevance × recency score explaining its ranking. Higher is more relevant.
|
||||
// relevance × decay score explaining its ranking. Higher is more relevant.
|
||||
type Result struct {
|
||||
Item MemoryItem
|
||||
Score float64
|
||||
// SourceConversation is the conversation the item was extracted from, for
|
||||
// 溯源 (design §2). It mirrors Item.SourceConversation so callers can trace a
|
||||
// hit without reaching into Item.
|
||||
SourceConversation string
|
||||
// Project is the linked project-index entry (设计 §3 项目分层), populated only
|
||||
// when the retriever is wired with a project-index store and the item is a
|
||||
// project-scoped memory carrying a ProjectID. It lets callers surface the
|
||||
// 项目索引 → 项目描述 → 章节内容 chain for a retrieved item.
|
||||
Project *projectindex.Project
|
||||
}
|
||||
|
||||
// Retriever ranks a persona's long-term memories by relevance (keyword and,
|
||||
// when an embedder is configured, semantic vectors) weighted by recency, then
|
||||
// returns the top K. It is stateless and safe for concurrent use; the ranking is
|
||||
// recomputed from the store on every call, so results always reflect persisted
|
||||
// state.
|
||||
// when an embedder or vector index is configured, semantic vectors) weighted by
|
||||
// decay (decay_weight × time decay), then returns the top K. It is stateless and
|
||||
// safe for concurrent use; the ranking is recomputed from the store on every
|
||||
// call, so results always reflect persisted state.
|
||||
type Retriever struct {
|
||||
store *Store
|
||||
embedder embedding.Embedder
|
||||
vindex vector.Index
|
||||
projectStore *projectindex.Store
|
||||
keywordWeight float64
|
||||
semanticWeight float64
|
||||
recencyHalfLife time.Duration
|
||||
@@ -57,14 +70,38 @@ type Retriever struct {
|
||||
// RetrieverOption customizes a Retriever built by NewRetriever.
|
||||
type RetrieverOption func(*Retriever)
|
||||
|
||||
// WithEmbedder sets the semantic embedder used for vector similarity. When nil
|
||||
// (the default), retrieval falls back to keyword matching plus recency only.
|
||||
// WithEmbedder sets the semantic embedder used for vector similarity computed
|
||||
// in-memory. When nil (the default), retrieval falls back to keyword matching
|
||||
// plus decay only. When a vector index is also wired via WithVectorIndex, the
|
||||
// index takes precedence: it is the persistent semantic path (design §3
|
||||
// 可选增强), and WithEmbedder computes similarity on the fly instead.
|
||||
func WithEmbedder(e embedding.Embedder) RetrieverOption {
|
||||
return func(r *Retriever) {
|
||||
r.embedder = e
|
||||
}
|
||||
}
|
||||
|
||||
// WithVectorIndex wires a persistent vector.Index into retrieval. When set,
|
||||
// semantic relevance is answered by idx.Search over the stored vectors rather
|
||||
// than by in-memory embedding, so a configured pgvector/SQLite vector store is
|
||||
// genuinely used (design §3 "真正启用向量检索"). When nil (the default), the
|
||||
// semantic path is skipped unless WithEmbedder is set. The caller is responsible
|
||||
// for keeping the index in sync with stored items (embed + add on write).
|
||||
func WithVectorIndex(idx vector.Index) RetrieverOption {
|
||||
return func(r *Retriever) {
|
||||
r.vindex = idx
|
||||
}
|
||||
}
|
||||
|
||||
// WithProjectIndex wires a projectindex.Store so retrieval can attach the linked
|
||||
// project-index entry (设计 §3 项目分层) to each project-scoped hit. When nil
|
||||
// (the default), Result.Project is left unset.
|
||||
func WithProjectIndex(s *projectindex.Store) RetrieverOption {
|
||||
return func(r *Retriever) {
|
||||
r.projectStore = s
|
||||
}
|
||||
}
|
||||
|
||||
// WithKeywordWeight sets the relative weight of keyword relevance. Non-positive
|
||||
// values are ignored and the default kept.
|
||||
func WithKeywordWeight(w float64) RetrieverOption {
|
||||
@@ -111,9 +148,10 @@ func NewRetriever(store *Store, opts ...RetrieverOption) *Retriever {
|
||||
}
|
||||
|
||||
// Retrieve returns up to topK items for personaID that best match query, ranked
|
||||
// by relevance (keyword + optional semantic) weighted by recency. Query and
|
||||
// candidate contents are both read under ctx; a canceled context aborts the
|
||||
// work. A blank query returns the most recently used items (recency only).
|
||||
// by relevance (keyword + optional semantic) weighted by decay (decay_weight ×
|
||||
// time decay). Query and candidate contents are both read under ctx; a canceled
|
||||
// context aborts the work. A blank query returns the most recently used items
|
||||
// (decay order only).
|
||||
func (r *Retriever) Retrieve(ctx context.Context, personaID, query string, topK int) ([]Result, error) {
|
||||
if topK <= 0 {
|
||||
topK = defaultTopK
|
||||
@@ -146,12 +184,18 @@ func (r *Retriever) Retrieve(ctx context.Context, personaID, query string, topK
|
||||
|
||||
results := make([]Result, 0, len(items))
|
||||
for _, it := range items {
|
||||
recency := r.recencyFactor(it, now)
|
||||
score := recency
|
||||
decay := r.decayFactor(it, now)
|
||||
score := decay
|
||||
if matched {
|
||||
score = relevance[it.ID] * recency
|
||||
score = relevance[it.ID] * decay
|
||||
}
|
||||
results = append(results, Result{Item: it, Score: score})
|
||||
res := Result{
|
||||
Item: it,
|
||||
Score: score,
|
||||
SourceConversation: it.SourceConversation,
|
||||
Project: r.projectFor(ctx, it),
|
||||
}
|
||||
results = append(results, res)
|
||||
}
|
||||
|
||||
sort.SliceStable(results, func(i, j int) bool {
|
||||
@@ -194,11 +238,17 @@ func (r *Retriever) keywordScores(items []MemoryItem, query string) map[string]f
|
||||
return scores
|
||||
}
|
||||
|
||||
// semanticScores embeds the query and each candidate content with the configured
|
||||
// embedder and returns the clamped cosine similarity per item. With no embedder
|
||||
// it returns an empty map, so semantic relevance contributes nothing.
|
||||
// semanticScores computes per-item semantic relevance for query. When a vector
|
||||
// index is wired (WithVectorIndex) it answers from the persistent index via
|
||||
// idx.Search — the real vector path (design §3). Otherwise, with an embedder
|
||||
// configured, it embeds query and candidates in-memory and uses clamped cosine
|
||||
// similarity. With neither it returns an empty map, so semantic relevance
|
||||
// contributes nothing.
|
||||
func (r *Retriever) semanticScores(ctx context.Context, items []MemoryItem, query string) map[string]float64 {
|
||||
scores := make(map[string]float64, len(items))
|
||||
if r.vindex != nil {
|
||||
return r.vectorIndexScores(ctx, items, query)
|
||||
}
|
||||
if r.embedder == nil {
|
||||
return scores
|
||||
}
|
||||
@@ -228,15 +278,48 @@ func (r *Retriever) semanticScores(ctx context.Context, items []MemoryItem, quer
|
||||
return scores
|
||||
}
|
||||
|
||||
// recencyFactor combines the stored decay_weight with the time since the item
|
||||
// was last touched, halving every recencyHalfLife days.
|
||||
func (r *Retriever) recencyFactor(it MemoryItem, now time.Time) float64 {
|
||||
age := now.Sub(it.UpdatedAt)
|
||||
if age < 0 {
|
||||
age = 0
|
||||
// vectorIndexScores answers semantic relevance from the persistent vector index:
|
||||
// it embeds the query with the index, searches for the top similar ids, and
|
||||
// returns their cosine similarity, normalized to [0,1] by the top hit. Because
|
||||
// idx.Search ranks over all stored ids, an item absent from the index carries no
|
||||
// semantic score. A blank query or an index error degrades to an empty map.
|
||||
func (r *Retriever) vectorIndexScores(ctx context.Context, items []MemoryItem, query string) map[string]float64 {
|
||||
scores := make(map[string]float64, len(items))
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return scores
|
||||
}
|
||||
halfLives := float64(age) / float64(r.recencyHalfLife)
|
||||
return it.DecayWeight * math.Pow(0.5, halfLives)
|
||||
qvecs, err := r.vindex.Embed(ctx, []string{query})
|
||||
if err != nil || len(qvecs) == 0 {
|
||||
return scores
|
||||
}
|
||||
hits, err := r.vindex.Search(ctx, qvecs[0], len(items))
|
||||
if err != nil || len(hits) == 0 {
|
||||
return scores
|
||||
}
|
||||
max := hits[0].Score
|
||||
for _, h := range hits {
|
||||
if max > 0 {
|
||||
scores[h.ID] = h.Score / max
|
||||
} else {
|
||||
scores[h.ID] = h.Score
|
||||
}
|
||||
}
|
||||
return scores
|
||||
}
|
||||
|
||||
// projectFor returns the project-index entry linked to a project-scoped item, or
|
||||
// nil when the retriever has no project-index store, the item is not
|
||||
// project-scoped, or it carries no ProjectID. A load failure is treated as "no
|
||||
// project" rather than failing the whole retrieval (design §3 项目索引各角色可查).
|
||||
func (r *Retriever) projectFor(ctx context.Context, it MemoryItem) *projectindex.Project {
|
||||
if r.projectStore == nil || it.Scope != ScopeProject || strings.TrimSpace(it.ProjectID) == "" {
|
||||
return nil
|
||||
}
|
||||
p, err := r.projectStore.GetProject(ctx, it.ProjectID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// keywordText is the text indexed for keyword matching: content plus tags.
|
||||
@@ -279,7 +362,7 @@ func (r *Retriever) Inject(ctx context.Context, personaID, query string, topK in
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("相关记忆（按相关度 × 最近度排序）：")
|
||||
b.WriteString("相关记忆（按相关度 × 衰减排序）：")
|
||||
for _, res := range results {
|
||||
b.WriteByte('\n')
|
||||
b.WriteString("- [")
|
||||
|
||||
Reference in new issue
Block a user