feat(memory): hybrid vector search via embedder
- repository: optional pgvector embedding column (migration with CREATE EXTENSION guarded by main), SearchByVector cosine search, vector-enabled insert path - service: SetEmbedder — memories are embedded on Add; Search merges vector hits with n-gram keyword matches (vector first, deduped) - embedder was already implemented (OpenAI-compatible API + n-gram FNV fallback) but never instantiated; main now wires it from EMBEDDING_API_URL/EMBEDDING_MODEL config
This commit is contained in:
1 parent
8190007865
commit
8495a4b055
4 files changed
+119
-8
No files matched your search
@@ -10,7 +10,8 @@ import (
|
||||
|
||||
// Service handles business logic for the memory system.
|
||||
type Service struct {
|
||||
repo *Repository
|
||||
repo *Repository
|
||||
embedder *Embedder
|
||||
}
|
||||
|
||||
// NewService creates a new memory service.
|
||||
@@ -18,6 +19,9 @@ func NewService(repo *Repository) *Service {
|
||||
return &Service{repo: repo}
|
||||
}
|
||||
|
||||
// SetEmbedder enables embedding-based semantic search.
|
||||
func (s *Service) SetEmbedder(e *Embedder) { s.embedder = e }
|
||||
|
||||
// Add stores a new memory with deduplication based on keyword overlap.
|
||||
func (s *Service) Add(ctx context.Context, userID, sourceID, content, category string, priority float64) (*Memory, error) {
|
||||
if priority <= 0 { priority = 0.5 }
|
||||
@@ -58,6 +62,11 @@ func (s *Service) Add(ctx context.Context, userID, sourceID, content, category s
|
||||
Priority: priority,
|
||||
Decay: 1.0,
|
||||
}
|
||||
if s.embedder != nil {
|
||||
if vec, err := s.embedder.Embed(ctx, content); err == nil {
|
||||
m.Embedding = vec
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := s.repo.Create(ctx, m); err != nil {
|
||||
return nil, fmt.Errorf("add memory: %w", err)
|
||||
@@ -65,23 +74,46 @@ func (s *Service) Add(ctx context.Context, userID, sourceID, content, category s
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// Search performs combined keyword search.
|
||||
// Search performs hybrid search: vector similarity (when an embedder is
|
||||
// configured) merged with n-gram keyword matches, vector hits first.
|
||||
func (s *Service) Search(ctx context.Context, q SearchQuery) ([]Memory, error) {
|
||||
if q.Limit <= 0 { q.Limit = 10 }
|
||||
|
||||
// Use n-gram based search for better Chinese text matching
|
||||
nGrams := extractNGrams(q.Query, 2)
|
||||
seen := make(map[string]bool)
|
||||
var memories []Memory
|
||||
var err error
|
||||
|
||||
if s.embedder != nil {
|
||||
if vec, err := s.embedder.Embed(ctx, q.Query); err == nil {
|
||||
if vecHits, err := s.repo.SearchByVector(ctx, q.UserID, vec, q.Limit); err == nil {
|
||||
for _, m := range vecHits {
|
||||
memories = append(memories, m)
|
||||
seen[m.ID] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// N-gram keyword search for better Chinese text matching
|
||||
nGrams := extractNGrams(q.Query, 2)
|
||||
var kwHits []Memory
|
||||
var err error
|
||||
if len(nGrams) > 0 {
|
||||
memories, err = s.repo.SearchByNGrams(ctx, q.UserID, nGrams[:min(3, len(nGrams))], q.Limit)
|
||||
kwHits, err = s.repo.SearchByNGrams(ctx, q.UserID, nGrams[:min(3, len(nGrams))], q.Limit)
|
||||
} else {
|
||||
memories, err = s.repo.ListByUser(ctx, q.UserID, q.Limit)
|
||||
kwHits, err = s.repo.ListByUser(ctx, q.UserID, q.Limit)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search memory: %w", err)
|
||||
}
|
||||
for _, m := range kwHits {
|
||||
if !seen[m.ID] {
|
||||
memories = append(memories, m)
|
||||
seen[m.ID] = true
|
||||
}
|
||||
}
|
||||
if len(memories) > q.Limit {
|
||||
memories = memories[:q.Limit]
|
||||
}
|
||||
|
||||
for _, m := range memories {
|
||||
go s.repo.RecordAccess(context.Background(), m.ID)
|
||||
|
||||
Reference in new issue
Block a user