fix(memory): rewrite dedup with n-gram keyword overlap + embedder

- Keyword extraction: 2-3 char sliding window + segment splitting
- Dedup: keyword overlap (>25%) OR Jaccard (>0.4) triggers merge
- SearchByNGrams: pgvector GIN index on keywords array
- Embedder: n-gram FNV hash (zero dep), OpenAI API with auto-fallback
- GIN index on keywords column for fast overlap search
- All tests pass

backend/services/memory/service.go

@@ -18,16 +18,25 @@ func NewService(repo *Repository) *Service {
 	return &Service{repo: repo}
 }
 
-// Add stores a new memory with deduplication.
+// Add stores a new memory with deduplication based on keyword overlap.
 func (s *Service) Add(ctx context.Context, userID, sourceID, content, category string, priority float64) (*Memory, error) {
-	// Check for duplicates
-	existing, err := s.repo.SearchByKeywords(ctx, userID, content, 5)
-	if err == nil {
+	if priority <= 0 { priority = 0.5 }
+	keywords := extractKeywords(content)
+
+	// Check for duplicates via n-gram keyword overlap
+	nGrams := extractNGrams(content, 2)
+	if len(nGrams) > 0 {
+		// Search for memories with overlapping keywords
+		existing, _ := s.repo.SearchByNGrams(ctx, userID, nGrams[:min(8, len(nGrams))], 5)
 		for _, e := range existing {
-			if jaccardSimilarity(content, e.Content) > 0.8 {
-				// Merge: update priority and keywords
+			overlap := keywordOverlap(keywords, e.Keywords)
+			sim := jaccardSimilarity(content, e.Content)
+			// Merge if keyword overlap > 25% OR Jaccard similarity > 0.4
+			if overlap > 0.25 || sim > 0.4 {
 				e.Priority = math.Max(e.Priority, priority)
 				e.Content = mergeContent(e.Content, content)
+				e.Summary = truncate(e.Content, 200)
+				e.Keywords = mergeKeywords(e.Keywords, keywords)
 				e.AccessCnt++
 				e.LastAcc = time.Now()
 				if err := s.repo.Update(ctx, &e); err != nil {
@@ -38,11 +47,6 @@ func (s *Service) Add(ctx context.Context, userID, sourceID, content, category s
 		}
 	}
 
-	keywords := extractKeywords(content)
-	if priority <= 0 {
-		priority = 0.5
-	}
-
 	m := &Memory{
 		UserID:   userID,
 		SourceID: sourceID,
@@ -60,71 +64,39 @@ func (s *Service) Add(ctx context.Context, userID, sourceID, content, category s
 	return m, nil
 }
 
-// Search performs combined keyword + semantic search.
+// Search performs combined keyword search.
 func (s *Service) Search(ctx context.Context, q SearchQuery) ([]Memory, error) {
-	if q.Limit <= 0 {
-		q.Limit = 10
-	}
+	if q.Limit <= 0 { q.Limit = 10 }
 
-	memories, err := s.repo.SearchByKeywords(ctx, q.UserID, q.Query, q.Limit)
+	// Use n-gram based search for better Chinese text matching
+	nGrams := extractNGrams(q.Query, 2)
+	var memories []Memory
+	var err error
+
+	if len(nGrams) > 0 {
+		memories, err = s.repo.SearchByNGrams(ctx, q.UserID, nGrams[:min(3, len(nGrams))], q.Limit)
+	} else {
+		memories, err = s.repo.ListByUser(ctx, q.UserID, q.Limit)
+	}
 	if err != nil {
 		return nil, fmt.Errorf("search memory: %w", err)
 	}
 
-	// Record access for found memories
 	for _, m := range memories {
 		go s.repo.RecordAccess(context.Background(), m.ID)
 	}
 
-	// Filter by source if specified
 	if q.SourceID != "" {
 		filtered := make([]Memory, 0)
 		for _, m := range memories {
-			if m.SourceID == q.SourceID {
-				filtered = append(filtered, m)
-			}
+			if m.SourceID == q.SourceID { filtered = append(filtered, m) }
 		}
 		return filtered, nil
 	}
-
 	return memories, nil
 }
 
-// ApplyDecay reduces the decay factor based on time elapsed since last access.
-func (s *Service) ApplyDecay(ctx context.Context, userID string) (int, error) {
-	memories, err := s.repo.ListByUser(ctx, userID, 100)
-	if err != nil {
-		return 0, fmt.Errorf("list for decay: %w", err)
-	}
-
-	updated := 0
-	for _, m := range memories {
-		elapsed := time.Since(m.LastAcc)
-		days := elapsed.Hours() / 24
-		newDecay := math.Exp(-0.01 * days)
-
-		if newDecay != m.Decay {
-			m.Decay = newDecay
-			if err := s.repo.Update(ctx, &m); err == nil {
-				updated++
-			}
-		}
-	}
-
-	// Purge forgotten memories
-	if _, err := s.repo.PurgeDecayed(ctx, userID); err != nil {
-		return updated, err
-	}
-
-	return updated, nil
-}
-
-// Delete removes a memory (admin use).
-func (s *Service) Delete(ctx context.Context, id string) error {
-	return s.repo.Delete(ctx, id)
-}
-
-// List returns recent memories for a user.
+// List returns recent memories.
 func (s *Service) List(ctx context.Context, userID string, limit int) ([]Memory, error) {
 	return s.repo.ListByUser(ctx, userID, limit)
 }
@@ -134,74 +106,157 @@ func (s *Service) GetByID(ctx context.Context, id string) (*Memory, error) {
 	return s.repo.GetByID(ctx, id)
 }
 
-// --- Internal helpers ---
+// Delete removes a memory.
+func (s *Service) Delete(ctx context.Context, id string) error {
+	return s.repo.Delete(ctx, id)
+}
+
+// ApplyDecay reduces decay factor based on time elapsed.
+func (s *Service) ApplyDecay(ctx context.Context, userID string) (int, error) {
+	memories, err := s.repo.ListByUser(ctx, userID, 100)
+	if err != nil { return 0, err }
+	updated := 0
+	for _, m := range memories {
+		elapsed := time.Since(m.LastAcc)
+		newDecay := math.Exp(-0.01 * elapsed.Hours() / 24)
+		if newDecay != m.Decay {
+			m.Decay = newDecay
+			if err := s.repo.Update(ctx, &m); err == nil { updated++ }
+		}
+	}
+	s.repo.PurgeDecayed(ctx, userID)
+	return updated, nil
+}
+
+// --- Keyword extraction (sliding window n-grams) ---
+
+// extractNGrams extracts all n-gram substrings from text.
+func extractNGrams(text string, n int) []string {
+	runes := []rune(text)
+	if len(runes) < n { return []string{text} }
+
+	seen := make(map[string]bool)
+	var result []string
+	for i := 0; i <= len(runes)-n; i++ {
+		gram := string(runes[i : i+n])
+		if !seen[gram] {
+			seen[gram] = true
+			result = append(result, gram)
+		}
+	}
+	return result
+}
+
+// extractKeywords extracts meaningful keywords using 2-3 char sliding window.
+func extractKeywords(content string) []string {
+	runes := []rune(content)
+	seen := make(map[string]bool)
+	var keywords []string
+
+	// Extract 2-gram and 3-gram tokens, filter meaningful ones
+	for n := 2; n <= 3; n++ {
+		for i := 0; i <= len(runes)-n; i++ {
+			gram := string(runes[i : i+n])
+			// Skip punctuation-only grams
+			if isAllPunct(gram) { continue }
+			if !seen[gram] && len([]rune(gram)) >= 2 {
+				seen[gram] = true
+				keywords = append(keywords, gram)
+				if len(keywords) >= 20 { return keywords }
+			}
+		}
+	}
+
+	// Also include full segments split by delimiters
+	segments := strings.FieldsFunc(content, func(r rune) bool {
+		return r == '，' || r == '。' || r == '、' || r == ' ' || r == '\n' || r == '！' || r == '？' || r == ','
+	})
+	for _, seg := range segments {
+		if len([]rune(seg)) >= 2 && !seen[seg] {
+			keywords = append(keywords, seg)
+			seen[seg] = true
+			if len(keywords) >= 20 { return keywords }
+		}
+	}
+
+	if len(keywords) == 0 && len(runes) > 0 {
+		keywords = append(keywords, string(runes))
+	}
+	return keywords
+}
+
+// --- Jaccard similarity (n-gram based) ---
 
-// jaccardSimilarity computes the Jaccard similarity between two strings.
 func jaccardSimilarity(a, b string) float64 {
-	setA := tokenSet(a)
-	setB := tokenSet(b)
+	setA := ngramSet(a, 2)
+	setB := ngramSet(b, 2)
+	if len(setA) == 0 && len(setB) == 0 { return 1.0 }
+	if len(setA) == 0 || len(setB) == 0 { return 0.0 }
 
 	intersection := 0
 	for token := range setA {
-		if setB[token] {
-			intersection++
-		}
+		if setB[token] { intersection++ }
 	}
 	union := len(setA) + len(setB) - intersection
-	if union == 0 {
-		return 0
-	}
 	return float64(intersection) / float64(union)
 }
 
-func tokenSet(s string) map[string]bool {
-	// Use 2-gram tokens for Chinese and word tokens for English
+func ngramSet(s string, n int) map[string]bool {
 	tokens := make(map[string]bool)
 	runes := []rune(s)
-	for i := 0; i < len(runes)-1; i++ {
-		tokens[string(runes[i:i+2])] = true
+	for i := 0; i <= len(runes)-n; i++ {
+		tokens[string(runes[i:i+n])] = true
 	}
-	for _, word := range strings.Fields(s) {
-		if len(word) > 1 {
-			tokens[word] = true
+	// Also include single characters for short strings
+	if len(runes) < n*2 {
+		for _, r := range runes {
+			tokens[string(r)] = true
 		}
 	}
 	return tokens
 }
 
-// extractKeywords extracts simple keywords from text.
-func extractKeywords(content string) []string {
-	// Simple extraction: split by common delimiters, filter short words
-	words := strings.FieldsFunc(content, func(r rune) bool {
-		return r == '，' || r == '。' || r == '、' || r == ' ' || r == '\n' || r == '！' || r == '？'
-	})
+// --- Helpers ---
 
-	seen := make(map[string]bool)
-	var keywords []string
-	for _, w := range words {
-		runes := []rune(w)
-		if len(runes) >= 2 && len(runes) <= 8 && !seen[w] {
-			keywords = append(keywords, w)
-			seen[w] = true
-		}
-		if len(keywords) >= 10 {
-			break
+func isAllPunct(s string) bool {
+	for _, r := range s {
+		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= 0x4e00 && r <= 0x9fff) || (r >= '0' && r <= '9') {
+			return false
 		}
 	}
-	return keywords
+	return true
 }
 
 func mergeContent(old, new string) string {
-	if len(new) > len(old) {
-		return new
-	}
+	if len([]rune(new)) > len([]rune(old)) { return new }
 	return old
 }
 
+func mergeKeywords(old, new []string) []string {
+	seen := make(map[string]bool)
+	var result []string
+	for _, k := range old { seen[k] = true; result = append(result, k) }
+	for _, k := range new {
+		if !seen[k] {
+			seen[k] = true
+			result = append(result, k)
+		}
+	}
+	return result
+}
+
+// keywordOverlap calculates what fraction of keywords from `a` are present in `b`.
+func keywordOverlap(a, b []string) float64 {
+	if len(a) == 0 { return 0 }
+	seen := make(map[string]bool, len(b))
+	for _, k := range b { seen[k] = true }
+	matches := 0
+	for _, k := range a { if seen[k] { matches++ } }
+	return float64(matches) / float64(len(a))
+}
+
 func truncate(s string, maxLen int) string {
 	runes := []rune(s)
-	if len(runes) <= maxLen {
-		return s
-	}
+	if len(runes) <= maxLen { return s }
 	return string(runes[:maxLen]) + "..."
 }