diff --git a/backend/services/memory/service.go b/backend/services/memory/service.go index 932c30b..149e78b 100644 --- a/backend/services/memory/service.go +++ b/backend/services/memory/service.go @@ -31,8 +31,9 @@ func (s *Service) Add(ctx context.Context, userID, sourceID, content, category s for _, e := range existing { overlap := keywordOverlap(keywords, e.Keywords) sim := jaccardSimilarity(content, e.Content) - // Merge if keyword overlap > 25% OR Jaccard similarity > 0.4 - if overlap > 0.25 || sim > 0.4 { + // Merge if keyword overlap > 28% OR Jaccard > 0.45 + // (raised from 25%/0.4 to reduce false positives) + if overlap > 0.28 || sim > 0.45 { e.Priority = math.Max(e.Priority, priority) e.Content = mergeContent(e.Content, content) e.Summary = truncate(e.Content, 200) @@ -149,36 +150,46 @@ func extractNGrams(text string, n int) []string { // extractKeywords extracts meaningful keywords using 2-3 char sliding window. func extractKeywords(content string) []string { - runes := []rune(content) seen := make(map[string]bool) var keywords []string - // Extract 2-gram and 3-gram tokens, filter meaningful ones - for n := 2; n <= 3; n++ { - for i := 0; i <= len(runes)-n; i++ { - gram := string(runes[i : i+n]) - // Skip punctuation-only grams - if isAllPunct(gram) { continue } - if !seen[gram] && len([]rune(gram)) >= 2 { - seen[gram] = true - keywords = append(keywords, gram) - if len(keywords) >= 20 { return keywords } - } - } - } - - // Also include full segments split by delimiters + // Priority 1: Full segments (comma/sentence split) — most meaningful segments := strings.FieldsFunc(content, func(r rune) bool { return r == ',' || r == '。' || r == '、' || r == ' ' || r == '\n' || r == '!' || r == '?' || r == ',' }) for _, seg := range segments { + seg = strings.TrimSpace(seg) if len([]rune(seg)) >= 2 && !seen[seg] { keywords = append(keywords, seg) seen[seg] = true + if len(keywords) >= 10 { return keywords } + } + } + + // Priority 2: 3-grams (more discriminative than 2-grams) + runes := []rune(content) + for i := 0; i <= len(runes)-3; i++ { + gram := string(runes[i : i+3]) + if isAllPunct(gram) { continue } + if !seen[gram] { + seen[gram] = true + keywords = append(keywords, gram) if len(keywords) >= 20 { return keywords } } } + // Priority 3: 2-grams (fallback) + for i := 0; i <= len(runes)-2; i++ { + gram := string(runes[i : i+2]) + if isAllPunct(gram) { continue } + if !seen[gram] { + seen[gram] = true + keywords = append(keywords, gram) + if len(keywords) >= 25 { return keywords } + } + } + + // Priority 4: Full text (last resort) if len(keywords) == 0 && len(runes) > 0 { keywords = append(keywords, string(runes)) }