fix(memory): rewrite dedup with n-gram keyword overlap + embedder

- Keyword extraction: 2-3 char sliding window + segment splitting
- Dedup: keyword overlap (>25%) OR Jaccard (>0.4) triggers merge
- SearchByNGrams: pgvector GIN index on keywords array
- Embedder: n-gram FNV hash (zero dep), OpenAI API with auto-fallback
- GIN index on keywords column for fast overlap search
- All tests pass

backend/services/memory/repository.go

@@ -90,34 +90,43 @@ func (r *Repository) ListByUser(ctx context.Context, userID string, limit int) (
 
 // SearchByKeywords performs full-text search on content and keywords.
 func (r *Repository) SearchByKeywords(ctx context.Context, userID, q string, limit int) ([]Memory, error) {
-	if limit <= 0 {
-		limit = 20
-	}
-	// Split query into tokens
+	if limit <= 0 { limit = 20 }
 	tokens := strings.Fields(q)
 	var conditions []string
 	var args []any
 	args = append(args, userID, DecayThreshold)
-
-	for i, token := range tokens {
+	for _, token := range tokens {
 		paramIdx := len(args) + 1
 		conditions = append(conditions, fmt.Sprintf("(content ILIKE $%d OR $%d = ANY(keywords))", paramIdx, paramIdx))
 		args = append(args, "%"+token+"%")
-		_ = i
 	}
-
 	query := fmt.Sprintf(`
 		SELECT id, user_id, source_id, content, summary, keywords, category, priority, decay, access_count, last_accessed_at, created_at, updated_at
-		FROM memories
-		WHERE user_id = $1 AND decay > $2 AND (%s)
-		ORDER BY priority DESC
-		LIMIT $%d
+		FROM memories WHERE user_id = $1 AND decay > $2 AND (%s) ORDER BY priority DESC LIMIT $%d
 	`, strings.Join(conditions, " OR "), len(args)+1)
-
 	args = append(args, limit)
 	return r.queryMemories(ctx, query, args...)
 }
 
+// SearchByNGrams searches memories by n-gram tokens matching on keywords array.
+// More effective for Chinese text than ILIKE substring matching.
+func (r *Repository) SearchByNGrams(ctx context.Context, userID string, nGrams []string, limit int) ([]Memory, error) {
+	if limit <= 0 { limit = 10 }
+	if len(nGrams) == 0 {
+		return r.ListByUser(ctx, userID, limit)
+	}
+
+	// Search where any n-gram matches any keyword in the keywords array
+	// Use postgres array overlap operator: keywords && ARRAY[$1, $2, ...]
+	query := fmt.Sprintf(`
+		SELECT id, user_id, source_id, content, summary, keywords, category, priority, decay, access_count, last_accessed_at, created_at, updated_at
+		FROM memories WHERE user_id = $1 AND decay > $2 AND keywords && $3
+		ORDER BY priority DESC LIMIT $4
+	`)
+
+	return r.queryMemories(ctx, query, userID, DecayThreshold, pq.Array(nGrams), limit)
+}
+
 // Update updates an existing memory.
 func (r *Repository) Update(ctx context.Context, m *Memory) error {
 	m.UpdatedAt = time.Now()
@@ -183,6 +192,7 @@ func CreateTableSQL() string {
 	CREATE INDEX IF NOT EXISTS idx_memories_source_id ON memories(source_id);
 	CREATE INDEX IF NOT EXISTS idx_memories_category ON memories(category);
 	CREATE INDEX IF NOT EXISTS idx_memories_decay ON memories(decay) WHERE decay < 0.1;
+	CREATE INDEX IF NOT EXISTS idx_memories_keywords ON memories USING gin(keywords);
 	`
 }