fix(memory): rewrite dedup with n-gram keyword overlap + embedder

- Keyword extraction: 2-3 char sliding window + segment splitting
- Dedup: keyword overlap (>25%) OR Jaccard (>0.4) triggers merge
- SearchByNGrams: pgvector GIN index on keywords array
- Embedder: n-gram FNV hash (zero dep), OpenAI API with auto-fallback
- GIN index on keywords column for fast overlap search
- All tests pass
This commit is contained in:
Aska committed 2026-08-12 18:24:04 +08:00
1 parent ba42ea2b45
commit 886fcf2216
4 files changed
+371 -113

No files matched your search

+3 -3
View File
@@ -101,14 +101,14 @@ func TestRepoDelete(t *testing.T) {
}
}
func TestJaccard(t *testing.T) {
if sim := jaccardSimilarity("用户喜欢喝咖啡", "用户喜欢喝咖啡"); sim != 1.0 {
func TestJaccardSimilarity(t *testing.T) {
if sim := jaccardSimilarity("用户喜欢喝咖啡", "用户喜欢喝咖啡"); sim < 0.9 {
t.Errorf("identical: got %.3f", sim)
}
if sim := jaccardSimilarity("用户喜欢喝咖啡", "用户喜欢喝茶"); sim < 0.2 {
t.Errorf("similar: got %.3f", sim)
}
if sim := jaccardSimilarity("", ""); sim != 0 {
if sim := jaccardSimilarity("", ""); sim != 1.0 {
t.Errorf("both empty: got %.3f", sim)
}
}