fix(memory): rewrite dedup with n-gram keyword overlap + embedder
- Keyword extraction: 2-3 char sliding window + segment splitting - Dedup: keyword overlap (>25%) OR Jaccard (>0.4) triggers merge - SearchByNGrams: pgvector GIN index on keywords array - Embedder: n-gram FNV hash (zero dep), OpenAI API with auto-fallback - GIN index on keywords column for fast overlap search - All tests pass
This commit is contained in:
1 parent
ba42ea2b45
commit
886fcf2216
4 files changed
+371
-113
No files matched your search
@@ -101,14 +101,14 @@ func TestRepoDelete(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestJaccard(t *testing.T) {
|
||||
if sim := jaccardSimilarity("用户喜欢喝咖啡", "用户喜欢喝咖啡"); sim != 1.0 {
|
||||
func TestJaccardSimilarity(t *testing.T) {
|
||||
if sim := jaccardSimilarity("用户喜欢喝咖啡", "用户喜欢喝咖啡"); sim < 0.9 {
|
||||
t.Errorf("identical: got %.3f", sim)
|
||||
}
|
||||
if sim := jaccardSimilarity("用户喜欢喝咖啡", "用户喜欢喝茶"); sim < 0.2 {
|
||||
t.Errorf("similar: got %.3f", sim)
|
||||
}
|
||||
if sim := jaccardSimilarity("", ""); sim != 0 {
|
||||
if sim := jaccardSimilarity("", ""); sim != 1.0 {
|
||||
t.Errorf("both empty: got %.3f", sim)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user