feat(memory): 记忆整理**应用路径 + 调度修复 + 手动真跑**(TIDY-1,主人投诉的根因)
CI / Backend (build / vet / test / cross-compile) (push) Has been cancelled
CI / Frontend (tsc + vite build) (push) Has been cancelled

主人实测:memory_items 1191 行、候选 77 条全 pending、memory_tidy_runs 仅 1 行、两晚定时没跑。
根因两条(都在这片里修掉):
① **以前根本没有执行面** —— 只有只读试算,候选永远 pending
② **调度被重启饿死** —— interval=24h 的 ticker 从进程启动算起,两天重启十几次 ⇒ 24h 永远走不完

- 应用路径:逐条 / 按 run 批量(默认 100、硬顶 500、超限 truncated)/ revert;**软合并**
  (status + merged_into/merged_from/merge_count,**绝不改 content、行不删**)+ 审计
  (memory.tidy.apply / memory.tidy.revert)+ **幂等**(候选状态条件更新 + 行 status 闸两道)
- 调度:改成**到点即跑**(catchUp:无历史+窗口内 / 距上次 run ≥ interval / 窗口起点晚于上次 run),
  「上次 run」以 memory_tidy_runs 为**持久事实**;注入时钟用例证明**重启不饿死**
- 手动真跑端点 POST /admin/memories/tidy/run(同档鉴权 + 审计 + 回执:新增/合并/软删/耗时)
- 候选表加 run_id 列 + 索引(前滚;顺序=建表→补列→建索引,否则既有库会 no such column)
- 3 个变异自检(删 catchUp / 去掉两道 status 闸 / truncated 恒 false)
⚠️ 生产库零动作:1191 行真跑与 77 条候选应用**未执行**(按红线由主人确认后主会话执行)
⚠️ 契约段在报告里(candidates 列表/apply/revert/tidy.run),Docs/ 待登记
This commit is contained in:
AskaEth committed 2026-10-06 22:39:35 +08:00
1 parent 57d0e8c543
commit 454ddb0b70
18 files changed
+3181 -71

No files matched your search

+477
View File
@@ -0,0 +1,477 @@
package memory
import (
"context"
"encoding/json"
"errors"
"sync"
"testing"
"time"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/embedding"
)
// 本文件锁定记忆整理**应用路径**(切片 TIDY-1;design/23 §7/§8)的六条判据:
//
// ① merge = 软合并:被吸收行 status='merged' + merged_into,存活行
// merged_from/merge_count 更新,**content 逐字不变**,memory_items 总行数不变;
// ② **幂等**:重复应用同一条候选 ⇒ skipped/already_decided,且不产生第二次副作用
// (merge_count 不再涨、没有第二行被改);
// ③ **可 revert**:回滚后字段复原(status/merged_into/merged_from/merge_count/kind),
// 行数不变,候选退回 pending;
// ④ archive = 软删(status='archived'),reclassify = 改 kind,都不碰 content;
// ⑤ link 不可应用 ⇒ skipped/action_not_applyable,零改动;
// ⑥ 批量(按 run)有上限、逐条结果、超限如实报 Truncated + Total。
// tidyItemRow 是断言用的一行记忆关键字段(含正文,用来钉「绝不改 content」)。
type tidyItemRow struct {
status string
mergedInto string
mergedFrom string
mergeCount int
content string
kind string
}
// loadTidyItemRow 读一行记忆的整理列 + 正文。
func loadTidyItemRow(t *testing.T, store *Store, id string) tidyItemRow {
t.Helper()
var r tidyItemRow
err := store.db.QueryRowContext(context.Background(),
`SELECT status, merged_into, merged_from, merge_count, content, kind
FROM memory_items WHERE id = ?`, id).
Scan(&r.status, &r.mergedInto, &r.mergedFrom, &r.mergeCount, &r.content, &r.kind)
if err != nil {
t.Fatalf("读记忆行 %s: %v", id, err)
}
return r
}
// countMemoryItems 数 memory_items 的总行数(软删除 ⇒ 应用前后必须不变)。
func countMemoryItems(t *testing.T, store *Store) int {
t.Helper()
var n int
if err := store.db.QueryRowContext(context.Background(),
`SELECT COUNT(*) FROM memory_items`).Scan(&n); err != nil {
t.Fatalf("数 memory_items: %v", err)
}
return n
}
// mergeCandidateFromRun 跑一轮整理并返回唯一的 merge 候选(顺带把运行 id 带出来)。
func mergeCandidateFromRun(t *testing.T, store *Store, emb embedding.Embedder) TidyCandidate {
t.Helper()
tr := newTestTidier(t, store, tidyTestCfg(0.80), emb)
stats, err := tr.RunOnce(context.Background())
if err != nil {
t.Fatalf("RunOnce: %v", err)
}
cands, err := store.ListTidyCandidates(context.Background(), "", 50)
if err != nil {
t.Fatalf("ListTidyCandidates: %v", err)
}
merges := candidatesByAction(cands, TidyMerge)
if len(merges) != 1 {
t.Fatalf("merge 候选 = %d, want 1(本轮 run=%s)", len(merges), stats.RunID)
}
if merges[0].RunID != stats.RunID {
t.Fatalf("候选 run_id = %q, want 本轮 run_id = %q", merges[0].RunID, stats.RunID)
}
return merges[0]
}
// TestTidyApplyMergeIsSoftIdempotentAndRevertible 是①②③三条判据的合并用例:
// 软合并 → 重复应用无副作用 → 回滚后字段复原。
func TestTidyApplyMergeIsSoftIdempotentAndRevertible(t *testing.T) {
t.Parallel()
ctx := context.Background()
store := newTestStore(t)
base := tidyTestTime(t)
short := seedTidyItem(t, store, habitItem(fixtureHabitShort, base, 0.9))
long := seedTidyItem(t, store, habitItem(fixtureHabitLong, base.Add(time.Hour), 0.9))
cand := mergeCandidateFromRun(t, store, stubEmbedder{vecs: map[string]embedding.Vector{
fixtureHabitShort: {1, 0},
fixtureHabitLong: simVector(0.95),
}})
if cand.SourceIDs[0] != short.ID || cand.TargetID != long.ID {
t.Fatalf("候选方向不对:sources=%v target=%q", cand.SourceIDs, cand.TargetID)
}
rowsBefore := countMemoryItems(t, store)
shortBefore := loadTidyItemRow(t, store, short.ID)
longBefore := loadTidyItemRow(t, store, long.ID)
res, err := store.ApplyTidyCandidate(ctx, cand.ID, TidyApplyOptions{Actor: "owner:test"})
if err != nil {
t.Fatalf("ApplyTidyCandidate: %v", err)
}
if res.Status != TidyApplyApplied {
t.Fatalf("应用结果 = %+v, want applied", res)
}
absorbed := loadTidyItemRow(t, store, short.ID)
if absorbed.status != ingestStatusMerged || absorbed.mergedInto != long.ID {
t.Fatalf("被吸收行 = %+v, want status=merged merged_into=%s", absorbed, long.ID)
}
if absorbed.content != shortBefore.content {
t.Fatalf("被吸收行 content 被改了:%q → %q", shortBefore.content, absorbed.content)
}
survivor := loadTidyItemRow(t, store, long.ID)
if survivor.content != longBefore.content {
t.Fatalf("存活行 content 被改了:%q → %q", longBefore.content, survivor.content)
}
if survivor.mergeCount != longBefore.mergeCount+1 {
t.Fatalf("存活行 merge_count = %d, want %d", survivor.mergeCount, longBefore.mergeCount+1)
}
if !jsonHasID(t, survivor.mergedFrom, short.ID) {
t.Fatalf("存活行 merged_from = %s, want 含 %s", survivor.mergedFrom, short.ID)
}
if got := countMemoryItems(t, store); got != rowsBefore {
t.Fatalf("软合并改变了行数:%d → %d(行永不物理删除)", rowsBefore, got)
}
applied, err := store.GetTidyCandidate(ctx, cand.ID)
if err != nil {
t.Fatalf("GetTidyCandidate: %v", err)
}
if applied.Status != tidyCandidateApplied {
t.Fatalf("候选状态 = %q, want %q", applied.Status, tidyCandidateApplied)
}
// ② 重复应用:skipped/already_decided,且**零副作用**。
again, err := store.ApplyTidyCandidate(ctx, cand.ID, TidyApplyOptions{Actor: "owner:test"})
if err != nil {
t.Fatalf("第二次 ApplyTidyCandidate 不该报错(幂等命中不是错误):%v", err)
}
if again.Status != TidyApplySkipped || again.Reason != tidySkipAlreadyDecided {
t.Fatalf("第二次应用 = %+v, want skipped/%s", again, tidySkipAlreadyDecided)
}
if len(again.AffectedIDs) != 0 {
t.Fatalf("跳过的结果不得声称改过行:%v", again.AffectedIDs)
}
if got := loadTidyItemRow(t, store, long.ID).mergeCount; got != survivor.mergeCount {
t.Fatalf("重复应用把 merge_count 又加了一次:%d → %d", survivor.mergeCount, got)
}
if got := countMemoryItems(t, store); got != rowsBefore {
t.Fatalf("重复应用改变了行数:%d → %d", rowsBefore, got)
}
// ③ 回滚:字段逐字复原(updated_at 除外 —— 它是「什么时候动过」的审计面)。
rev, err := store.RevertTidyCandidate(ctx, cand.ID, TidyApplyOptions{Actor: "owner:test"})
if err != nil {
t.Fatalf("RevertTidyCandidate: %v", err)
}
if rev.Status != TidyApplyApplied {
t.Fatalf("回滚结果 = %+v, want applied", rev)
}
if got := loadTidyItemRow(t, store, short.ID); got != shortBefore {
t.Fatalf("回滚后被吸收行未复原:\n got=%+v\nwant=%+v", got, shortBefore)
}
if got := loadTidyItemRow(t, store, long.ID); got.mergeCount != longBefore.mergeCount || got.mergedFrom != "[]" {
t.Fatalf("回滚后存活行合并元信息未复原:%+v(want merge_count=%d merged_from=[])", got, longBefore.mergeCount)
}
if got := countMemoryItems(t, store); got != rowsBefore {
t.Fatalf("回滚改变了行数:%d → %d", rowsBefore, got)
}
if back, err := store.GetTidyCandidate(ctx, cand.ID); err != nil || back.Status != tidyCandidatePending {
t.Fatalf("回滚后候选状态 = %q (err=%v), want pending", back.Status, err)
}
// 回滚是幂等的:再回一次 ⇒ skipped/not_applied。
if againRev, err := store.RevertTidyCandidate(ctx, cand.ID, TidyApplyOptions{Actor: "owner:test"}); err != nil {
t.Fatalf("第二次回滚不该报错:%v", err)
} else if againRev.Status != TidyApplySkipped || againRev.Reason != tidySkipNotApplied {
t.Fatalf("第二次回滚 = %+v, want skipped/%s", againRev, tidySkipNotApplied)
}
}
// TestTidyApplyMergeUnderConcurrencyHasSingleWinner 钉住执行闸(候选状态的条件
// 更新)在**并发**下也只有一次生效:16 路同时应用同一条候选,恰好一条 applied,
// merge_count 只涨一次。少了这道闸,多路会各自把源行当作 active 再合并一次。
func TestTidyApplyMergeUnderConcurrencyHasSingleWinner(t *testing.T) {
t.Parallel()
ctx := context.Background()
store := newTestStore(t)
base := tidyTestTime(t)
short := seedTidyItem(t, store, habitItem(fixtureHabitShort, base, 0.9))
long := seedTidyItem(t, store, habitItem(fixtureHabitLong, base.Add(time.Hour), 0.9))
cand := mergeCandidateFromRun(t, store, stubEmbedder{vecs: map[string]embedding.Vector{
fixtureHabitShort: {1, 0},
fixtureHabitLong: simVector(0.95),
}})
const racers = 16
var (
mu sync.Mutex
applied int
skipped int
start = make(chan struct{})
wg sync.WaitGroup
)
for i := 0; i < racers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
<-start
res, err := store.ApplyTidyCandidate(ctx, cand.ID, TidyApplyOptions{Actor: "owner:race"})
mu.Lock()
defer mu.Unlock()
switch {
case err != nil:
t.Errorf("并发应用报错:%v", err)
case res.Status == TidyApplyApplied:
applied++
default:
skipped++
}
}()
}
close(start)
wg.Wait()
if applied != 1 || skipped != racers-1 {
t.Fatalf("并发应用 applied=%d skipped=%d, want 1/%d", applied, skipped, racers-1)
}
if got := loadTidyItemRow(t, store, long.ID).mergeCount; got != 2 {
t.Fatalf("存活行 merge_count = %d, want 2(只允许一次吸收)", got)
}
if got := loadTidyItemRow(t, store, short.ID); got.status != ingestStatusMerged || got.mergedInto != long.ID {
t.Fatalf("被吸收行 = %+v, want merged into %s", got, long.ID)
}
}
// TestTidyApplyArchiveAndRevert 钉住④:archive 是软删(status='archived',内容不变、
// 行数不变),revert 复原为 active。
func TestTidyApplyArchiveAndRevert(t *testing.T) {
t.Parallel()
ctx := context.Background()
store := newTestStore(t)
base := tidyTestTime(t)
oneOff := seedTidyItem(t, store, habitItem(fixtureOneOffEvent, base, 0.9))
tr := newTestTidier(t, store, tidyTestCfg(0.80), nil)
if _, err := tr.RunOnce(ctx); err != nil {
t.Fatalf("RunOnce: %v", err)
}
cands, err := store.ListTidyCandidates(ctx, "", 50)
if err != nil {
t.Fatalf("ListTidyCandidates: %v", err)
}
archives := candidatesByAction(cands, TidyArchive)
if len(archives) != 1 {
t.Fatalf("archive 候选 = %d, want 1(%+v)", len(archives), cands)
}
rowsBefore := countMemoryItems(t, store)
before := loadTidyItemRow(t, store, oneOff.ID)
if res, err := store.ApplyTidyCandidate(ctx, archives[0].ID, TidyApplyOptions{Actor: "owner:test"}); err != nil {
t.Fatalf("ApplyTidyCandidate: %v", err)
} else if res.Status != TidyApplyApplied {
t.Fatalf("archive 应用 = %+v, want applied", res)
}
archived := loadTidyItemRow(t, store, oneOff.ID)
if archived.status != tidyStatusArchived {
t.Fatalf("归档后 status = %q, want %q", archived.status, tidyStatusArchived)
}
if archived.content != before.content {
t.Fatalf("软删改了 content:%q → %q", before.content, archived.content)
}
if got := countMemoryItems(t, store); got != rowsBefore {
t.Fatalf("软删改变了行数:%d → %d(design/23 §8:行保留)", rowsBefore, got)
}
if res, err := store.RevertTidyCandidate(ctx, archives[0].ID, TidyApplyOptions{Actor: "owner:test"}); err != nil {
t.Fatalf("RevertTidyCandidate: %v", err)
} else if res.Status != TidyApplyApplied {
t.Fatalf("archive 回滚 = %+v, want applied", res)
}
if got := loadTidyItemRow(t, store, oneOff.ID); got != before {
t.Fatalf("回滚后未复原:\n got=%+v\nwant=%+v", got, before)
}
if got := countMemoryItems(t, store); got != rowsBefore {
t.Fatalf("回滚改变了行数:%d → %d", rowsBefore, got)
}
}
// TestTidyApplyReclassifyAndRevert 钉住④的另一半:reclassify 只改 kind 一列,
// 内容不变;revert 把 kind 还原成候选证据里的旧值。
func TestTidyApplyReclassifyAndRevert(t *testing.T) {
t.Parallel()
ctx := context.Background()
store := newTestStore(t)
base := tidyTestTime(t)
misclassified := MemoryItem{
PersonaID: "persona.a", Scope: ScopePersona, Kind: KindWorldEvent,
Content: fixtureHabitShort, Confidence: 0.9, CreatedAt: base,
}
superset := habitItem(fixtureHabitLong, base.Add(time.Hour), 0.9)
bad := seedTidyItem(t, store, misclassified)
seedTidyItem(t, store, superset)
tr := newTestTidier(t, store, tidyTestCfg(0.80), stubEmbedder{vecs: map[string]embedding.Vector{
fixtureHabitShort: {1, 0},
fixtureHabitLong: simVector(0.95),
}})
if _, err := tr.RunOnce(ctx); err != nil {
t.Fatalf("RunOnce: %v", err)
}
cands, err := store.ListTidyCandidates(ctx, "", 50)
if err != nil {
t.Fatalf("ListTidyCandidates: %v", err)
}
res := candidatesByAction(cands, TidyReclassify)
if len(res) != 1 {
t.Fatalf("reclassify 候选 = %d, want 1(%+v)", len(res), cands)
}
before := loadTidyItemRow(t, store, bad.ID)
if out, err := store.ApplyTidyCandidate(ctx, res[0].ID, TidyApplyOptions{Actor: "owner:test"}); err != nil {
t.Fatalf("ApplyTidyCandidate: %v", err)
} else if out.Status != TidyApplyApplied {
t.Fatalf("reclassify 应用 = %+v, want applied", out)
}
after := loadTidyItemRow(t, store, bad.ID)
if after.kind != string(KindRelationship) {
t.Fatalf("reclassify 后 kind = %q, want %q", after.kind, KindRelationship)
}
if after.content != before.content || after.status != before.status {
t.Fatalf("reclassify 改了不该改的列:%+v(before=%+v)", after, before)
}
if _, err := store.RevertTidyCandidate(ctx, res[0].ID, TidyApplyOptions{Actor: "owner:test"}); err != nil {
t.Fatalf("RevertTidyCandidate: %v", err)
}
if got := loadTidyItemRow(t, store, bad.ID); got.kind != before.kind {
t.Fatalf("回滚后 kind = %q, want %q(还原成候选证据里的旧值)", got.kind, before.kind)
}
}
// TestTidyApplyLinkIsNotApplyable 钉住⑤:link 没有执行语义(design/23 §14:
// 链接的存储形态未定),应用它只能得到 skipped,且**零改动**。
func TestTidyApplyLinkIsNotApplyable(t *testing.T) {
t.Parallel()
ctx := context.Background()
store := newTestStore(t)
base := tidyTestTime(t)
seedTidyItem(t, store, habitItem(fixtureEnergyRedBull, base, 0.95))
seedTidyItem(t, store, habitItem(fixtureEnergyIced, base.Add(time.Minute), 0.95))
tr := newTestTidier(t, store, tidyTestCfg(0.80), stubEmbedder{vecs: map[string]embedding.Vector{
fixtureEnergyRedBull: {1, 0},
fixtureEnergyIced: simVector(0.99),
}})
if _, err := tr.RunOnce(ctx); err != nil {
t.Fatalf("RunOnce: %v", err)
}
cands, err := store.ListTidyCandidates(ctx, "", 50)
if err != nil {
t.Fatalf("ListTidyCandidates: %v", err)
}
links := candidatesByAction(cands, TidyLink)
if len(links) != 1 {
t.Fatalf("link 候选 = %d, want 1", len(links))
}
before := dumpTidyTable(t, store, "memory_items", memoryItemsColumns)
out, err := store.ApplyTidyCandidate(ctx, links[0].ID, TidyApplyOptions{Actor: "owner:test"})
if err != nil {
t.Fatalf("ApplyTidyCandidate(link): %v", err)
}
if out.Status != TidyApplySkipped || out.Reason != tidySkipActionNotApplyable {
t.Fatalf("link 应用结果 = %+v, want skipped/%s", out, tidySkipActionNotApplyable)
}
if len(out.AffectedIDs) != 0 {
t.Fatalf("不可应用的动作不得声称改过行:%v", out.AffectedIDs)
}
if got := dumpTidyTable(t, store, "memory_items", memoryItemsColumns); len(got) != len(before) {
t.Fatalf("link 应用改动了记忆:%d → %d 行", len(before), len(got))
}
if cand, err := store.GetTidyCandidate(ctx, links[0].ID); err != nil || cand.Status != tidyCandidatePending {
t.Fatalf("不可应用的候选应保持 pending,实际 %q (err=%v)", cand.Status, err)
}
}
// TestTidyApplyBatchByRunHonoursLimitAndReportsTruncated 钉住⑥:按 run 批量应用
// 逐条给结果、上限生效、超限如实报 Truncated + Total,并把该轮留痕的 applied 累加。
func TestTidyApplyBatchByRunHonoursLimitAndReportsTruncated(t *testing.T) {
t.Parallel()
ctx := context.Background()
store := newTestStore(t)
base := tidyTestTime(t)
const oneOffs = 5
ids := make([]string, 0, oneOffs)
for i := 0; i < oneOffs; i++ {
item := habitItem(fixtureOneOffEvent, base.Add(time.Duration(i)*time.Minute), 0.9)
ids = append(ids, seedTidyItem(t, store, item).ID)
}
tr := newTestTidier(t, store, tidyTestCfg(0.80), nil)
stats, err := tr.RunOnce(ctx)
if err != nil {
t.Fatalf("RunOnce: %v", err)
}
if stats.ByAction[string(TidyArchive)] != oneOffs {
t.Fatalf("ByAction[archive] = %d, want %d(%+v)", stats.ByAction[string(TidyArchive)], oneOffs, stats.ByAction)
}
batch, err := store.ApplyTidyCandidates(ctx, stats.RunID, TidyApplyOptions{Actor: "owner:test", Limit: 2})
if err != nil {
t.Fatalf("ApplyTidyCandidates: %v", err)
}
if batch.Total != oneOffs || len(batch.Results) != 2 || !batch.Truncated {
t.Fatalf("批量回执 = %+v, want total=%d results=2 truncated=true", batch, oneOffs)
}
if batch.Applied != 2 || batch.Archived != 2 || batch.Skipped != 0 || batch.Failed != 0 {
t.Fatalf("批量计数 = %+v, want applied=2 archived=2", batch)
}
for _, res := range batch.Results {
if res.Status != TidyApplyApplied || res.Action != TidyArchive || len(res.AffectedIDs) != 1 {
t.Fatalf("逐条结果 = %+v, want 每条 applied/archive 带 1 个 affected id", res)
}
}
archived := 0
for _, id := range ids {
if loadTidyItemRow(t, store, id).status == tidyStatusArchived {
archived++
}
}
if archived != 2 {
t.Fatalf("实际归档 = %d, want 2", archived)
}
runs, err := store.ListTidyRuns(ctx, 10)
if err != nil {
t.Fatalf("ListTidyRuns: %v", err)
}
if len(runs) != 1 || runs[0].Applied != 2 {
t.Fatalf("运行留痕 applied = %d (runs=%d), want 2", runs[0].Applied, len(runs))
}
// 剩余 3 条:第二批应用,超限标记消失(Total 已 ≤ 上限)。
batch2, err := store.ApplyTidyCandidates(ctx, stats.RunID, TidyApplyOptions{Actor: "owner:test", Limit: 2})
if err != nil {
t.Fatalf("第二批: %v", err)
}
if batch2.Total != oneOffs-2 || !batch2.Truncated || batch2.Applied != 2 {
t.Fatalf("第二批回执 = %+v, want total=3 truncated=true applied=2", batch2)
}
// runId 拼错必须是「查不到这一轮」,不是「这一轮没有待定候选」。
if _, err := store.ApplyTidyCandidates(ctx, "no-such-run", TidyApplyOptions{Actor: "owner:test"}); !errors.Is(err, ErrTidyRunRecordNotFound) {
t.Fatalf("未知 runId 的 err = %v, want ErrTidyRunRecordNotFound", err)
}
// runId 为空 ⇒ 「全部待定应用」这一档(没有轮次筛)。
rest, err := store.ApplyTidyCandidates(ctx, "", TidyApplyOptions{Actor: "owner:test"})
if err != nil {
t.Fatalf("全部待定应用: %v", err)
}
if rest.Total != 1 || rest.Applied != 1 || rest.Archived != 1 {
t.Fatalf("全部待定回执 = %+v, want total=1 applied=1 archived=1", rest)
}
}
// jsonHasID 报告一个 JSON 数组文本里是否含 id。
func jsonHasID(t *testing.T, jsonText, id string) bool {
t.Helper()
var ids []string
if err := json.Unmarshal([]byte(jsonText), &ids); err != nil {
t.Fatalf("解析 %q: %v", jsonText, err)
}
return containsID(ids, id)
}