feat(memory): 记忆整理 v1（design/23）——低峰定期扫描 + 事实包含判定 + 规则否决 + 只产建议队列（不自动应用）
This commit is contained in:
AskaEth committed
1 parent
012e9f3ad4
commit
91a3209c9b
25 files changed
+3418
-13
No files matched your search
@@ -0,0 +1,264 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 本文件是记忆整理的存储层：分区扫描、候选落库与查询、运行留痕。它复用
|
||||
// memory 包既有的 *sql.DB 与格式化辅助（formatTime/parseTime/scanItem），
|
||||
// 不新建存储机制。
|
||||
//
|
||||
// 第一版唯一的写入就是候选表与运行表：**没有任何一条 memory_items 会被改写
|
||||
// 或删除**（design/23 §3 非目标 + §7「apply 才写记忆」）。
|
||||
|
||||
// tidyCandidatePending 是候选的初始状态（design/23 §10）。
|
||||
const tidyCandidatePending = "pending"
|
||||
|
||||
// tidyPartition 是整理的隔离单元：design/23 §6 的分区键
|
||||
// (persona_id, scope, project_id)。分区之间**永不比较**。
|
||||
type tidyPartition struct {
|
||||
personaID string
|
||||
scope Scope
|
||||
projectID string
|
||||
}
|
||||
|
||||
// label 渲染分区标签，供日志与截断登记使用（不含任何记忆正文）。
|
||||
func (p tidyPartition) label() string {
|
||||
return fmt.Sprintf("%s/%s/%s", p.personaID, p.scope, p.projectID)
|
||||
}
|
||||
|
||||
// tidyStatusActive 是「在册」状态。整理只扫描在册条目：已合并/已归档的行
|
||||
// 不再参与比对（design §10 status 列）。
|
||||
const tidyStatusActive = "active"
|
||||
|
||||
// listTidyPartitionItemsSQL 按分区读取**在册**（status='active'）条目，
|
||||
// 最新在前；rowid 打破 created_at 并列，使同一刻写入的条目顺序稳定
|
||||
// （与 ListByPersona 同一口径）。分区过滤写在 SQL 里，而不是读全表再在内存
|
||||
// 里筛 —— design/23 §6 要求在分区切分层就隔断。
|
||||
const listTidyPartitionItemsSQL = `SELECT id, persona_id, role_id, project_id, scope, kind,
|
||||
content, tags, confidence, source_conversation, created_at, updated_at,
|
||||
decay_weight FROM memory_items
|
||||
WHERE persona_id = ? AND scope = ? AND project_id = ? AND status = ?
|
||||
ORDER BY created_at DESC, rowid DESC LIMIT ?`
|
||||
|
||||
const listTidyPartitionsSQL = `SELECT DISTINCT persona_id, scope, project_id
|
||||
FROM memory_items WHERE status = ?
|
||||
ORDER BY persona_id, scope, project_id`
|
||||
|
||||
const insertTidyCandidateSQL = `INSERT OR IGNORE INTO memory_tidy_candidates
|
||||
(id, persona_id, scope, project_id, kind, action, source_ids, target_id,
|
||||
proposed_content, merged_content, reason, evidence, rule_version,
|
||||
score_min, score_max, status, idempotency_key, created_at, decided_at, decided_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '', '')`
|
||||
|
||||
const insertTidyRunSQL = `INSERT INTO memory_tidy_runs
|
||||
(id, started_at, finished_at, scanned, candidates, applied, skipped, errors,
|
||||
vetoed, veto_reasons, rule_version)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
|
||||
const listTidyCandidatesSQL = `SELECT id, persona_id, scope, project_id, kind, action,
|
||||
source_ids, target_id, proposed_content, merged_content, reason, evidence,
|
||||
rule_version, score_min, score_max, status, idempotency_key, created_at
|
||||
FROM memory_tidy_candidates
|
||||
WHERE status = ?
|
||||
ORDER BY created_at DESC, rowid DESC LIMIT ?`
|
||||
|
||||
const listTidyRunsSQL = `SELECT id, started_at, finished_at, scanned, candidates,
|
||||
applied, skipped, errors, vetoed, veto_reasons, rule_version
|
||||
FROM memory_tidy_runs ORDER BY started_at DESC, rowid DESC LIMIT ?`
|
||||
|
||||
// listTidyPartitions 返回所有含在册条目的分区。DISTINCT 让每个分区的
|
||||
// 扫描上限（design §9 每分区 5000 条）各自独立。
|
||||
func (s *Store) listTidyPartitions(ctx context.Context) ([]tidyPartition, error) {
|
||||
rows, err := s.db.QueryContext(ctx, listTidyPartitionsSQL, string(tidyStatusActive))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("memory: list tidy partitions: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
parts := make([]tidyPartition, 0)
|
||||
for rows.Next() {
|
||||
var p tidyPartition
|
||||
var scope string
|
||||
if err := rows.Scan(&p.personaID, &scope, &p.projectID); err != nil {
|
||||
return nil, fmt.Errorf("memory: scan tidy partition: %w", err)
|
||||
}
|
||||
p.scope = Scope(scope)
|
||||
parts = append(parts, p)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("memory: iterate tidy partitions: %w", err)
|
||||
}
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
// listTidyPartitionItems 读取一个分区内的在册条目（最新在前，最多 limit 条）。
|
||||
func (s *Store) listTidyPartitionItems(ctx context.Context, p tidyPartition, limit int) ([]MemoryItem, error) {
|
||||
rows, err := s.db.QueryContext(ctx, listTidyPartitionItemsSQL,
|
||||
p.personaID, string(p.scope), p.projectID, string(tidyStatusActive), limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("memory: list tidy partition items: %w", err)
|
||||
}
|
||||
return scanItems(rows, "tidy partition item")
|
||||
}
|
||||
|
||||
// insertTidyCandidate 落库一条候选，返回它是否是**新建**的。幂等键
|
||||
// （design/23 §9）唯一：同一批数据重复运行命中已存在行时返回 (false, nil)，
|
||||
// 绝不重复产候选。
|
||||
func (s *Store) insertTidyCandidate(ctx context.Context, c TidyCandidate) (bool, error) {
|
||||
c.IdempotencyKey = tidyIdempotencyKey(c)
|
||||
sourceJSON, err := json.Marshal(c.SourceIDs)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("memory: encode tidy source ids: %w", err)
|
||||
}
|
||||
evidenceJSON, err := json.Marshal(c.Evidence)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("memory: encode tidy evidence: %w", err)
|
||||
}
|
||||
if c.CreatedAt.IsZero() {
|
||||
c.CreatedAt = time.Now().UTC()
|
||||
}
|
||||
res, err := s.db.ExecContext(ctx, insertTidyCandidateSQL,
|
||||
c.ID, c.PersonaID, string(c.Scope), c.ProjectID, string(c.Kind), string(c.Action),
|
||||
string(sourceJSON), c.TargetID, c.ProposedContent, c.MergedContent, c.Reason,
|
||||
string(evidenceJSON), c.RuleVersion, c.ScoreMin, c.ScoreMax, c.Status,
|
||||
c.IdempotencyKey, formatTime(c.CreatedAt))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("memory: insert tidy candidate: %w", err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("memory: tidy candidate rows affected: %w", err)
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
// persistTidyRun 写一条运行留痕（design/23 §10 memory_tidy_runs）。
|
||||
func (s *Store) persistTidyRun(ctx context.Context, run TidyRun) error {
|
||||
if run.ID == "" {
|
||||
run.ID = mustNewTidyID()
|
||||
}
|
||||
reasons, err := json.Marshal(run.VetoReasons)
|
||||
if err != nil {
|
||||
return fmt.Errorf("memory: encode tidy veto reasons: %w", err)
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, insertTidyRunSQL,
|
||||
run.ID, formatTime(run.StartedAt), formatTime(run.FinishedAt),
|
||||
run.Scanned, run.Candidates, run.Applied, run.Skipped, run.Errors,
|
||||
run.Vetoed, string(reasons), run.RuleVersion); err != nil {
|
||||
return fmt.Errorf("memory: insert tidy run: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListTidyCandidates 返回指定状态的整理候选（最新在前，最多 limit 条，
|
||||
// 非正 limit 取 defaultTidyCandidateLimit）。它是读侧入口：候选正文含隐私，
|
||||
// 只应由 owner-only 的读侧使用（design/23 §11）。
|
||||
func (s *Store) ListTidyCandidates(ctx context.Context, status string, limit int) ([]TidyCandidate, error) {
|
||||
if strings.TrimSpace(status) == "" {
|
||||
status = tidyCandidatePending
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = defaultTidyCandidateLimit
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, listTidyCandidatesSQL, status, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("memory: list tidy candidates: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
out := make([]TidyCandidate, 0)
|
||||
for rows.Next() {
|
||||
c, err := scanTidyCandidate(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("memory: scan tidy candidate: %w", err)
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("memory: iterate tidy candidates: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListTidyRuns 返回整理运行留痕（最新在前，最多 limit 条）。非正 limit 取
|
||||
// 一个保守的默认条数。
|
||||
func (s *Store) ListTidyRuns(ctx context.Context, limit int) ([]TidyRun, error) {
|
||||
if limit <= 0 {
|
||||
limit = defaultTidyCandidateLimit
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, listTidyRunsSQL, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("memory: list tidy runs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
out := make([]TidyRun, 0)
|
||||
for rows.Next() {
|
||||
var (
|
||||
r TidyRun
|
||||
startedAt string
|
||||
finishedAt string
|
||||
reasonsJSON string
|
||||
)
|
||||
if err := rows.Scan(&r.ID, &startedAt, &finishedAt, &r.Scanned, &r.Candidates,
|
||||
&r.Applied, &r.Skipped, &r.Errors, &r.Vetoed, &reasonsJSON, &r.RuleVersion); err != nil {
|
||||
return nil, fmt.Errorf("memory: scan tidy run: %w", err)
|
||||
}
|
||||
var err error
|
||||
if r.StartedAt, err = parseTime(startedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.FinishedAt, err = parseTime(finishedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal([]byte(reasonsJSON), &r.VetoReasons); err != nil {
|
||||
return nil, fmt.Errorf("memory: decode tidy veto reasons: %w", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("memory: iterate tidy runs: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// scanTidyCandidate 读取一行候选：source_ids 与 evidence 以 JSON 存储。
|
||||
func scanTidyCandidate(sc rowScanner) (TidyCandidate, error) {
|
||||
var (
|
||||
c TidyCandidate
|
||||
scope, kind string
|
||||
action string
|
||||
sourceJSON string
|
||||
evidenceRaw string
|
||||
createdAt string
|
||||
)
|
||||
if err := sc.Scan(&c.ID, &c.PersonaID, &scope, &c.ProjectID, &kind, &action,
|
||||
&sourceJSON, &c.TargetID, &c.ProposedContent, &c.MergedContent, &c.Reason,
|
||||
&evidenceRaw, &c.RuleVersion, &c.ScoreMin, &c.ScoreMax, &c.Status,
|
||||
&c.IdempotencyKey, &createdAt); err != nil {
|
||||
return TidyCandidate{}, err
|
||||
}
|
||||
c.Scope = Scope(scope)
|
||||
c.Kind = Kind(kind)
|
||||
c.Action = TidyAction(action)
|
||||
if err := json.Unmarshal([]byte(sourceJSON), &c.SourceIDs); err != nil {
|
||||
return TidyCandidate{}, fmt.Errorf("memory: decode tidy source ids: %w", err)
|
||||
}
|
||||
if c.SourceIDs == nil {
|
||||
c.SourceIDs = []string{}
|
||||
}
|
||||
if err := json.Unmarshal([]byte(evidenceRaw), &c.Evidence); err != nil {
|
||||
return TidyCandidate{}, fmt.Errorf("memory: decode tidy evidence: %w", err)
|
||||
}
|
||||
t, err := parseTime(createdAt)
|
||||
if err != nil {
|
||||
return TidyCandidate{}, err
|
||||
}
|
||||
c.CreatedAt = t
|
||||
return c, nil
|
||||
}
|
||||
Reference in new issue
Block a user