feat(events): 渠道事件存储地基（Event/Store + SQLite 实现 + 有界计数聚合 + 保留期 GC + 迁移 0021）
- 新包 internal/runtime/events：Tier/Category/Platform/ChannelType/ActorRole 具名常量 + Event/Filter/Counter + Store 六方法 - 留存口径：T2/T3 逐条落行；T1 只计数；unknown / decode_error / link 三例外逐条落行（链路是否活着是事故定案唯一证据） - 计数：内存聚合 + 上限 512 的有界缓冲 + 满即整批 upsert，落盘失败回填而不丢；GC：保留期 720h/间隔 1h，≤0 = 关闭 - 迁移 0021_channel_events（两表 + 6 索引，含计数 upsert 唯一键）；Event 无 flag/正文/URL 字段（design/26 §4.2 红线）
This commit is contained in:
AskaEth committed
1 parent
52dac4575f
commit
ec3f6e810a
10 files changed
+2172
No files matched your search
@@ -0,0 +1,175 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// channelEventsMigrationID 是渠道事件留存两张表的迁移 id（design/26 §4.1，切片 E1）。
|
||||
const channelEventsMigrationID = "0021_channel_events"
|
||||
|
||||
// schemaObjectExists 报告 sqlite_master 里是否存在某类对象（table / index）。
|
||||
// 与 columnExists 同族，都只读元数据、不碰 data/cyrene.db。
|
||||
func schemaObjectExists(t *testing.T, ctx context.Context, conn *sql.DB, kind, name string) bool {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := conn.QueryRowContext(ctx,
|
||||
"SELECT count(*) FROM sqlite_master WHERE type = ? AND name = ?", kind, name).Scan(&n); err != nil {
|
||||
t.Fatalf("sqlite_master(%s %s): %v", kind, name, err)
|
||||
}
|
||||
return n == 1
|
||||
}
|
||||
|
||||
// TestMigrateChannelEventsTablesReversible 钉住 0021（channel_events +
|
||||
// channel_event_counters，design/26 §4.1）：up 建两张表与全部索引、重复 up 是空操作、
|
||||
// down 整表消失、再 up 恢复端到端可用。它用自己的内存库，绝不碰 data/cyrene.db。
|
||||
//
|
||||
// 断言的是**库里的结构事实**（表/列/索引在不在、默认值读出来是什么），不按日志文本
|
||||
// 断言 —— 与 migrate_turn_id_test.go / migrate_document_test.go 同一口径。
|
||||
func TestMigrateChannelEventsTablesReversible(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
conn := openMemory(t)
|
||||
|
||||
if err := MigrateUp(ctx, conn, testLogger()); err != nil {
|
||||
t.Fatalf("MigrateUp: %v", err)
|
||||
}
|
||||
|
||||
// 两张表与它们的列必须逐个到位（列名错一个，写入面就会静默写错地方）。
|
||||
columns := map[string][]string{
|
||||
"channel_events": {
|
||||
"id", "platform", "account_id", "tier", "category", "platform_type",
|
||||
"channel_id", "channel_type", "group_id", "actor_id", "actor_role",
|
||||
"target_id", "self_related", "message_id", "summary", "created_at", "updated_at",
|
||||
},
|
||||
"channel_event_counters": {
|
||||
"id", "platform", "account_id", "tier", "platform_type", "category",
|
||||
"count", "first_seen_at", "last_seen_at",
|
||||
},
|
||||
}
|
||||
for table, cols := range columns {
|
||||
if !schemaObjectExists(t, ctx, conn, "table", table) {
|
||||
t.Fatalf("%s should exist after MigrateUp", table)
|
||||
}
|
||||
for _, col := range cols {
|
||||
if !columnExists(t, ctx, conn, table, col) {
|
||||
t.Fatalf("%s.%s should exist after MigrateUp", table, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 索引：查询面与 GC 依赖 idx_channel_events_created_at，upsert 依赖计数表的唯一键。
|
||||
for _, idx := range []string{
|
||||
"idx_channel_events_created_at",
|
||||
"idx_channel_events_platform_category",
|
||||
"idx_channel_events_channel_id",
|
||||
"idx_channel_events_tier",
|
||||
"idx_channel_event_counters_key",
|
||||
"idx_channel_event_counters_category",
|
||||
} {
|
||||
if !schemaObjectExists(t, ctx, conn, "index", idx) {
|
||||
t.Fatalf("%s should exist after MigrateUp", idx)
|
||||
}
|
||||
}
|
||||
|
||||
// 默认值：只写必填列也能落行，可选列读回来是 ''/0，时间戳由 SQLite 默认填。
|
||||
if _, err := conn.ExecContext(ctx,
|
||||
`INSERT INTO channel_events (platform) VALUES (?)`, "qq-obv11"); err != nil {
|
||||
t.Fatalf("insert minimal channel event: %v", err)
|
||||
}
|
||||
var (
|
||||
tier, category, summary, createdAt string
|
||||
selfRelated int
|
||||
)
|
||||
if err := conn.QueryRowContext(ctx,
|
||||
`SELECT tier, category, summary, self_related, created_at FROM channel_events WHERE platform = ?`,
|
||||
"qq-obv11").Scan(&tier, &category, &summary, &selfRelated, &createdAt); err != nil {
|
||||
t.Fatalf("read minimal channel event: %v", err)
|
||||
}
|
||||
if tier != "" || category != "" || summary != "" || selfRelated != 0 {
|
||||
t.Fatalf("defaults = (%q, %q, %q, %d), want ('', '', '', 0)", tier, category, summary, selfRelated)
|
||||
}
|
||||
if createdAt == "" {
|
||||
t.Fatal("created_at should be filled by the migration default")
|
||||
}
|
||||
|
||||
// 重复 up：已记录的迁移被跳过，结构仍在且不报错（幂等）。
|
||||
if err := MigrateUp(ctx, conn, testLogger()); err != nil {
|
||||
t.Fatalf("MigrateUp second call: %v", err)
|
||||
}
|
||||
if !schemaObjectExists(t, ctx, conn, "table", "channel_events") {
|
||||
t.Fatal("channel_events should still exist after idempotent re-run")
|
||||
}
|
||||
|
||||
// 恰好回滚到 0021（含它之后新加的迁移），日后再加迁移也不会让本测试失效。
|
||||
ids := upMigrationIDs(t)
|
||||
idx := -1
|
||||
for i, id := range ids {
|
||||
if id == channelEventsMigrationID {
|
||||
idx = i
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
t.Fatalf("%s not found in %v", channelEventsMigrationID, ids)
|
||||
}
|
||||
if err := MigrateDown(ctx, conn, len(ids)-idx, testLogger()); err != nil {
|
||||
t.Fatalf("MigrateDown(%d): %v", len(ids)-idx, err)
|
||||
}
|
||||
for _, table := range []string{"channel_events", "channel_event_counters"} {
|
||||
if schemaObjectExists(t, ctx, conn, "table", table) {
|
||||
t.Fatalf("%s should be dropped by the 0021 down", table)
|
||||
}
|
||||
}
|
||||
|
||||
// 重新 up 恢复端到端可用。
|
||||
if err := MigrateUp(ctx, conn, testLogger()); err != nil {
|
||||
t.Fatalf("re-MigrateUp: %v", err)
|
||||
}
|
||||
for _, table := range []string{"channel_events", "channel_event_counters"} {
|
||||
if !schemaObjectExists(t, ctx, conn, "table", table) {
|
||||
t.Fatalf("%s should exist again after re-MigrateUp", table)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMigrate0021CountersUniqueKey_正常 钉住计数表唯一键的**行为**（不只是「索引
|
||||
// 存在」）：同一 `(platform, account_id, platform_type)` 二次 upsert 必须累加成
|
||||
// **一行**，否则「上游推过几次」的答案会分裂成多行、计数上界也随之失控
|
||||
// （design/26 §4.1 表二的全部理由）。
|
||||
func TestMigrate0021CountersUniqueKey_正常(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
conn := openMemory(t)
|
||||
|
||||
if err := MigrateUp(ctx, conn, testLogger()); err != nil {
|
||||
t.Fatalf("MigrateUp: %v", err)
|
||||
}
|
||||
|
||||
const upsert = `INSERT INTO channel_event_counters
|
||||
(platform, account_id, tier, platform_type, category, count, first_seen_at, last_seen_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(platform, account_id, platform_type) DO UPDATE SET
|
||||
count = count + excluded.count,
|
||||
last_seen_at = excluded.last_seen_at`
|
||||
for _, count := range []int{1, 2} {
|
||||
if _, err := conn.ExecContext(ctx, upsert,
|
||||
"qq-obv11", "bot1", "t1", "heartbeat", "heartbeat", count,
|
||||
"2026-10-05T00:00:00.000Z", "2026-10-05T00:00:00.000Z"); err != nil {
|
||||
t.Fatalf("upsert counter (%d): %v", count, err)
|
||||
}
|
||||
}
|
||||
|
||||
var rows, total int
|
||||
if err := conn.QueryRowContext(ctx,
|
||||
`SELECT count(*), COALESCE(sum(count), 0) FROM channel_event_counters WHERE platform = ?`,
|
||||
"qq-obv11").Scan(&rows, &total); err != nil {
|
||||
t.Fatalf("read counters: %v", err)
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Fatalf("counter rows = %d, want 1 (the unique key must merge, not duplicate)", rows)
|
||||
}
|
||||
if total != 3 {
|
||||
t.Fatalf("counter total = %d, want 3 (1 + 2)", total)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
-- 0021 的回滚：先删索引、再删表（与 0019 的回滚同一顺序口径）。
|
||||
-- 两张表都是本迁移新建的，回滚不触碰任何既有表。
|
||||
DROP INDEX IF EXISTS idx_channel_event_counters_category;
|
||||
DROP INDEX IF EXISTS idx_channel_event_counters_key;
|
||||
DROP TABLE IF EXISTS channel_event_counters;
|
||||
DROP INDEX IF EXISTS idx_channel_events_tier;
|
||||
DROP INDEX IF EXISTS idx_channel_events_channel_id;
|
||||
DROP INDEX IF EXISTS idx_channel_events_platform_category;
|
||||
DROP INDEX IF EXISTS idx_channel_events_created_at;
|
||||
DROP TABLE IF EXISTS channel_events;
|
||||
@@ -0,0 +1,71 @@
|
||||
-- 0021_channel_events: 渠道非消息事件的**留存地基**（design/26 §4.1；切片 E1）。
|
||||
--
|
||||
-- 动因（design/26 §0.2，已确证事故）：qq_obv11 的入站总闸曾把 post_type != message
|
||||
-- 的帧零日志、零计数地丢掉，于是「上游推过没有 / 是不是被我们过滤了」在数据里
|
||||
-- 再也答不上来。本表就是那块地基：让到达适配器的每一帧至少留下可查的结构级痕迹。
|
||||
--
|
||||
-- 两张表**刻意不合并**（design/26 §4.1 已论证）：
|
||||
-- ① 写放大 —— 心跳 / client_status / 在线状态是高频遥测，逐帧一行会把库写胖；
|
||||
-- 计数表按「平台+账号+平台原生类型」upsert，行数上界是「平台数 × 事件类型数」。
|
||||
-- ② 保留期不同 —— channel_events 有审计价值、按保留期清扫；
|
||||
-- 计数表本身就是「历史已浓缩成一行」的形态，**删了就再也答不上「推过没有」**，
|
||||
-- 因此它保留 first_seen_at / last_seen_at，**不参与 GC**。
|
||||
-- ③ 两张表不同构（一张逐条、一张聚合），不是「同一份东西抄了两遍」。
|
||||
-- T3 与 T2 的列**完全相同**（只差 tier 取值），故合在同一张表里：拆开会让「单一
|
||||
-- 查询面 / 单一保留期 / 单一 GC」变成三份实现（design/26 §4.1）。
|
||||
--
|
||||
-- 命名口径：表名复数、主键 id、时间戳 created_at/updated_at、索引 idx_{table}_{col}
|
||||
-- （援引 AGENTS §10 已登记例外，与既有迁移一致）。时间戳沿用 0001 起的
|
||||
-- `strftime('%Y-%m-%dT%H:%M:%fZ','now')`（UTC ISO-8601 毫秒），定宽 ⇒ 字典序比较
|
||||
-- 等价于时间序，时间窗筛选因此能直接吃上 idx_channel_events_created_at。
|
||||
--
|
||||
-- 脱敏（design/26 §4.2，**表结构层面也留了痕**）：本表**没有**承载平台原文的列 ——
|
||||
-- summary 只存适配器按事件类型生成的**固定文案**，OneBot request.flag（可执行凭据）
|
||||
-- 与平台正文**绝不入库**（故本迁移里连一个 flag/comment/raw/content 列都不存在）。
|
||||
|
||||
CREATE TABLE IF NOT EXISTS channel_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
platform TEXT NOT NULL, -- qq-obv11 / kook / telegram / weixin-ilink / webhook
|
||||
account_id TEXT NOT NULL DEFAULT '', -- 多账号（CHANNELS_JSON 的账号 id）；空 = 单账号/未知
|
||||
tier TEXT NOT NULL DEFAULT '', -- t1 / t2 / t3（design/26 §2 的留存档）
|
||||
category TEXT NOT NULL DEFAULT '', -- 归一化类别（join/leave/recall/ban/admin/friend_request/
|
||||
-- group_request/poke/reaction/button/item/vote/link/unknown）
|
||||
platform_type TEXT NOT NULL DEFAULT '', -- 平台原生类型码（notice_type / meta_event_type / request_type /
|
||||
-- KOOK extra.type / Telegram update 键名），原样保留便于排障
|
||||
channel_id TEXT NOT NULL DEFAULT '',
|
||||
channel_type TEXT NOT NULL DEFAULT '', -- direct / group
|
||||
group_id TEXT NOT NULL DEFAULT '',
|
||||
actor_id TEXT NOT NULL DEFAULT '', -- 结构化字段，**绝不来自文本**
|
||||
actor_role TEXT NOT NULL DEFAULT '', -- owner/member/system/unknown（结构化解析所得）
|
||||
target_id TEXT NOT NULL DEFAULT '', -- 被作用者（poke / 禁言 / 运气王…）
|
||||
self_related INTEGER NOT NULL DEFAULT 0, -- 是否命中 self_id / 她自己
|
||||
message_id TEXT NOT NULL DEFAULT '', -- 关联消息 id（撤回 / 回应 / 按钮 / 置顶）
|
||||
summary TEXT NOT NULL DEFAULT '', -- 脱敏结构级摘要（固定文案，不含平台原文）
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
||||
);
|
||||
|
||||
-- 查询面与 GC 的**唯一依赖**：列表倒序、时间窗筛选、过期行清扫都靠它。
|
||||
CREATE INDEX IF NOT EXISTS idx_channel_events_created_at ON channel_events (created_at);
|
||||
-- 面板/端点的「平台 + 类别」筛选（design/26 §4.5 的查询参数）。
|
||||
CREATE INDEX IF NOT EXISTS idx_channel_events_platform_category ON channel_events (platform, category);
|
||||
CREATE INDEX IF NOT EXISTS idx_channel_events_channel_id ON channel_events (channel_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_channel_events_tier ON channel_events (tier);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS channel_event_counters (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
platform TEXT NOT NULL,
|
||||
account_id TEXT NOT NULL DEFAULT '',
|
||||
tier TEXT NOT NULL DEFAULT '',
|
||||
platform_type TEXT NOT NULL DEFAULT '',
|
||||
category TEXT NOT NULL DEFAULT '',
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
||||
last_seen_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
||||
);
|
||||
|
||||
-- 唯一约束承载 upsert（`ON CONFLICT(platform,account_id,platform_type) DO UPDATE`）：
|
||||
-- 一次 flush 因此是同键累加，而不是每次心跳插一行。
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_channel_event_counters_key
|
||||
ON channel_event_counters (platform, account_id, platform_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_channel_event_counters_category ON channel_event_counters (category);
|
||||
Reference in new issue
Block a user