fix(memory): 定宽时间戳存储 + rowid 确定性排序 + 遗留数据归一化

- formatTime 改用固定 30 字节布局(9 位小数秒),让 TEXT 列的字典序与时间序一致(RFC3339Nano 会去掉尾随零产生变宽字符串,秒内会反序)
- memory_items / project / project_chapter 的 ORDER BY 全部补 rowid 兜底,消除同 tick 打平时的顺序不确定
- 开库时幂等归一化变宽的历史值;无法解析的行原样保留,不阻塞开库
- 回归测试:变宽排序反例、同一时间戳插入序、归一化幂等与跳过
This commit is contained in:
AskaEth committed 2026-10-03 16:08:50 +08:00
1 parent 1ac126fc9e
commit 31086ed1ab
10 files changed
+745 -61

No files matched your search

@@ -61,8 +61,10 @@ func (s *Store) AddChapter(ctx context.Context, projectID string, ch Chapter) er
return nil
}
// ListChapters returns every chapter for projectID in insertion order. It returns
// an empty (non-nil) slice when there are none; a blank projectID returns
// ListChapters returns every chapter for projectID in insertion order
// (created_at ASC, rowid ASC) — rowid breaks created_at ties so chapters written
// within the same clock tick keep their insertion order. It returns an empty
// (non-nil) slice when there are none; a blank projectID returns
// ErrMissingProject.
func (s *Store) ListChapters(ctx context.Context, projectID string) ([]Chapter, error) {
if strings.TrimSpace(projectID) == "" {
@@ -70,7 +72,7 @@ func (s *Store) ListChapters(ctx context.Context, projectID string) ([]Chapter,
}
rows, err := s.db.QueryContext(ctx, `SELECT id, project_id, title, content,
created_at, updated_at FROM project_chapter
WHERE project_id = ? ORDER BY created_at ASC, id ASC`, projectID)
WHERE project_id = ? ORDER BY created_at ASC, rowid ASC`, projectID)
if err != nil {
return nil, fmt.Errorf("projectindex: list chapters: %w", err)
}
@@ -142,21 +142,3 @@ func newID() (string, error) {
}
return hex.EncodeToString(b), nil
}
// formatTime encodes t in UTC as an RFC3339Nano string so ordering and
// round-trips are stable across processes.
func formatTime(t time.Time) string {
return t.UTC().Format(time.RFC3339Nano)
}
// parseTime decodes a stored RFC3339 string; blank input yields a zero time.
func parseTime(s string) (time.Time, error) {
if strings.TrimSpace(s) == "" {
return time.Time{}, nil
}
t, err := time.Parse(time.RFC3339Nano, s)
if err != nil {
return time.Time{}, fmt.Errorf("projectindex: parse time %q: %w", s, err)
}
return t, nil
}
+11 -5
View File
@@ -79,7 +79,8 @@ type rowScanner interface {
// internal/db). A nil db returns ErrNilDB. log may be nil, in which case a
// discard logger is used; the active logger is tagged with source
// "memory/projectindex" so every record is traceable. ctx controls schema
// creation and must be honored by the caller.
// creation and must be honored by the caller. Legacy variable-width timestamps
// are normalized in place (see normalizeTimestamps).
func New(ctx context.Context, db *sql.DB, log *slog.Logger) (*Store, error) {
if db == nil {
return nil, ErrNilDB
@@ -94,6 +95,9 @@ func New(ctx context.Context, db *sql.DB, log *slog.Logger) (*Store, error) {
return nil, fmt.Errorf("projectindex: apply schema: %w", err)
}
}
if err := normalizeTimestamps(ctx, db); err != nil {
return nil, err
}
return &Store{db: db, log: log}, nil
}
@@ -106,12 +110,14 @@ func discardLogger(log *slog.Logger) *slog.Logger {
}
// ListProjects returns every project index entry in insertion order (created_at
// ASC, id ASC). An empty (non-nil) slice is returned when there are none, so
// callers can range over it unconditionally. A read error is wrapped with %w so
// callers can use errors.Is against the package sentinels.
// ASC, rowid ASC) — rowid breaks created_at ties so projects written within the
// same clock tick keep their insertion order. An empty (non-nil) slice is
// returned when there are none, so callers can range over it unconditionally. A
// read error is wrapped with %w so callers can use errors.Is against the package
// sentinels.
func (s *Store) ListProjects(ctx context.Context) ([]Project, error) {
rows, err := s.db.QueryContext(ctx, `SELECT id, title, desc, created_at, updated_at
FROM project ORDER BY created_at ASC, id ASC`)
FROM project ORDER BY created_at ASC, rowid ASC`)
if err != nil {
return nil, fmt.Errorf("projectindex: list projects: %w", err)
}
@@ -0,0 +1,141 @@
package projectindex
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
)
// timeLayout is the fixed-width timestamp layout used for every stored
// created_at/updated_at value. Unlike time.RFC3339Nano it never drops trailing
// fractional zeros, so every value is exactly storedTimeLen bytes wide; SQLite
// compares these TEXT columns as plain strings and equal width is what makes
// that comparison agree with chronological order. UTC values render the zone as
// a single "Z".
const timeLayout = "2006-01-02T15:04:05.000000000Z07:00"
// storedTimeLen is the byte length of a UTC value formatted with timeLayout:
// "2006-01-02T15:04:05.000000000Z" — the Z07:00 zone collapses to "Z" in UTC.
const storedTimeLen = 30
// Table names normalized on open; they must match the DDL in store.go.
const (
projectTable = "project"
projectChapterTable = "project_chapter"
)
// formatTime encodes t in UTC with the fixed-width timeLayout. Always writing
// UTC keeps ordering and round-trips stable across processes, and the fixed
// width is what lets ORDER BY created_at order these TEXT columns by time.
func formatTime(t time.Time) string {
return t.UTC().Format(timeLayout)
}
// parseTime decodes a stored timestamp. time.RFC3339Nano accepts both the
// fixed-width values this package writes and the legacy variable-width ones, so
// the same parser serves reads and the open-time normalization. Blank input
// yields a zero time.
func parseTime(s string) (time.Time, error) {
if strings.TrimSpace(s) == "" {
return time.Time{}, nil
}
t, err := time.Parse(time.RFC3339Nano, s)
if err != nil {
return time.Time{}, fmt.Errorf("projectindex: parse time %q: %w", s, err)
}
return t, nil
}
// normalizeTimestamps rewrites every project / project_chapter timestamp that is
// not storedTimeLen bytes wide into the fixed-width timeLayout. Rows written
// before the layout existed carry variable-width RFC3339Nano text whose string
// order is not its time order; this brings them in line.
//
// It is idempotent: only rows whose column length differs from storedTimeLen are
// selected, so running it again on a normalized table is a no-op. A row whose
// value does not parse is left exactly as it is — one bad legacy value must not
// keep the store from opening.
func normalizeTimestamps(ctx context.Context, db *sql.DB) error {
for _, table := range []string{projectTable, projectChapterTable} {
for _, col := range []string{"created_at", "updated_at"} {
if err := normalizeTimeColumn(ctx, db, table, col); err != nil {
return err
}
}
}
return nil
}
// normalizeTimeColumn rewrites every table.col value that is not storedTimeLen
// bytes wide. table and col are compile-time constants, never user input, so
// splicing them into the statement is safe; the length comparison and every
// value written are parameterized. All rewrites are collected before any UPDATE
// runs, so no write happens while the SELECT cursor is still open.
func normalizeTimeColumn(ctx context.Context, db *sql.DB, table, col string) error {
fixes, err := collectLegacyTimestamps(ctx, db, table, col)
if err != nil {
return err
}
for _, f := range fixes {
if _, err := db.ExecContext(ctx,
"UPDATE "+f.table+" SET "+f.col+" = ? WHERE id = ?", f.value, f.id); err != nil {
return fmt.Errorf("projectindex: normalize %s.%s: %w", f.table, f.col, err)
}
}
return nil
}
// timeFix is one pending timestamp rewrite: which table and column, which row
// (by primary key id), and the fixed-width value to write.
type timeFix struct {
table string
col string
id string
value string
}
// collectLegacyTimestamps reads the rows of table.col whose value is not
// storedTimeLen bytes wide and returns the parseable ones as pending rewrites.
// Unparseable values are skipped, so normalization can never fail a row: the
// only errors are genuine read failures.
func collectLegacyTimestamps(ctx context.Context, db *sql.DB, table, col string) ([]timeFix, error) {
rows, err := db.QueryContext(ctx,
"SELECT id, "+col+" FROM "+table+" WHERE length("+col+") != ?", storedTimeLen)
if err != nil {
return nil, fmt.Errorf("projectindex: query legacy %s.%s: %w", table, col, err)
}
defer func() { _ = rows.Close() }()
fixes := make([]timeFix, 0)
for rows.Next() {
var id, raw string
if err := rows.Scan(&id, &raw); err != nil {
return nil, fmt.Errorf("projectindex: scan legacy %s.%s: %w", table, col, err)
}
fixed, ok := reformatTime(raw)
if !ok {
continue
}
fixes = append(fixes, timeFix{table: table, col: col, id: id, value: fixed})
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("projectindex: iterate legacy %s.%s: %w", table, col, err)
}
return fixes, nil
}
// reformatTime re-encodes raw with timeLayout. The bool is false when raw is
// already storedTimeLen wide or when it does not parse, in which case the caller
// leaves the stored value untouched.
func reformatTime(raw string) (string, bool) {
if len(raw) == storedTimeLen {
return "", false
}
t, err := parseTime(raw)
if err != nil || t.IsZero() {
return "", false
}
return formatTime(t), true
}
@@ -0,0 +1,187 @@
package projectindex
import (
"context"
"testing"
"time"
)
// insertLegacyChapterSQL writes a project_chapter row with caller-supplied raw
// timestamps, standing in for a row persisted before the fixed-width layout.
const insertLegacyChapterSQL = `INSERT INTO project_chapter
(id, project_id, title, content, created_at, updated_at)
VALUES (?, 'proj.1', '旧章节', '旧内容', ?, ?)`
func TestFormatTime_定宽_正常(t *testing.T) {
t.Parallel()
tests := []struct {
name string
at time.Time
want string
}{
{"一位小数秒", time.Date(2026, 9, 7, 3, 20, 5, 830000000, time.UTC), "2026-09-07T03:20:05.830000000Z"},
{"纳秒", time.Date(2026, 9, 7, 3, 20, 5, 830978900, time.UTC), "2026-09-07T03:20:05.830978900Z"},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := formatTime(tt.at)
if got != tt.want {
t.Errorf("formatTime = %q, want %q", got, tt.want)
}
if len(got) != storedTimeLen {
t.Errorf("formatTime length = %d, want %d", len(got), storedTimeLen)
}
})
}
}
// TestStore_ListChapters_变宽时间戳_排序 is the projectindex regression for the
// variable-width timestamp bug: RFC3339Nano would store these as ":05.83" and
// ":05.8309789", whose text order is the reverse of their time order.
func TestStore_ListChapters_变宽时间戳_排序(t *testing.T) {
t.Parallel()
ctx := context.Background()
s := newTestStore(t)
sec := time.Date(2026, 9, 7, 3, 20, 5, 0, time.UTC)
older := sec.Add(830 * time.Millisecond)
newer := older.Add(978900 * time.Nanosecond)
// The newer chapter is inserted first, so a rowid-only tiebreak would put it
// last: only correctly ordered created_at text places it last-but-one.
if err := s.AddChapter(ctx, "proj.1", Chapter{
Title: "较晚", Content: "x", CreatedAt: newer, UpdatedAt: newer,
}); err != nil {
t.Fatalf("AddChapter(较晚): %v", err)
}
if err := s.AddChapter(ctx, "proj.1", Chapter{
Title: "较早", Content: "y", CreatedAt: older, UpdatedAt: older,
}); err != nil {
t.Fatalf("AddChapter(较早): %v", err)
}
var raw string
if err := s.db.QueryRowContext(ctx,
`SELECT created_at FROM project_chapter WHERE title = ?`, "较早").Scan(&raw); err != nil {
t.Fatalf("read stored created_at: %v", err)
}
if len(raw) != storedTimeLen {
t.Errorf("stored created_at = %q (%d bytes), want fixed width %d", raw, len(raw), storedTimeLen)
}
list, err := s.ListChapters(ctx, "proj.1")
if err != nil {
t.Fatalf("ListChapters: %v", err)
}
if len(list) != 2 {
t.Fatalf("ListChapters = %d chapters, want 2", len(list))
}
if list[0].Title != "较早" || list[1].Title != "较晚" {
t.Errorf("titles = [%q %q], want oldest first (较早, 较晚)", list[0].Title, list[1].Title)
}
}
// TestStore_ListChapters_同一时间戳_插入序 covers the tie case: consecutive
// AddChapter calls can share one clock tick, so rowid ASC must keep them in
// insertion order.
func TestStore_ListChapters_同一时间戳_插入序(t *testing.T) {
t.Parallel()
ctx := context.Background()
s := newTestStore(t)
at := time.Date(2026, 9, 7, 3, 20, 5, 830978900, time.UTC)
for _, title := range []string{"第一章", "第二章", "第三章"} {
if err := s.AddChapter(ctx, "proj.1", Chapter{
Title: title, Content: "内容", CreatedAt: at, UpdatedAt: at,
}); err != nil {
t.Fatalf("AddChapter(%s): %v", title, err)
}
}
list, err := s.ListChapters(ctx, "proj.1")
if err != nil {
t.Fatalf("ListChapters: %v", err)
}
want := []string{"第一章", "第二章", "第三章"}
if len(list) != len(want) {
t.Fatalf("ListChapters = %d chapters, want %d", len(list), len(want))
}
for i, w := range want {
if list[i].Title != w {
t.Errorf("chapters[%d].Title = %q, want %q (rowid ASC)", i, list[i].Title, w)
}
}
}
// TestNew_归一化历史时间戳_正常 writes legacy variable-width rows into both
// tables, re-opens the store to trigger normalization, and checks the repaired
// text plus idempotence and the unparseable-row skip.
func TestNew_归一化历史时间戳_正常(t *testing.T) {
t.Parallel()
ctx := context.Background()
s := newTestStore(t)
const legacy = "2026-09-07T03:20:05.83Z"
if _, err := s.db.ExecContext(ctx, `INSERT INTO project (id, title, desc, created_at, updated_at)
VALUES ('proj.legacy', 'T', '', ?, ?)`, legacy, legacy); err != nil {
t.Fatalf("insert legacy project: %v", err)
}
if _, err := s.db.ExecContext(ctx, insertLegacyChapterSQL, "ch.legacy", legacy, legacy); err != nil {
t.Fatalf("insert legacy chapter: %v", err)
}
if _, err := s.db.ExecContext(ctx, insertLegacyChapterSQL, "ch.broken", "not-a-time", "not-a-time"); err != nil {
t.Fatalf("insert unparseable chapter: %v", err)
}
// Re-opening normalizes both tables; an unparseable value must not block it.
if _, err := New(ctx, s.db, testLogger()); err != nil {
t.Fatalf("New (normalize): %v", err)
}
const want = "2026-09-07T03:20:05.830000000Z"
for _, tc := range []struct{ name, query, id string }{
{"project", `SELECT created_at FROM project WHERE id = ?`, "proj.legacy"},
{"project_chapter", `SELECT created_at FROM project_chapter WHERE id = ?`, "ch.legacy"},
} {
var raw string
if err := s.db.QueryRowContext(ctx, tc.query, tc.id).Scan(&raw); err != nil {
t.Fatalf("read %s created_at: %v", tc.name, err)
}
if raw != want {
t.Errorf("%s created_at = %q, want %q", tc.name, raw, want)
}
}
var broken string
if err := s.db.QueryRowContext(ctx,
`SELECT created_at FROM project_chapter WHERE id = ?`, "ch.broken").Scan(&broken); err != nil {
t.Fatalf("read unparseable created_at: %v", err)
}
if broken != "not-a-time" {
t.Errorf("unparseable created_at = %q, want it left as is", broken)
}
// A further open must be a no-op.
if _, err := New(ctx, s.db, testLogger()); err != nil {
t.Fatalf("New (second pass): %v", err)
}
var again string
if err := s.db.QueryRowContext(ctx,
`SELECT created_at FROM project WHERE id = ?`, "proj.legacy").Scan(&again); err != nil {
t.Fatalf("read created_at after second open: %v", err)
}
if again != want {
t.Errorf("second open changed created_at: %q -> %q", want, again)
}
got, err := s.GetProject(ctx, "proj.legacy")
if err != nil {
t.Fatalf("GetProject: %v", err)
}
wantTime := time.Date(2026, 9, 7, 3, 20, 5, 830000000, time.UTC)
if !got.CreatedAt.Equal(wantTime) {
t.Errorf("project created_at = %v, want %v", got.CreatedAt, wantTime)
}
}