diff --git a/backend/ai-core/internal/background/thinker.go b/backend/ai-core/internal/background/thinker.go index f1a3ade..ddf4e3a 100644 --- a/backend/ai-core/internal/background/thinker.go +++ b/backend/ai-core/internal/background/thinker.go @@ -596,6 +596,8 @@ func (t *Thinker) Start() { if t.lightThinkEnabled && t.lightThinkInterval > 0 { t.wg.Add(1) go t.lightThinkLoop() + // 死锁监控:每30秒检测 t.mu 是否正常 + t.StartDeadlockMonitor() } // 启动平台静默观察循环 @@ -2670,3 +2672,26 @@ func (t *Thinker) DeadlockDetected(timeout time.Duration) bool { func (t *Thinker) LockHolder() (string, time.Time) { return t.muLockedBy, t.muLockedAt } + +// StartDeadlockMonitor 启动死锁监控:定期检查 t.mu 是否有 orphaned lock +func (t *Thinker) StartDeadlockMonitor() { + go func() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for range ticker.C { + done := make(chan struct{}) + go func() { + t.muLock() + t.muUnlock() + close(done) + }() + select { + case <-done: + // lock is healthy + case <-time.After(15 * time.Second): + log.Printf("[deadlock-monitor] ⚠️ t.mu 超过15秒无法获取!可能死锁。锁持有者: %s, 持有时长: %v", + t.muLockedBy, time.Since(t.muLockedAt).Round(time.Second)) + } + } + }() +} diff --git a/backend/ai-core/internal/crashlog/debug.go b/backend/ai-core/internal/crashlog/debug.go new file mode 100644 index 0000000..808ed36 --- /dev/null +++ b/backend/ai-core/internal/crashlog/debug.go @@ -0,0 +1,98 @@ +package crashlog + +import ( + "fmt" + "log" + "runtime" + "strings" + "sync" + "time" +) + +// ── Mutex with debug tracking ── + +// DebugMutex wraps a sync.Mutex with lock-holder tracking. +type DebugMutex struct { + mu sync.Mutex + holder string + lockedAt time.Time +} + +// Lock acquires the mutex with caller tracking. +func (m *DebugMutex) Lock() { + m.mu.Lock() + _, file, line, _ := runtime.Caller(1) + if idx := strings.LastIndex(file, "Cyrene/"); idx >= 0 { + file = file[idx+len("Cyrene/"):] + } + m.holder = fmt.Sprintf("%s:%d", file, line) + m.lockedAt = time.Now() +} + +// Unlock releases the mutex. +func (m *DebugMutex) Unlock() { + m.holder = "" + m.mu.Unlock() +} + +// TryLock attempts to acquire the lock with a timeout. Returns true if acquired. +func (m *DebugMutex) TryLock(timeout time.Duration) bool { + done := make(chan struct{}) + go func() { + m.mu.Lock() + m.mu.Unlock() + close(done) + }() + select { + case <-done: + return true + case <-time.After(timeout): + return false + } +} + +// Holder returns who holds the lock. +func (m *DebugMutex) Holder() (string, time.Time) { + return m.holder, m.lockedAt +} + +// ── Channel debug helpers ── + +// ChSend sends to a channel with a timeout and logs if it blocks. +func ChSend[T any](name string, ch chan<- T, val T, timeout time.Duration) bool { + select { + case ch <- val: + return true + case <-time.After(timeout): + log.Printf("[debug] ChSend timeout: %s after %v", name, timeout) + return false + } +} + +// ── Goroutine lifecycle ── + +// GoStart logs goroutine start and wraps fn with panic recovery. +func GoStart(name string, fn func()) { + log.Printf("[debug] goroutine START: %s", name) + go func() { + defer func() { + if r := recover(); r != nil { + stack := make([]byte, 4096) + n := runtime.Stack(stack, false) + log.Printf("[debug] goroutine PANIC: %s panic=%v\n%s", name, r, stack[:n]) + } + log.Printf("[debug] goroutine EXIT: %s", name) + }() + fn() + }() +} + +// ── Timing ── + +// Since logs if a duration exceeds a threshold. +func Since(name string, start time.Time, threshold time.Duration) { + elapsed := time.Since(start) + if elapsed > threshold { + log.Printf("[debug] SLOW: %s took %v (threshold=%v)", name, elapsed.Round(time.Millisecond), threshold) + } +}