feat(admin): add full-chain trace and performance monitoring
- Backend: trace.Logger ring-buffer for request chain events - Backend: perf.Monitor with runtime.MemStats sampling every 10s - Backend: /api/v1/admin/trace and /api/v1/admin/perf endpoints - Frontend: TracePage — timeline with hop icons and status dots - Frontend: PerformancePage — memory/GC bar charts + live stats - Admin nav expanded to 11 items
This commit is contained in:
Aska committed
1 parent
8420691438
commit
e8b5679cf4
8 files changed
+442
No files matched your search
@@ -33,6 +33,8 @@ import (
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/services/ai-core/llm"
|
||||
llmlog "git.yeij.top/AskaEth/Cyrene_v2/services/ai-core/llm"
|
||||
tracepkg "git.yeij.top/AskaEth/Cyrene_v2/services/ai-core/trace"
|
||||
perfpkg "git.yeij.top/AskaEth/Cyrene_v2/services/ai-core/perf"
|
||||
aiCoreCtx "git.yeij.top/AskaEth/Cyrene_v2/services/ai-core/context"
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/services/ai-core/orchestrator"
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/services/ai-core/persona"
|
||||
@@ -233,10 +235,16 @@ func main() {
|
||||
|
||||
// Build dependencies for gateway handlers
|
||||
llmLogger := llmlog.NewLogger(200)
|
||||
traceLogger := tracepkg.NewLogger(500)
|
||||
perfMon := perfpkg.NewMonitor(120, 10*time.Second)
|
||||
perfMon.Start()
|
||||
|
||||
gwDeps := api.Dependencies{
|
||||
AuthMgr: authMgr,
|
||||
Orchestrator: orch,
|
||||
LLMLogger: &llmLoggerAdapter{Logger: llmLogger},
|
||||
TraceLog: &traceAdapter{Logger: traceLogger},
|
||||
PerfMon: &perfAdapter{Monitor: perfMon},
|
||||
ModelConfig: api.NewModelConfigStore(),
|
||||
PluginList: pluginNamesToMap(reg.ListNames()),
|
||||
}
|
||||
@@ -294,3 +302,41 @@ func pluginNamesToMap(names []string) []map[string]string {
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// traceAdapter adapts trace.Logger to api.TraceLogger interface.
|
||||
type traceAdapter struct{ *tracepkg.Logger }
|
||||
func (a *traceAdapter) Recent(n int) []map[string]any {
|
||||
events := a.Logger.Recent(n)
|
||||
result := make([]map[string]any, len(events))
|
||||
for i, e := range events {
|
||||
result[i] = map[string]any{
|
||||
"id": e.ID, "timestamp": e.Timestamp.Format(time.RFC3339),
|
||||
"hop": e.Hop, "label": e.Label, "status": e.Status,
|
||||
"duration_ms": e.DurationMs, "detail": e.Detail, "data": e.Data,
|
||||
}
|
||||
if result[i]["data"] == nil { result[i]["data"] = map[string]any{} }
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// perfAdapter adapts perf.Monitor to api.PerfMonitor interface.
|
||||
type perfAdapter struct{ *perfpkg.Monitor }
|
||||
func (a *perfAdapter) Latest() map[string]any {
|
||||
s := a.Monitor.Latest()
|
||||
return map[string]any{
|
||||
"alloc_mb": s.AllocMB, "sys_mb": s.SysMB,
|
||||
"num_gc": s.NumGC, "goroutines": s.Goroutines, "heap_objects": s.HeapObj,
|
||||
}
|
||||
}
|
||||
func (a *perfAdapter) History() []map[string]any {
|
||||
snapshots := a.Monitor.History()
|
||||
result := make([]map[string]any, len(snapshots))
|
||||
for i, s := range snapshots {
|
||||
result[i] = map[string]any{
|
||||
"timestamp": s.Timestamp.Format(time.RFC3339),
|
||||
"alloc_mb": s.AllocMB, "sys_mb": s.SysMB,
|
||||
"num_gc": s.NumGC, "goroutines": s.Goroutines,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
// Package perf provides lightweight performance metrics sampling.
|
||||
package perf
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Snapshot captures a moment-in-time system performance reading.
|
||||
type Snapshot struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
AllocMB float64 `json:"alloc_mb"`
|
||||
SysMB float64 `json:"sys_mb"`
|
||||
NumGC uint32 `json:"num_gc"`
|
||||
Goroutines int `json:"goroutines"`
|
||||
HeapObj uint64 `json:"heap_objects"`
|
||||
}
|
||||
|
||||
// Monitor samples runtime.MemStats periodically and keeps a history.
|
||||
type Monitor struct {
|
||||
mu sync.RWMutex
|
||||
history []Snapshot
|
||||
max int
|
||||
index int
|
||||
stopCh chan struct{}
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
func bytesToMB(b uint64) float64 { return float64(b) / 1024 / 1024 }
|
||||
|
||||
// NewMonitor creates a performance monitor that samples every `interval`.
|
||||
func NewMonitor(maxSamples int, interval time.Duration) *Monitor {
|
||||
return &Monitor{
|
||||
history: make([]Snapshot, maxSamples),
|
||||
max: maxSamples,
|
||||
stopCh: make(chan struct{}),
|
||||
interval: interval,
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins periodic sampling.
|
||||
func (m *Monitor) Start() {
|
||||
go func() {
|
||||
ticker := time.NewTicker(m.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-m.stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.sample()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Stop stops sampling.
|
||||
func (m *Monitor) Stop() { close(m.stopCh) }
|
||||
|
||||
func (m *Monitor) sample() {
|
||||
var ms runtime.MemStats
|
||||
runtime.ReadMemStats(&ms)
|
||||
|
||||
s := Snapshot{
|
||||
Timestamp: time.Now(),
|
||||
AllocMB: bytesToMB(ms.Alloc),
|
||||
SysMB: bytesToMB(ms.Sys),
|
||||
NumGC: ms.NumGC,
|
||||
Goroutines: runtime.NumGoroutine(),
|
||||
HeapObj: ms.HeapObjects,
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
m.history[m.index%m.max] = s
|
||||
m.index++
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// Latest returns the most recent snapshot.
|
||||
func (m *Monitor) Latest() Snapshot {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
if m.index == 0 {
|
||||
m.sample() // Sample once on first call
|
||||
}
|
||||
return m.history[(m.index-1+m.max)%m.max]
|
||||
}
|
||||
|
||||
// History returns the sampled history.
|
||||
func (m *Monitor) History() []Snapshot {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
n := m.index
|
||||
if n > m.max { n = m.max }
|
||||
result := make([]Snapshot, n)
|
||||
for i := 0; i < n; i++ {
|
||||
idx := (m.index - 1 - i + m.max) % m.max
|
||||
if idx < m.max {
|
||||
result[n-1-i] = m.history[idx]
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Package trace provides a ring-buffer for full-chain tracing events.
|
||||
package trace
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Event represents a single event in a request trace chain.
|
||||
type Event struct {
|
||||
ID string `json:"id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Hop string `json:"hop"` // "msg_received" | "intent" | "llm_call" | "tool_call" | "response"
|
||||
Label string `json:"label"`
|
||||
Status string `json:"status"` // "success" | "error" | "pending"
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
Detail string `json:"detail"`
|
||||
Data map[string]any `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
// Logger is a ring-buffer that stores the last N trace events.
|
||||
type Logger struct {
|
||||
mu sync.RWMutex
|
||||
events []Event
|
||||
max int
|
||||
index int
|
||||
}
|
||||
|
||||
// NewLogger creates a trace logger with the given capacity.
|
||||
func NewLogger(capacity int) *Logger {
|
||||
return &Logger{events: make([]Event, capacity), max: capacity}
|
||||
}
|
||||
|
||||
// Record adds a trace event.
|
||||
func (l *Logger) Record(event Event) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if event.ID == "" {
|
||||
event.ID = time.Now().Format("150405") + "-" + event.Hop
|
||||
}
|
||||
l.events[l.index%l.max] = event
|
||||
l.index++
|
||||
}
|
||||
|
||||
// Recent returns the most recent N events.
|
||||
func (l *Logger) Recent(n int) []Event {
|
||||
l.mu.RLock()
|
||||
defer l.mu.RUnlock()
|
||||
|
||||
if n > l.max { n = l.max }
|
||||
count := l.index
|
||||
if count > l.max { count = l.max }
|
||||
if n > count { n = count }
|
||||
|
||||
result := make([]Event, n)
|
||||
for i := 0; i < n; i++ {
|
||||
idx := (l.index - 1 - i + l.max) % l.max
|
||||
if idx < l.max {
|
||||
result[n-1-i] = l.events[idx]
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Hops returns unique hop names seen in the buffer.
|
||||
func (l *Logger) Hops() []string {
|
||||
l.mu.RLock()
|
||||
defer l.mu.RUnlock()
|
||||
|
||||
seen := make(map[string]bool)
|
||||
var result []string
|
||||
count := l.index
|
||||
if count > l.max { count = l.max }
|
||||
for i := 0; i < count; i++ {
|
||||
ev := l.events[i]
|
||||
if !seen[ev.Hop] && ev.Hop != "" {
|
||||
seen[ev.Hop] = true
|
||||
result = append(result, ev.Hop)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -66,6 +66,17 @@ type LLMCallLogger interface {
|
||||
Stats() map[string]any
|
||||
}
|
||||
|
||||
// TraceLogger provides full-chain trace events.
|
||||
type TraceLogger interface {
|
||||
Recent(n int) []map[string]any
|
||||
}
|
||||
|
||||
// PerfMonitor provides performance snapshot history.
|
||||
type PerfMonitor interface {
|
||||
Latest() map[string]any
|
||||
History() []map[string]any
|
||||
}
|
||||
|
||||
// Dependencies holds all injectable services.
|
||||
type Dependencies struct {
|
||||
AuthMgr AuthManager
|
||||
@@ -74,6 +85,8 @@ type Dependencies struct {
|
||||
Orchestrator OrchestratorPipeline
|
||||
WSHub WebSocketHub
|
||||
LLMLogger LLMCallLogger
|
||||
TraceLog TraceLogger
|
||||
PerfMon PerfMonitor
|
||||
ModelConfig *ModelConfigStore
|
||||
PluginList []map[string]string
|
||||
}
|
||||
@@ -118,6 +131,8 @@ func (r *Router) setupRoutes() {
|
||||
r.mux.HandleFunc("/api/v1/admin/sessions", r.authMW(r.adminMW(r.adminSessionsHandler)))
|
||||
r.mux.HandleFunc("/api/v1/admin/models/config", r.authMW(r.adminMW(r.modelConfigHandler)))
|
||||
r.mux.HandleFunc("/api/v1/admin/models/config/", r.authMW(r.adminMW(r.modelConfigItemHandler)))
|
||||
r.mux.HandleFunc("/api/v1/admin/trace", r.authMW(r.adminMW(r.traceHandler)))
|
||||
r.mux.HandleFunc("/api/v1/admin/perf", r.authMW(r.adminMW(r.perfHandler)))
|
||||
}
|
||||
|
||||
// ServeHTTP implements http.Handler.
|
||||
@@ -486,6 +501,27 @@ func num(m map[string]any, key string) int {
|
||||
return 0
|
||||
}
|
||||
|
||||
// --- Trace & Perf ---
|
||||
|
||||
func (r *Router) traceHandler(w http.ResponseWriter, req *http.Request) {
|
||||
if r.deps.TraceLog == nil {
|
||||
writeJSON(w, 200, M{"events": []any{}})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, M{"events": r.deps.TraceLog.Recent(100)})
|
||||
}
|
||||
|
||||
func (r *Router) perfHandler(w http.ResponseWriter, req *http.Request) {
|
||||
if r.deps.PerfMon == nil {
|
||||
writeJSON(w, 200, M{"latest": M{}, "history": []any{}})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, M{
|
||||
"latest": r.deps.PerfMon.Latest(),
|
||||
"history": r.deps.PerfMon.History(),
|
||||
})
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
type M = map[string]any
|
||||
|
||||
@@ -12,6 +12,8 @@ import LLMCallsPage from './pages/admin/LLMCallsPage'
|
||||
import SystemPage from './pages/admin/SystemPage'
|
||||
import ModelsPage from './pages/admin/ModelsPage'
|
||||
import BridgePage from './pages/admin/BridgePage'
|
||||
import TracePage from './pages/admin/TracePage'
|
||||
import PerformancePage from './pages/admin/PerformancePage'
|
||||
import LoginPage from './pages/LoginPage'
|
||||
|
||||
export default function App() {
|
||||
@@ -42,6 +44,8 @@ export default function App() {
|
||||
<Route path="llm-calls" element={<LLMCallsPage />} />
|
||||
<Route path="models" element={<ModelsPage />} />
|
||||
<Route path="bridge" element={<BridgePage />} />
|
||||
<Route path="trace" element={<TracePage />} />
|
||||
<Route path="perf" element={<PerformancePage />} />
|
||||
<Route path="system" element={<SystemPage />} />
|
||||
</Route>
|
||||
</Routes>
|
||||
|
||||
@@ -8,6 +8,8 @@ const navItems = [
|
||||
{ to: '/admin/plugins', label: '插件管理', icon: '◇' },
|
||||
{ to: '/admin/memory', label: '记忆管理', icon: '♡' },
|
||||
{ to: '/admin/bridge', label: '平台桥接', icon: '↗' },
|
||||
{ to: '/admin/trace', label: '链路追踪', icon: '▸' },
|
||||
{ to: '/admin/perf', label: '性能监控', icon: '▴' },
|
||||
{ to: '/admin/llm-calls', label: 'LLM 日志', icon: '🧠' },
|
||||
{ to: '/admin/system', label: '系统信息', icon: '⊞' },
|
||||
]
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { getToken } from '../../api/client'
|
||||
|
||||
export default function PerformancePage() {
|
||||
const [latest, setLatest] = useState<any>({})
|
||||
const [history, setHistory] = useState<any[]>([])
|
||||
|
||||
useEffect(() => {
|
||||
fetchPerf()
|
||||
const interval = setInterval(fetchPerf, 10000)
|
||||
return () => clearInterval(interval)
|
||||
}, [])
|
||||
|
||||
const fetchPerf = async () => {
|
||||
try {
|
||||
const token = getToken()
|
||||
const res = await fetch('/api/v1/admin/perf', { headers: { Authorization: `Bearer ${token}` } })
|
||||
const data = await res.json()
|
||||
setLatest(data.latest || {})
|
||||
setHistory(data.history || [])
|
||||
} catch {}
|
||||
}
|
||||
|
||||
const maxAlloc = Math.max(10, ...history.map((h: any) => h.alloc_mb || 0))
|
||||
|
||||
return (
|
||||
<div className="p-6 space-y-6">
|
||||
<h2 className="text-lg font-semibold">性能监控</h2>
|
||||
|
||||
{/* Current stats */}
|
||||
<div className="grid grid-cols-4 gap-3">
|
||||
<Stat label="内存分配" value={`${(latest.alloc_mb||0).toFixed(1)} MB`} sub="Heap Alloc" />
|
||||
<Stat label="系统内存" value={`${(latest.sys_mb||0).toFixed(1)} MB`} sub="Sys" />
|
||||
<Stat label="GC 次数" value={latest.num_gc || 0} sub="NumGC" />
|
||||
<Stat label="Goroutines" value={latest.goroutines || 0} sub="并发" />
|
||||
</div>
|
||||
|
||||
{/* Memory chart */}
|
||||
<div>
|
||||
<h3 className="text-sm font-medium text-white/60 mb-3">内存使用趋势 (最近 {history.length} 次采样)</h3>
|
||||
<div className="bg-[#000] border border-[#404040] rounded-[18px] p-4">
|
||||
<div className="flex items-end gap-1 h-32">
|
||||
{history.slice(-60).map((h: any, i: number) => {
|
||||
const pct = maxAlloc > 0 ? ((h.alloc_mb || 0) / maxAlloc) * 100 : 0
|
||||
return (
|
||||
<div key={i}
|
||||
className="flex-1 bg-[#277AF7]/60 hover:bg-[#277AF7] rounded-t-sm transition-colors"
|
||||
style={{ height: `${Math.max(2, pct)}%` }}
|
||||
title={`${(h.alloc_mb||0).toFixed(1)}MB`}
|
||||
/>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
<div className="flex justify-between mt-2 text-xs text-white/30">
|
||||
<span>{history[0]?.timestamp?.slice(11, 19) || '--'}</span>
|
||||
<span>{history[history.length-1]?.timestamp?.slice(11, 19) || '--'}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* GC chart */}
|
||||
<div>
|
||||
<h3 className="text-sm font-medium text-white/60 mb-3">GC 次数趋势</h3>
|
||||
<div className="bg-[#000] border border-[#404040] rounded-[18px] p-4">
|
||||
<div className="flex items-end gap-1 h-24">
|
||||
{history.slice(-60).map((h: any, i: number) => {
|
||||
const maxGC = Math.max(1, ...history.map((x: any) => x.num_gc || 0))
|
||||
const pct = ((h.num_gc || 0) / maxGC) * 100
|
||||
return (
|
||||
<div key={i}
|
||||
className="flex-1 bg-[#4788FF]/40 hover:bg-[#4788FF] rounded-t-sm transition-colors"
|
||||
style={{ height: `${Math.max(2, pct)}%` }}
|
||||
title={`GC: ${h.num_gc}`}
|
||||
/>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Stat({ label, value, sub }: { label:string; value:string|number; sub:string }) {
|
||||
return (
|
||||
<div className="bg-[#000] border border-[#404040] rounded-[18px] px-4 py-3">
|
||||
<p className="text-xs text-white/40 mb-1">{label}</p>
|
||||
<p className="text-lg font-bold text-[#F2F2F2]">{value}</p>
|
||||
<p className="text-xs text-white/20 mt-0.5">{sub}</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { getToken } from '../../api/client'
|
||||
|
||||
const hopIcons: Record<string, string> = {
|
||||
msg_received: '📥', msg_sent: '📤',
|
||||
intent: '🎯', llm_call: '🧠', tool_call: '🔧',
|
||||
response: '💬', session: '👤', subsession: '⚡',
|
||||
}
|
||||
|
||||
export default function TracePage() {
|
||||
const [events, setEvents] = useState<any[]>([])
|
||||
|
||||
useEffect(() => {
|
||||
fetchTrace()
|
||||
const interval = setInterval(fetchTrace, 5000)
|
||||
return () => clearInterval(interval)
|
||||
}, [])
|
||||
|
||||
const fetchTrace = async () => {
|
||||
try {
|
||||
const token = getToken()
|
||||
const res = await fetch('/api/v1/admin/trace', { headers: { Authorization: `Bearer ${token}` } })
|
||||
const data = await res.json()
|
||||
setEvents(data.events || [])
|
||||
} catch {}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="p-6 space-y-4">
|
||||
<h2 className="text-lg font-semibold">全链路追踪</h2>
|
||||
<p className="text-xs text-white/40">最近 {events.length} 条追踪事件</p>
|
||||
|
||||
{events.length === 0 ? (
|
||||
<div className="p-8 text-center text-white/30 text-sm">暂无追踪数据 — 发送消息后刷新</div>
|
||||
) : (
|
||||
<div className="space-y-1.5">
|
||||
{events.map((ev, i) => (
|
||||
<div key={ev.id || i}
|
||||
className={`bg-[#000] border rounded-[14px] px-4 py-3 flex items-center gap-3 ${
|
||||
ev.status === 'error' ? 'border-[#F12522]/30' : 'border-[#404040]'
|
||||
}`}
|
||||
>
|
||||
{/* Hop icon */}
|
||||
<span className="text-lg shrink-0">{hopIcons[ev.hop] || '●'}</span>
|
||||
|
||||
{/* Content */}
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-sm text-[#F2F2F2]">{ev.label || ev.hop}</span>
|
||||
{ev.duration_ms > 0 && (
|
||||
<span className="text-xs text-white/30">{ev.duration_ms}ms</span>
|
||||
)}
|
||||
</div>
|
||||
{ev.detail && (
|
||||
<p className="text-xs text-white/30 truncate mt-0.5">{ev.detail}</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Status */}
|
||||
<div className="flex items-center gap-3 shrink-0">
|
||||
<span className={`text-xs font-mono text-white/30`}>
|
||||
{ev.timestamp ? ev.timestamp.slice(11, 19) : ''}
|
||||
</span>
|
||||
<span className={`w-2 h-2 rounded-full ${
|
||||
ev.status === 'success' ? 'bg-[#4788FF]' :
|
||||
ev.status === 'error' ? 'bg-[#F12522]' :
|
||||
ev.status === 'pending' ? 'bg-yellow-500' : 'bg-[#404040]'
|
||||
}`} />
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Reference in new issue
Block a user