feat: 全链路优化 — 死锁修复、MD3主题、上下文持久化、群聊自然化、打字状态、知识库

**死锁根因修复**
- periodicThinkLoop:1015 orphaned lock → 删除(字段已原子化)
- RecordUserMessage 隔离为 recordMu
- atomic.Int64 替换 lastUserMessage/lastThinkTime 等

**MD3 / Android 17 主题**
- 毛玻璃卡片 (backdrop-filter)
- MD3 色彩令牌 (pink primary #f472b6)
- icons.js 独立矢量图标库 + 运行时 emoji 替换
- 无边框卡片、圆角按钮、阴影层次

**上下文持久化**
- AddMessage → saveToDB 异步写 PostgreSQL
- LoadFromDB 恢复 (admin-session-main + 懒加载)
- LLMMessage.Timestamp 字段

**群聊与适配器**
- group_ambient 模式: 非@消息让 LLM 自己判断是否插话
- 戳一戳动作消息总是回复
- NapCat 打字状态 (set_input_status, 最小3秒显示)
- HTTP API 配置 (http_url/http_token)

**知识库 & 防编造**
- knowledge.CanHandle 对 chat 意图也触发
- 关键词预筛选避免无关 embedding 调用
- persona + synthesizer 三重诚实规则
- 工具结果持久化到会话历史

**平台桥接器**
- detached:true Go进程独立存活
- ethend 重启自动接管已运行服务
- stop() 接管模式 taskkill/F/ PID
- Windows netstat 替代 fuser 获取 PID
- 重复适配器种子逻辑修复
- 失败转发日志 Direction: error

**崩溃诊断**
- crashlog 包 (Recover + WrapHTTP + LLMCall)
- /api/v1/debug/goroutines 端点
- thinker 操作日志 + 30s stats
- 日志写入 logs/ 目录持久化

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-06-28 11:29:29 +08:00
parent 0d6970a2d3
commit fd44b15d81
23 changed files with 1447 additions and 254 deletions
+67 -17
View File
@@ -42,8 +42,14 @@ app.use((req, res, next) => {
next();
});
// 静态文件 - Web控制台
app.use(express.static(path.join(__dirname, '../public')));
// 静态文件 - Web控制台(禁用缓存,开发阶段每次刷新拿最新)
app.use(express.static(path.join(__dirname, '../public'), {
setHeaders: (res) => {
res.set('Cache-Control', 'no-store, no-cache, must-revalidate');
res.set('Pragma', 'no-cache');
res.set('Expires', '0');
}
}));
// ========== WebSocket ==========
const server = http.createServer(app);
@@ -276,22 +282,31 @@ app.get('/api/health', (_req, res) => {
});
});
// ---- ethend 自重启 ----
// ---- ethend 自重启 (Windows-safe) ----
app.post('/api/ethend/restart', (_req, res) => {
res.json({ success: true, message: 'ethend 正在重启...' });
// 延迟 500ms 确保响应已发送,然后 spawn 新进程并退出
// 步骤1: 立即关闭 HTTP 服务器,释放端口 9090
server.close(() => {
console.log('ethend HTTP 服务已关闭');
});
// 步骤2: 延迟 2 秒确保端口完全释放,然后启动新进程
setTimeout(() => {
const scriptPath = path.join(__dirname, 'index.js');
const child = spawn(process.execPath, [scriptPath, ...process.argv.slice(2)], {
cwd: ROOT,
detached: true,
// 注意: 不使用 detached:true
// - detached 在 Windows 上会创建新进程组→弹出 cmd 窗口
// - Windows 默认子进程不随父进程退出而终止,所以不需要 detached
// - windowsHide:true 确保即使有窗口也被隐藏
const child = spawn(process.execPath, [scriptPath], {
cwd: path.join(ROOT, 'ethend'),
detached: false,
stdio: 'ignore',
windowsHide: true,
});
child.unref();
process.exit(0);
}, 500);
}, 2000);
});
// ---- 仪表盘数据 (必须在 /api/services/:id 之前以避免路由冲突) ----
@@ -1281,20 +1296,17 @@ app.get('/api/trace/recent', async (req, res) => {
proxyToAICore(`/api/v1/llm-calls?limit=${limit}`).catch(() => ({ status: 502, body: [] })),
proxyToAICore(`/api/v1/tools/calls?limit=${limit}`).catch(() => ({ status: 502, body: { calls: [] } })),
proxyToGateway('/api/v1/admin/sessions/active').catch(() => ({ status: 502, body: { users: {} } })),
// platform-bridge 最近的消息日志
Promise.all((['obv11']).map(platform =>
proxyToPlatformBridge(`/api/v1/logs/${platform}?limit=50`).catch(() => ({ status: 502, body: { logs: [] } }))
)),
// 直接从磁盘读取 platform-bridge 日志文件
Promise.resolve(readBridgeLogs(50)),
// 拉 ai-core 追踪事件
proxyToAICore(`/api/v1/trace/events?limit=${limit}`).catch(() => ({ status: 502, body: { events: [] } })),
]);
const traces = [];
// === 消息收发 (platform-bridge 日志) ===
for (const logResult of bridgeLogsResult) {
const entries = logResult.body?.logs || logResult.body?.entries || logResult.body || [];
for (const entry of entries) {
// === 消息收发 (platform-bridge 日志文件) ===
const entries = bridgeLogsResult || [];
for (const entry of entries) {
const ts = entry.timestamp ? new Date(entry.timestamp).getTime() : Date.now();
const isIncoming = entry.direction === 'incoming';
const hop = isIncoming ? 'msg_received' : 'msg_sent';
@@ -1314,7 +1326,6 @@ app.get('/api/trace/recent', async (req, res) => {
data: { platform: entry.platform, sender, channel, content: entry.content, direction: entry.direction, contentType: entry.content_type },
});
}
}
// === LLM 调用 ===
const llmCalls = Array.isArray(llmResult.body) ? llmResult.body : (llmResult.body?.calls || []);
@@ -1905,8 +1916,47 @@ server.listen(ETHEND_PORT, () => {
for (const [id, svc] of Object.entries(SERVICES)) {
console.log(` - ${svc.name} (${id}): ${svc.healthUrl || 'N/A'}`);
}
// ethend 重启后自动接管已运行的服务(延迟 2s 等待健康检查就绪)
setTimeout(async () => {
console.log('[ethend] 扫描已运行的服务...');
for (const [id] of Object.entries(SERVICES)) {
if (id === 'frontend') continue; // 前端不需要接管
try {
const adopted = await processManager.tryAdopt(id);
if (adopted) {
console.log(`${id} 已接管`);
}
} catch { /* skip */ }
}
console.log('[ethend] 服务扫描完成');
}, 2000);
});
// readBridgeLogs 从磁盘读取 platform-bridge 的 JSONL 日志文件
function readBridgeLogs(limit) {
const bridgeLogDir = path.join(ROOT, 'backend', 'platform-bridge', 'logs');
const logFiles = ['obv11-main.log', 'obv11.log']; // 新旧平台名都试试
const entries = [];
for (const name of logFiles) {
const logPath = path.join(bridgeLogDir, name);
if (!fs.existsSync(logPath)) continue;
try {
const content = fs.readFileSync(logPath, 'utf-8');
const lines = content.trim().split('\n');
// 取最后 limit 行
const recent = lines.slice(-limit);
for (const line of recent) {
try {
entries.push(JSON.parse(line));
} catch { /* skip bad lines */ }
}
break; // 优先用第一个找到的文件
} catch { /* skip */ }
}
return entries.slice(-limit);
}
// 时间格式化:使用系统本地时区
function formatLocalTime(ts) {
var d = new Date(ts);
+200 -8
View File
@@ -196,10 +196,23 @@ async function ensureDBOnline(serviceId, emitter) {
emitter.emit('log', serviceId, 'error', '⚠️ 数据库无法启动,请手动检查 Docker。将继续启动后端服务...');
}
// ── Watchdog 配置 ──
// 自动重启策略:进程异常退出时自动重启,指数退避,超过上限后停止
const WATCHDOG = {
MAX_RETRIES: 5, // 滑动窗口内最大重试次数
RETRY_WINDOW_MS: 5 * 60 * 1000, // 重试计数滑动窗口 (5 分钟)
BASE_DELAY_MS: 1000, // 首次重试延迟
MAX_DELAY_MS: 60 * 1000, // 最大重试延迟 (指数退避上限)
STABLE_RESET_MS: 2 * 60 * 1000, // 稳定运行此时间后重置重试计数
// 可以为每个服务单独覆盖配置,按 serviceId 索引
OVERRIDES: {},
};
class ProcessManager extends EventEmitter {
constructor() {
super();
/** @type {Map<string, {process: ChildProcess|null, status: string, startTime: number|null, pid: number|null, buildLog: string[]}>} */
/** @type {Map<string, {process: ChildProcess|null, status: string, startTime: number|null, pid: number|null, buildLog: string[], retryCount: number, retryHistory: number[], stableTimer: NodeJS.Timeout|null}>} */
this.processes = new Map();
for (const id of Object.keys(SERVICES)) {
@@ -209,10 +222,54 @@ class ProcessManager extends EventEmitter {
startTime: null,
pid: null,
buildLog: [],
retryCount: 0,
retryHistory: [], // crash timestamps in current window
stableTimer: null,
});
}
}
/**
* 获取服务的 watchdog 配置(合并默认值和覆盖)
*/
_watchdogConfig(serviceId) {
const defaults = {
maxRetries: WATCHDOG.MAX_RETRIES,
retryWindowMs: WATCHDOG.RETRY_WINDOW_MS,
baseDelayMs: WATCHDOG.BASE_DELAY_MS,
maxDelayMs: WATCHDOG.MAX_DELAY_MS,
stableResetMs: WATCHDOG.STABLE_RESET_MS,
};
const overrides = WATCHDOG.OVERRIDES[serviceId] || {};
return { ...defaults, ...overrides };
}
/**
* 清理滑动窗口外的旧崩溃记录
*/
_pruneRetryHistory(procInfo, windowMs) {
const cutoff = Date.now() - windowMs;
procInfo.retryHistory = procInfo.retryHistory.filter(ts => ts > cutoff);
procInfo.retryCount = procInfo.retryHistory.length;
}
/**
* 启动稳定运行计时器:服务稳定运行一段时间后重置重试计数
*/
_startStableTimer(serviceId, procInfo) {
const cfg = this._watchdogConfig(serviceId);
if (procInfo.stableTimer) clearTimeout(procInfo.stableTimer);
procInfo.stableTimer = setTimeout(() => {
if (procInfo.retryCount > 0) {
this.emit('log', serviceId, 'system',
`✅ Watchdog: 服务已稳定运行 ${Math.round(cfg.stableResetMs / 1000)}s,重置崩溃计数 (之前 ${procInfo.retryCount} 次)`);
}
procInfo.retryCount = 0;
procInfo.retryHistory = [];
procInfo.stableTimer = null;
}, cfg.stableResetMs);
}
/**
* 启动服务
*/
@@ -230,6 +287,39 @@ class ProcessManager extends EventEmitter {
throw new Error(`${svc.name} 已在运行中`);
}
// 如果服务已在运行(例如手动启动),通过健康检查自动接管,避免杀进程
if (svc.healthUrl) {
try {
const resp = await fetch(svc.healthUrl, { signal: AbortSignal.timeout(3000) });
if (resp.ok) {
let pid = null;
try {
if (isWin) {
// Windows: use netstat -ano to find PID by port
const out = execSync(`netstat -ano | findstr ":${svc.port} " | findstr "LISTENING"`, { timeout: 3000, stdio: 'pipe' }).toString().trim();
const lines = out.split('\n');
for (const line of lines) {
const parts = line.trim().split(/\s+/);
const last = parts[parts.length - 1];
if (/^\d+$/.test(last)) { pid = parseInt(last); break; }
}
} else {
const out = execSync(`fuser ${svc.port}/tcp 2>/dev/null || true`, { timeout: 2000 }).toString().trim();
const match = out.match(/(\d+)/);
if (match) pid = parseInt(match[1]);
}
} catch { /* ignore */ }
procInfo.pid = pid;
procInfo.startTime = Date.now();
procInfo.status = 'running';
procInfo.process = null;
this.emit('log', serviceId, 'system', `${svc.name} 已在运行 (PID: ${pid || '未知'}),已接管(无需重启)`);
this._startStableTimer(serviceId, procInfo);
return { success: true, message: `${svc.name} 已接管 (无需重启)` };
}
} catch { /* 不可达,继续启动新进程 */ }
}
// 对需要数据库的服务做前置检查
if (['gateway', 'ai-core', 'memory-service', 'plugin-manager', 'platform-bridge'].includes(serviceId)) {
this.emit('log', serviceId, 'system', '检查数据库连接状态...');
@@ -297,13 +387,20 @@ class ProcessManager extends EventEmitter {
// .cmd/.bat on Windows needs shell:true
const needsShell = isWin && (command.endsWith('.cmd') || command.endsWith('.bat'));
// Go 后端进程使用 detached 确保 ethend 崩溃后独立存活
// npx/node 进程不使用 detached 避免 Windows 弹窗
const isGoSvc = svc.command === './main';
const child = spawn(command, args, {
cwd: svc.cwd,
env,
stdio: ['ignore', 'pipe', 'pipe'],
shell: needsShell,
windowsHide: true,
detached: isGoSvc,
});
if (isGoSvc) {
child.unref(); // 解除父子关系,ethen 退出不影响子进程
}
child.stdout.on('data', (data) => {
const text = data.toString();
@@ -335,15 +432,74 @@ class ProcessManager extends EventEmitter {
});
child.on('close', (code) => {
const msg = `进程退出,退出码: ${code}`;
logStream.write(msg + '\n');
this.emit('log', serviceId, 'system', msg);
const exitMsg = `进程退出,退出码: ${code}`;
logStream.write(exitMsg + '\n');
this.emit('log', serviceId, 'system', exitMsg);
procInfo.status = 'stopped';
procInfo.process = null;
procInfo.pid = null;
logStream.end();
// 清除稳定运行计时器
if (procInfo.stableTimer) {
clearTimeout(procInfo.stableTimer);
procInfo.stableTimer = null;
}
// ── Watchdog 自动重启判断 ──
const isNormalExit = code === 0 || code === null; // null = signal kill (e.g. SIGTERM from manual stop)
// 检查是否是用户手动停止 (通过 _manualStop 标记)
const wasManualStop = procInfo._manualStop === true;
procInfo._manualStop = false; // 重置标记
if (isNormalExit || wasManualStop) {
// 正常退出或手动停止:不触发 watchdog
procInfo.retryCount = 0;
procInfo.retryHistory = [];
return;
}
// 异常退出 (code !== 0):尝试自动重启
const wdCfg = this._watchdogConfig(serviceId);
this._pruneRetryHistory(procInfo, wdCfg.retryWindowMs);
procInfo.retryHistory.push(Date.now());
procInfo.retryCount = procInfo.retryHistory.length;
if (procInfo.retryCount > wdCfg.maxRetries) {
const windowSec = Math.round(wdCfg.retryWindowMs / 1000);
this.emit('log', serviceId, 'error',
`🚨 Watchdog: ${svc.name}${windowSec}s 内崩溃 ${procInfo.retryCount} 次,已达上限 (${wdCfg.maxRetries}),停止自动重启。请手动排查问题后重启。`);
procInfo.status = 'crashed';
return;
}
// 计算指数退避延迟
const delay = Math.min(
wdCfg.baseDelayMs * Math.pow(2, procInfo.retryCount - 1),
wdCfg.maxDelayMs
);
const delaySec = Math.round(delay / 1000);
this.emit('log', serviceId, 'system',
`🔄 Watchdog: ${svc.name} 异常退出 (第 ${procInfo.retryCount}/${wdCfg.maxRetries} 次)${delaySec}s 后自动重启...`);
setTimeout(async () => {
try {
this.emit('log', serviceId, 'system', `🔄 Watchdog: 正在自动重启 ${svc.name}...`);
const result = await this.start(serviceId);
if (result.success) {
this.emit('log', serviceId, 'system', `✅ Watchdog: ${svc.name} 自动重启成功`);
} else {
this.emit('log', serviceId, 'error', `❌ Watchdog: ${svc.name} 自动重启失败: ${result.message}`);
}
} catch (err) {
this.emit('log', serviceId, 'error', `❌ Watchdog: ${svc.name} 自动重启异常: ${err.message}`);
}
}, delay);
});
// 启动稳定运行计时器
this._startStableTimer(serviceId, procInfo);
return { success: true, message: `${svc.name} 启动中...` };
}
@@ -360,12 +516,36 @@ class ProcessManager extends EventEmitter {
const procInfo = this.processes.get(serviceId);
if (!procInfo.process) {
// 接管模式(未持有进程句柄):通过端口反查 PID 并强制杀死
if (svc.port && procInfo.pid) {
try {
if (isWin) {
execSync(`taskkill /F /PID ${procInfo.pid}`, { timeout: 5000, stdio: 'pipe' });
} else {
execSync(`kill -9 ${procInfo.pid}`, { timeout: 5000, stdio: 'pipe' });
}
this.emit('log', serviceId, 'system', `${svc.name} 已通过 PID 强制停止 (PID: ${procInfo.pid})`);
} catch (e) {
this.emit('log', serviceId, 'system', `${svc.name} PID ${procInfo.pid} 已不存在或无法杀死`);
}
} else if (svc.port) {
// 没有 PID 但有端口:用 fuser 释放端口
try {
execSync(`fuser -k ${svc.port}/tcp 2>/dev/null || true`, { timeout: 5000, stdio: 'pipe' });
this.emit('log', serviceId, 'system', `${svc.name} 端口 ${svc.port} 已释放`);
} catch { /* fuser not available */ }
}
procInfo.status = 'stopped';
procInfo.pid = null;
procInfo.startTime = null;
// 等待 1 秒确保端口释放
await new Promise(r => setTimeout(r, 1000));
return { success: true, message: `${svc.name} 已停止` };
}
// 标记为手动停止,防止 watchdog 自动重启
procInfo._manualStop = true;
return new Promise((resolve) => {
const timeout = setTimeout(() => {
// 强制杀死
@@ -488,6 +668,7 @@ class ProcessManager extends EventEmitter {
port: svc.port,
healthUrl: svc.healthUrl,
source,
retryCount: info.retryCount || 0,
...(docker ? { containerName: docker.containerName, containerId: docker.containerId } : {}),
};
}
@@ -518,6 +699,7 @@ class ProcessManager extends EventEmitter {
port: svc.port,
healthUrl: svc.healthUrl,
source,
retryCount: info.retryCount || 0,
...(docker ? { containerName: docker.containerName, containerId: docker.containerId } : {}),
};
}
@@ -556,12 +738,22 @@ class ProcessManager extends EventEmitter {
const resp = await fetch(svc.healthUrl, { signal: AbortSignal.timeout(3000) });
if (resp.ok) {
const procInfo = this.processes.get(serviceId);
// 尝试通过 fuser 获取 PID
// 通过端口反查 PID
let pid = null;
try {
const out = execSync(`fuser ${svc.port}/tcp 2>/dev/null || true`, { timeout: 2000 }).toString().trim();
const match = out.match(/(\d+)/);
if (match) pid = parseInt(match[1]);
if (isWin) {
const out = execSync(`netstat -ano | findstr ":${svc.port} " | findstr "LISTENING"`, { timeout: 3000, stdio: 'pipe' }).toString().trim();
const lines = out.split('\n');
for (const line of lines) {
const parts = line.trim().split(/\s+/);
const last = parts[parts.length - 1];
if (/^\d+$/.test(last)) { pid = parseInt(last); break; }
}
} else {
const out = execSync(`fuser ${svc.port}/tcp 2>/dev/null || true`, { timeout: 2000 }).toString().trim();
const match = out.match(/(\d+)/);
if (match) pid = parseInt(match[1]);
}
} catch { /* ignore */ }
procInfo.pid = pid;