'use strict' const crypto = require('crypto') const fs = require('fs') const http = require('http') const path = require('path') const { EventEmitter } = require('events') const { spawn } = require('child_process') const { withConfigArg } = require('./userpaths') const STATE_LABEL = { stopped: '已停止', starting: '启动中', running: '运行中', stopping: '停止中', // 服务端自己按设计退出(切换赛事 / 一键恢复)后的等待期:与「异常退出」必须分开显示, // 现场看到红灯会以为服务挂了,实际是正常收尾等 Console 拉回来。 restarting: '重启中', crashed: '异常退出' } // 优雅关闭握手:POST server.shutdownPath 并带上启动时下发的 token, // 让服务端自己走 signal.NotifyContext 的关闭流程;请求失败或接口未实现时回退到强杀。 const DEFAULT_SHUTDOWN_PATH = '/api/shutdown' const SHUTDOWN_TOKEN_ENV = 'SCOREBOARD_SHUTDOWN_TOKEN' const SHUTDOWN_TOKEN_HEADER = 'X-Shutdown-Token' const SHUTDOWN_REQUEST_TIMEOUT_MS = 3000 // 服务端收到关闭请求后承诺 3 秒内退出,客户端留 5 秒余量再强杀 const DEFAULT_SHUTDOWN_TIMEOUT_MS = 5000 // 握手不可用时信号已能结束进程,不必再等满超时 const FALLBACK_FORCE_KILL_MS = 1500 // 连续健康这么久才认为崩溃计数可以清零 const HEALTHY_RESET_MS = 60000 // 「预期退出窗口」的下限:最后一次确认服务端还活着之后的这段时间内退出,才可能是按设计退出。 // 健康探测默认 5 秒一次,窗口至少覆盖两三个探测周期,避免刚好卡在探测空档时漏判。 const EXPECTED_EXIT_WINDOW_MS = 15000 // 副作用边界集中在这里注入:进程、时钟、HTTP。生产路径直接用 Node 的实现, // 单元测试传假实现,才能在毫秒内跑完退避、超时、优雅关闭这些真实要等几秒的分支。 const defaultTimers = { setTimeout: (fn, ms) => setTimeout(fn, ms), clearTimeout: (id) => clearTimeout(id), setInterval: (fn, ms) => setInterval(fn, ms), clearInterval: (id) => clearInterval(id) } // 服务端可能监听 0.0.0.0/::,本机访问(健康检查、关闭握手、大屏、管理面板)统一走回环地址 function healthHost(server) { const host = (server && server.host) || '127.0.0.1' return host === '0.0.0.0' || host === '::' ? '127.0.0.1' : host } // 服务端守护:拉起 scoreboard.exe、转发日志、健康探测、崩溃自动重启。 // 现场无人值守,所以「崩了自动拉起」比「暴露复杂控制」更重要。 class ServiceSupervisor extends EventEmitter { constructor({ values, exePath, log, dataDir, configPath, spawn: spawnFn, http: httpClient, timers, now }) { super() this.values = values this.server = values.server this.exePath = exePath this.log = log // 绝对路径,非空时下发给服务端 -data;为空表示沿用服务端默认数据目录 this.dataDir = dataDir || null // 绝对路径,非空时覆盖 args 里的 -config;为空表示沿用 args 原值(开发态) this.configPath = configPath || null // 可注入的副作用边界,默认就是 Node 的实现 this.spawn = spawnFn || spawn this.http = httpClient || http this.timers = { ...defaultTimers, ...(timers || {}) } this.now = now || (() => Date.now()) this.child = null this.state = 'stopped' this.healthy = false this.pid = null this.restarts = 0 this.startedAt = null this.lastError = null this.stopRequested = false this.healthySince = null this.disposed = false // 最近一次确认服务端还在正常服务(健康检查通过)的时刻,用于「预期退出窗口」判定 this.lastAliveAt = null this.healthTimer = null this.restartTimer = null this.forceKillTimer = null this.stopTimer = null // 本次进程的关闭握手 token 与握手结果,仅用于日志区分「优雅退出 / 被强杀」 this.shutdownToken = null this.gracefulAccepted = false this.forcedKill = false } get status() { const host = this.healthHost() return { state: this.state, stateLabel: STATE_LABEL[this.state] || this.state, healthy: this.healthy, pid: this.pid, restarts: this.restarts, startedAt: this.startedAt, lastError: this.lastError, exePath: this.exePath, url: `http://${host}:${this.server.port}`, adminUrl: `http://${host}:${this.server.port}/admin` } } // 服务端可能监听 0.0.0.0,本机探测要走回环地址 healthHost() { return healthHost(this.server) } emitStatus() { this.emit('status', this.status) } setState(state, patch = {}) { const changed = this.state !== state this.state = state Object.assign(this, patch) if (changed) { this.log.push('console', `服务状态:${STATE_LABEL[state] || state}${this.lastError ? `(${this.lastError})` : ''}`) } this.emitStatus() } start() { if (this.child || this.state === 'starting' || this.state === 'running') return this.status this.stopRequested = false if (!this.exePath || !fs.existsSync(this.exePath)) { this.setState('crashed', { lastError: `未找到服务端程序:${this.exePath || this.server.exe}` }) return this.status } const args = Array.isArray(this.server.args) ? this.server.args : [] // -config 换成 Console 解析出的那份:打包态在用户数据目录里, // 安装目录里那份只是随包的模板(会被卸载器删掉) const withConfig = withConfigArg(args, this.configPath) // 显式下发数据目录:默认位置在安装目录之外,覆盖安装时才不会被卸载器连目录删掉 const spawnArgs = this.dataDir ? [...withConfig, '-data', this.dataDir] : withConfig // 工作目录取 exe 所在目录(部署包根目录);config.yaml 与数据目录都走绝对路径, // 不再依赖 cwd const cwd = path.dirname(this.exePath) this.lastError = null this.healthySince = null // 新进程重新起算「还活着」的时刻:否则上一个进程的健康记录会让 // 「刚起来就退出 0」被误判成按设计退出 this.lastAliveAt = null this.gracefulAccepted = false this.forcedKill = false this.startedAt = new Date(this.now()).toISOString() this.setState('starting') this.log.push( 'console', `启动服务端:${this.exePath} ${spawnArgs.join(' ')}(cwd=${cwd},数据目录=${this.dataDir || '服务端默认'})` ) // 每次拉起都换一把新 token,只通过环境变量交给这一个子进程, // 停止时凭它调用服务端的 /api/shutdown this.shutdownToken = crypto.randomBytes(32).toString('hex') let child try { child = this.spawn(this.exePath, spawnArgs, { cwd, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, [SHUTDOWN_TOKEN_ENV]: this.shutdownToken } }) } catch (err) { this.setState('crashed', { lastError: `启动失败:${err.message}` }) this.scheduleRestart() return this.status } this.child = child this.pid = child.pid if (child.stdout) { child.stdout.setEncoding('utf8') child.stdout.on('data', (chunk) => this.log.append('server', chunk)) } if (child.stderr) { child.stderr.setEncoding('utf8') child.stderr.on('data', (chunk) => this.log.append('server!', chunk)) } child.on('error', (err) => { this.log.push('console', `服务端进程错误:${err.message}`) this.handleSpawnError(child, err) }) child.on('exit', (code, signal) => this.handleExit(child, code, signal)) this.setState('running') this.startHealthCheck() return this.status } // spawn 失败(ENOENT/EACCES 等)时 Node 只发 error、不发 exit: // 必须在这里清掉 child/pid 并调度重启,否则 start() 因 child 非空直接返回、 // stop() 永远等不到 exit,守护器彻底卡死。 handleSpawnError(child, err) { if (this.disposed || this.child !== child) return this.stopHealthCheck() this.child = null this.pid = null this.healthy = false this.healthySince = null this.setState('crashed', { lastError: `服务端进程错误:${err.message}` }) if (!this.stopRequested) this.scheduleRestart() } handleExit(child, code, signal) { // 迟到的 exit(error 分支已收尾、已换进程、或已经 dispose)不能动当前状态; // 尤其是 dispose 之后:应用正在退出,这里再排一次自动重启会留下一个没人管的循环 if (this.disposed || this.child !== child) return if (this.forceKillTimer) { this.timers.clearTimeout(this.forceKillTimer) this.forceKillTimer = null } this.stopHealthCheck() this.child = null this.pid = null this.healthy = false this.healthySince = null const reason = signal ? `信号 ${signal}` : `退出码 ${code}` if (this.stopRequested) { const how = this.forcedKill ? '已被强制结束' : this.gracefulAccepted ? '已优雅退出' : '已停止' this.log.push('console', `服务端${how}(${reason})`) this.gracefulAccepted = false this.forcedKill = false this.setState('stopped', { lastError: null }) return } if (this.isPlannedExit(code)) { // 切换赛事 / 一键恢复:服务端先回执再自行优雅退出,Console 的职责是把它拉回来。 // 这不能按崩溃算——否则现场正常切个赛事会显示红灯、崩溃计数 +1。 this.log.push('console', `服务端按设计退出(${reason},切换赛事 / 一键恢复),准备自动拉起`) this.setState('restarting', { lastError: null }) this.scheduleRestart({ planned: true }) return } this.setState('crashed', { lastError: `服务端意外退出(${reason})` }) this.scheduleRestart() } // 「按设计退出」的判定窗口(毫秒):最后一次确认服务端还活着之后允许退出的时间。 // 健康间隔配得大时窗口跟着放大,至少覆盖两个探测周期。 expectedExitWindowMs() { const interval = Number(this.server.healthIntervalMs) || 5000 return Math.max(EXPECTED_EXIT_WINDOW_MS, interval * 2) } // 服务端主动退出(退出码 0)与崩溃靠两点区分: // 1) 退出码:Go 只有从 main 正常返回才是 0,panic 是 2、装配失败是 os.Exit(1)、被信号杀则没有退出码; // 2) 是否在预期退出窗口内:切换赛事 / 一键恢复一定发生在服务端刚刚还在正常服务的时候 // (管理面板先拿到 200 回执,服务端随后才 Shutdown),所以窗口从最后一次健康检查通过起算。 // 窗口外退出 0 说明这次运行从没健康过,那不是切赛事而是异常,仍旧按崩溃处理。 isPlannedExit(code) { return ( code === 0 && this.lastAliveAt !== null && this.now() - this.lastAliveAt <= this.expectedExitWindowMs() ) } // planned=true 表示按设计退出后的拉起:不占崩溃配额、不递增崩溃计数, // 延迟仍用 restartDelayMs——服务端刚退出,端口与文件句柄要留一点回收时间。 scheduleRestart({ planned = false } = {}) { const max = Number(this.server.maxRestarts) || 0 if (!planned && max > 0 && this.restarts >= max) { this.log.push('console', `已达到最大重启次数 ${max},不再自动拉起`) return } // 退避是线性的 ×1..×5(不是翻倍):第 N 次等 N × restartDelayMs const attempt = planned ? 1 : this.restarts + 1 const delay = (Number(this.server.restartDelayMs) || 3000) * Math.min(attempt, 5) if (!planned) this.restarts += 1 this.log.push( 'console', planned ? `${Math.round(delay / 1000)} 秒后自动拉起服务端(按设计退出,不计入崩溃次数)` : `${Math.round(delay / 1000)} 秒后自动重启服务端(第 ${this.restarts} 次)` ) this.emitStatus() if (this.restartTimer) this.timers.clearTimeout(this.restartTimer) this.restartTimer = this.timers.setTimeout(() => { this.restartTimer = null if (!this.stopRequested) this.start() }, delay) } stop() { return new Promise((resolve) => { if (this.restartTimer) { this.timers.clearTimeout(this.restartTimer) this.restartTimer = null } const child = this.child if (!child) { this.setState('stopped', { lastError: null }) resolve() return } this.stopRequested = true this.setState('stopping') this.log.push('console', '正在优雅关闭服务端…') const graceMs = Number(this.server.shutdownTimeoutMs) || DEFAULT_SHUTDOWN_TIMEOUT_MS let settled = false const finish = () => { if (settled) return settled = true if (this.stopTimer) { this.timers.clearTimeout(this.stopTimer) this.stopTimer = null } resolve() } // 兜底:握手与强杀链路都可能因进程状态异常收不到 exit, // 退出流程不能永远挂住,超时后无条件放行并补一刀强杀。 this.stopTimer = this.timers.setTimeout(() => { this.stopTimer = null this.log.push('console', '等待服务端退出超时,停止流程强制返回') if (this.child === child) { this.forcedKill = true try { child.kill('SIGKILL') } catch { // 进程可能已经没了 } } finish() }, SHUTDOWN_REQUEST_TIMEOUT_MS + graceMs + FALLBACK_FORCE_KILL_MS + 2000) child.once('exit', () => finish()) this.requestShutdown().then((accepted) => { // 请求期间进程可能已经退了,handleExit 会清掉兜底定时器 if (this.child !== child) return if (accepted) { this.gracefulAccepted = true this.log.push('console', `服务端已接受优雅关闭请求,最多等待 ${Math.round(graceMs / 1000)} 秒自行退出`) this.armForceKill(graceMs) return } // 握手不可用:服务端还是老版本没有该接口,或进程已无响应。 // 退回进程信号;Windows 上 Node 的 kill() 等同强杀,所以这条路只是兜底, // 真正的优雅关闭依赖服务端实现 shutdownPath 接口。 this.log.push('console', '优雅关闭接口不可用,退回强制结束') this.armForceKill(FALLBACK_FORCE_KILL_MS) try { child.kill() } catch (err) { this.log.push('console', `发送停止信号失败:${err.message}`) } }) }) } // 优雅关闭握手:带启动时下发的 X-Shutdown-Token 请求服务端退出, // 2xx 表示已受理。老版本服务端没有该接口(404)或连不上时返回 false, // 由调用方回退到强杀。 requestShutdown() { return new Promise((resolve) => { const body = Buffer.from('{}') const request = this.http.request( { host: this.healthHost(), port: this.server.port, path: this.server.shutdownPath || DEFAULT_SHUTDOWN_PATH, method: 'POST', timeout: SHUTDOWN_REQUEST_TIMEOUT_MS, headers: { 'Content-Type': 'application/json', 'Content-Length': body.length, [SHUTDOWN_TOKEN_HEADER]: this.shutdownToken || '', Connection: 'close' } }, (response) => { response.resume() const accepted = response.statusCode >= 200 && response.statusCode < 300 if (!accepted) this.log.push('console', `关闭接口返回 HTTP ${response.statusCode}(按老版本处理)`) resolve(accepted) } ) request.on('timeout', () => request.destroy(new Error('请求超时'))) request.on('error', (err) => { this.log.push('console', `关闭接口请求失败:${err.message}`) resolve(false) }) request.end(body) }) } armForceKill(ms) { if (this.forceKillTimer) this.timers.clearTimeout(this.forceKillTimer) this.forceKillTimer = this.timers.setTimeout(() => { this.forceKillTimer = null if (!this.child || !this.pid) return this.forcedKill = true this.log.push('console', `服务端未在 ${Math.round(ms / 1000)} 秒内退出,强制结束`) try { this.child.kill('SIGKILL') } catch { // 进程可能已经没了 } }, ms) } async restart() { await this.stop() this.restarts = 0 return this.start() } startHealthCheck() { this.stopHealthCheck() const interval = Number(this.server.healthIntervalMs) || 5000 const tick = () => this.probe() this.timers.setTimeout(tick, 800) this.healthTimer = this.timers.setInterval(tick, interval) } stopHealthCheck() { if (this.healthTimer) { this.timers.clearInterval(this.healthTimer) this.healthTimer = null } } probe() { if (!this.child) return const request = this.http.get( { host: this.healthHost(), port: this.server.port, path: '/api/health', timeout: 2500 }, (response) => { response.resume() const ok = response.statusCode === 200 this.setHealthy(ok) } ) request.on('timeout', () => request.destroy()) request.on('error', () => this.setHealthy(false)) } setHealthy(ok) { const changed = this.healthy !== ok if (ok) { // 探测通过 = 这一刻服务端还在正常服务,是「预期退出窗口」的起算点 this.lastAliveAt = this.now() if (!this.healthySince) this.healthySince = this.now() } else { this.healthySince = null } this.healthy = ok if (changed) { this.log.push('console', ok ? '服务端健康检查通过' : '服务端健康检查失败(等待重试)') } // 连续健康一段时间后清零崩溃计数:否则长时间赛事里历史崩溃会耗尽 // maxRestarts 配额,之后真崩了反而不自动拉起 let reset = false if (ok && this.restarts > 0) { const resetAfter = Number(this.server.healthyResetMs) || HEALTHY_RESET_MS if (this.now() - this.healthySince >= resetAfter) { this.restarts = 0 reset = true this.log.push('console', '服务端持续健康,崩溃计数已清零') } } if (changed || reset) this.emitStatus() } dispose() { // dispose 之后连子进程的 exit 都不再处理(见 handleExit 的守卫), // 否则「应用已退出、子进程后死」会重新排一次重启,留下一个没人观察的守护循环 this.disposed = true if (this.restartTimer) this.timers.clearTimeout(this.restartTimer) if (this.forceKillTimer) this.timers.clearTimeout(this.forceKillTimer) if (this.stopTimer) this.timers.clearTimeout(this.stopTimer) this.stopHealthCheck() this.removeAllListeners() } } module.exports = { ServiceSupervisor, healthHost }