fix(audiod): 长流卡死根治 — 防卡死护栏全量落地(AUDIOD_DESIGN §12)
根因(级联死锁): 内核驱动 wedge(hbc3000/cs43198 时钟域族, write/PREPARE/amixer ioctl 卡死在内核) → 播放线程永不返回 → ring 填满 → IO 线程在 ring_write 无超时 cond_wait 永久睡眠 → accept 停摆(backlog 满后 CLI ECONNREFUSED) → ncm 堵 socket 写 → 全链冻结。 同效路径: popen_read_tmo 无界 waitpid 随挂死的 amixer 子进程拖死持锁线程。 修法: - ring 写有界等待(MEOW_AUDIO_RING_WAIT_MS 缺省 5s)超时丢弃+计数(dropped=), IO 永生; 等待期 sid 代际守卫(抢占中等待的旧流 PCM 不再注入新流 ring) - 全部 cond 等待 timedwait 化(播放 1s 周期自愈, g_cond 改 CLOCK_MONOTONIC); 信号处理器只置 g_stop(不再调非 async-signal-safe 的 cond_signal) - popen 收割有界: WNOHANG 300ms → SIGKILL 300ms → D 态放弃收割(泄漏换永生); 路由线程 amixer 调用不持 g_mu; QUERY 改报 g_route_cur 缓存(IO 热路径不再 popen) - ALSA 改 O_NONBLOCK + poll(POLLOUT,1s): 连续 3 次停摆判 wedge → PREPARE 自救无效即 关卡进丢弃模式(stall=), 下一路 OPEN 重开重试 - server→client 下行帧单发 write 写不全即整帧丢弃(txdrop=), 杜绝半帧错位/慢客户端堵 IO - 主线程 watchdog: IO/播放心跳 10s 静默告警; io 线程 poll 出错不退出(listener 不丢); backlog 4→8; RESUME/PAUSE 补 cond 唤醒 - SDK meow_ui_audio.c + CLI: connect 非阻塞+poll 2s, SO_SNDTIMEO/RCVTIMEO 2s (audiod 真死时 App 报错重开, 不再永久阻塞) 验证: 宿主 38/38 PASS(旧 19 回归 + 新增 T5 长流 3min 等效 31.7MB 持续灌 played==written underrun/dropped=0 喂流中 listener 探活 15/15; T6 wedge 故障注入生存 — 旧代码同款注入 挂死(90s timeout 强制 kill), 新代码 IO/accept/preempt/SIGTERM 全活; T7 慢客户端 tx 丢弃); qemu-mipsel 静态二进制 status/tone 全通(written=played=13230 underrun=0); -Wall -Wextra 0 warning(host+mipsel+SDK 双端编译)。 meow_audiod md5 86a8b6793d516c90bbd12c2997b30d8f / meow_audio c23b96490842f54324b855fd9c2467a5 (fw_content.conf 已更)。暂不部署真机(主会话统一); meow_ui.a/App 重链留主会话。
This commit is contained in:
MeowOS committed
1 parent
243f328499
commit
e1e1331e44
10 files changed
+477
-67
No files matched your search
@@ -18,10 +18,12 @@
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <math.h>
|
||||
#include <poll.h>
|
||||
#include <signal.h>
|
||||
#include <time.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
#include <sys/wait.h>
|
||||
@@ -38,17 +40,36 @@ static void check(int cond, const char *name)
|
||||
else { printf(" FAIL %s\n", name); g_fail++; }
|
||||
}
|
||||
|
||||
/* ---- 原始 IPC 客户端 ---- */
|
||||
/* ---- 原始 IPC 客户端（全有界：connect 非阻塞 2s + SO_SNDTIMEO 2s——回归卡死时
|
||||
* 测试 FAIL 而不是跟着挂死） ---- */
|
||||
struct cli { int fd; };
|
||||
static int cli_connect(struct cli *c)
|
||||
{
|
||||
c->fd = socket(AF_UNIX, SOCK_STREAM, 0);
|
||||
if (c->fd < 0) return -1;
|
||||
int fl = fcntl(c->fd, F_GETFL, 0);
|
||||
if (fl >= 0) fcntl(c->fd, F_SETFL, fl | O_NONBLOCK);
|
||||
struct sockaddr_un a;
|
||||
memset(&a, 0, sizeof a);
|
||||
a.sun_family = AF_UNIX;
|
||||
snprintf(a.sun_path, sizeof a.sun_path, "%s", TSOCK);
|
||||
if (connect(c->fd, (struct sockaddr *)&a, sizeof a) < 0) { close(c->fd); return -1; }
|
||||
int r = connect(c->fd, (struct sockaddr *)&a, sizeof a);
|
||||
if (r < 0 && errno == EINPROGRESS) {
|
||||
struct pollfd p;
|
||||
p.fd = c->fd; p.events = POLLOUT; p.revents = 0;
|
||||
if (poll(&p, 1, 2000) > 0) {
|
||||
int err = 0;
|
||||
socklen_t el = sizeof err;
|
||||
if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &el) == 0 && err != 0) { errno = err; r = -1; }
|
||||
} else {
|
||||
r = -1;
|
||||
}
|
||||
}
|
||||
if (r < 0) { close(c->fd); return -1; }
|
||||
if (fl >= 0) fcntl(c->fd, F_SETFL, fl);
|
||||
struct timeval tv;
|
||||
tv.tv_sec = 2; tv.tv_usec = 0;
|
||||
setsockopt(c->fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof tv);
|
||||
return 0;
|
||||
}
|
||||
static int w_all(int fd, const void *p, size_t n)
|
||||
@@ -96,20 +117,42 @@ static void stop_audiod(void)
|
||||
if (g_audiod_pid > 0) { kill(g_audiod_pid, SIGTERM); waitpid(g_audiod_pid, NULL, 0); g_audiod_pid = -1; }
|
||||
}
|
||||
|
||||
static int start_audiod(void)
|
||||
/* 带超时的停止：SIGTERM 后最多等 tmo_ms；不退 → SIGKILL 并返回 0（卡死回归显形）。 */
|
||||
static int stop_audiod_tmo(int tmo_ms)
|
||||
{
|
||||
if (g_audiod_pid <= 0) return 1;
|
||||
kill(g_audiod_pid, SIGTERM);
|
||||
int ok = 0;
|
||||
for (int i = 0; i < tmo_ms / 50; i++) {
|
||||
if (waitpid(g_audiod_pid, NULL, WNOHANG) == g_audiod_pid) { ok = 1; break; }
|
||||
usleep(50000);
|
||||
}
|
||||
if (!ok) { kill(g_audiod_pid, SIGKILL); waitpid(g_audiod_pid, NULL, 0); }
|
||||
g_audiod_pid = -1;
|
||||
return ok;
|
||||
}
|
||||
|
||||
static int start_audiod_ex(const char *sink, const char *wedge_after, const char *ring_wait_ms)
|
||||
{
|
||||
pid_t pid = fork();
|
||||
if (pid == 0) {
|
||||
setenv("MEOW_AUDIO_SINK", "file", 1);
|
||||
setenv("MEOW_AUDIO_SINK", sink, 1);
|
||||
setenv("MEOW_AUDIO_OUT", TWAV, 1);
|
||||
setenv("MEOW_AUDIO_SOCK", TSOCK, 1);
|
||||
setenv("MEOW_AUDIO_FORCE", "1", 1);
|
||||
if (wedge_after) setenv("MEOW_AUDIO_TEST_WEDGE_AFTER_BYTES", wedge_after, 1);
|
||||
if (ring_wait_ms) setenv("MEOW_AUDIO_RING_WAIT_MS", ring_wait_ms, 1);
|
||||
execl("./meow_audiod_host", "meow_audiod_host", (char *)0);
|
||||
_exit(127);
|
||||
}
|
||||
return pid;
|
||||
}
|
||||
|
||||
static int start_audiod(void)
|
||||
{
|
||||
return start_audiod_ex("file", NULL, NULL);
|
||||
}
|
||||
|
||||
/* 直接客户端 open */
|
||||
static int do_open(struct cli *c, int rate, int ch, int fmt, m_audio_open_ack_t *ack)
|
||||
{
|
||||
@@ -289,6 +332,65 @@ int main(void)
|
||||
check(strstr(q, "active=0") != NULL, "EOF recycle -> inactive");
|
||||
close(c3q.fd);
|
||||
|
||||
/* ---- T5：长流回归（3 分钟等效 PCM 持续灌；FILE sink 全速排空 → 加速时钟等效）。
|
||||
* 回归目标：真机「长流卡死」——喂流期间 listener/IO 循环必须持续可服务，
|
||||
* 帧数零丢失、零 underrun、零丢弃，STOP 后排空落盘完整。 ---- */
|
||||
printf("[T5 long stream (3min equivalent)]\n");
|
||||
struct cli l5;
|
||||
check(cli_connect(&l5) == 0, "connect L5");
|
||||
m_audio_open_ack_t ackL5;
|
||||
check(do_open(&l5, 44100, 2, AUDIO_FMT_S16, &ackL5) == 0 && ackL5.err == MEOW_OK, "open L5");
|
||||
uint32_t fs5 = (uint32_t)ackL5.grant_ch * (ackL5.grant_fmt == AUDIO_FMT_S16 ? 2u : 4u);
|
||||
if (fs5 == 0) fs5 = 4;
|
||||
uint64_t total_frames = 44100ULL * 180; /* 3 分钟等效流 */
|
||||
uint64_t total_bytes = total_frames * fs5;
|
||||
uint8_t chunk[4096];
|
||||
memset(chunk, 0x5A, sizeof chunk); /* 内容无关：测完整性/不卡死 */
|
||||
size_t per = (sizeof chunk / fs5) * fs5;
|
||||
uint64_t sent = 0;
|
||||
int werr = 0, probe_n = 0, probe_ok = 0;
|
||||
while (sent < total_bytes) {
|
||||
uint64_t w = total_bytes - sent;
|
||||
if (w > per) w = per;
|
||||
if (do_write(&l5, chunk, (uint32_t)w) != 0) { werr = 1; break; }
|
||||
sent += w;
|
||||
if (sent % (512 * per) == 0) { /* 每 2MB 探一次 listener 活性 */
|
||||
struct cli s5;
|
||||
s5.fd = -1;
|
||||
probe_n++;
|
||||
if (cli_connect(&s5) == 0) {
|
||||
char q5[1024];
|
||||
if (do_query(&s5, q5, sizeof q5) == 0 && strstr(q5, "active=1")) probe_ok++;
|
||||
close(s5.fd);
|
||||
}
|
||||
}
|
||||
}
|
||||
check(!werr && sent == total_bytes, "long feed completed (io never stalled)");
|
||||
printf(" fed %llu bytes (%llu frames), mid-feed status probes %d/%d alive\n",
|
||||
(unsigned long long)sent, (unsigned long long)(sent / fs5), probe_ok, probe_n);
|
||||
check(probe_n >= 10 && probe_ok == probe_n, "listener/io alive during long feed");
|
||||
do_ctrl(&l5, MSG_AUDIO_STOP, ackL5.stream_id);
|
||||
char expect[64];
|
||||
snprintf(expect, sizeof expect, "played=%llu", (unsigned long long)total_frames);
|
||||
int drained = 0;
|
||||
char q5[1024];
|
||||
for (int i = 0; i < 300; i++) { /* 最多 30s 等排空 */
|
||||
if (do_query(&l5, q5, sizeof q5) == 0 && strstr(q5, expect)) { drained = 1; break; }
|
||||
usleep(100000);
|
||||
}
|
||||
check(drained, "long stream fully played (played == fed frames)");
|
||||
if (drained)
|
||||
check(strstr(q5, "underrun=0") != NULL && strstr(q5, "dropped=0") != NULL,
|
||||
"no underrun/drop on long stream");
|
||||
usleep(300000); /* 等 dev_close 回填 WAV 头 */
|
||||
close(l5.fd);
|
||||
{
|
||||
FILE *f = fopen(TWAV, "rb");
|
||||
long sz = -1;
|
||||
if (f) { fseek(f, 0, SEEK_END); sz = ftell(f); fclose(f); }
|
||||
check(sz == (long)(44 + total_bytes), "long WAV size == 44 + fed bytes");
|
||||
}
|
||||
|
||||
/* ---- T4：tone 写 WAV（由 CLI 走完整 IPC 链路；mipsel/host CLI 共用） ---- */
|
||||
printf("[T4 tone -> WAV]\n");
|
||||
unlink(TWAV);
|
||||
@@ -305,6 +407,78 @@ int main(void)
|
||||
{ FILE *f = fopen(TWAV, "rb"); if (f) { fseek(f, 0, SEEK_END); printf("%ld bytes\n", ftell(f)); fclose(f); } }
|
||||
|
||||
stop_audiod();
|
||||
|
||||
/* ---- T6：设备 wedge 生存（故障注入复现真机长流卡死场景）。
|
||||
* MEOW_AUDIO_TEST_WEDGE_AFTER_BYTES=128KB → 播放线程永久堵在 dev_write（睡到 g_stop）。
|
||||
* 旧代码：ring 满 → IO 线程 ring_write cond_wait 永不醒 → accept 停摆 → 全链冻结。
|
||||
* 护栏后：ring 写有界（RING_WAIT_MS=50ms）超时丢弃，IO/accept/preempt 全部存活。 ---- */
|
||||
printf("[T6 device wedge survival]\n");
|
||||
unlink(TSOCK);
|
||||
g_audiod_pid = start_audiod_ex("null", "131072", "50");
|
||||
usleep(400000);
|
||||
struct cli w6;
|
||||
check(cli_connect(&w6) == 0, "connect W6");
|
||||
m_audio_open_ack_t ackW6;
|
||||
check(do_open(&w6, 44100, 2, AUDIO_FMT_S16, &ackW6) == 0 && ackW6.err == MEOW_OK, "open W6");
|
||||
int werr6 = 0;
|
||||
uint64_t sent6 = 0;
|
||||
for (int i = 0; i < 100; i++) { /* 400KB：128KB 后 wedge，其余全靠丢弃 */
|
||||
if (do_write(&w6, chunk, (uint32_t)per) != 0) { werr6 = 1; break; }
|
||||
sent6 += per;
|
||||
}
|
||||
check(!werr6 && sent6 == 100 * (uint64_t)per, "feed continues across wedge (no deadlock)");
|
||||
struct cli s6;
|
||||
s6.fd = -1;
|
||||
char q6[1024];
|
||||
int io_alive = 0;
|
||||
if (cli_connect(&s6) == 0) {
|
||||
if (do_query(&s6, q6, sizeof q6) == 0 && strstr(q6, "route_mode=")) io_alive = 1;
|
||||
close(s6.fd);
|
||||
}
|
||||
check(io_alive, "io loop alive after wedge (connect+query served)");
|
||||
check(io_alive && strstr(q6, "dropped=") != NULL && atoi(strstr(q6, "dropped=") + 8) > 0,
|
||||
"drop policy engaged (dropped>0)");
|
||||
struct cli p6;
|
||||
p6.fd = -1;
|
||||
int pre6 = 0;
|
||||
if (cli_connect(&p6) == 0) {
|
||||
m_audio_open_ack_t ak6;
|
||||
if (do_open(&p6, 48000, 2, AUDIO_FMT_S16, &ak6) == 0 && ak6.err == MEOW_OK) pre6 = 1;
|
||||
close(p6.fd);
|
||||
}
|
||||
check(pre6, "preempt OPEN granted while wedged");
|
||||
close(w6.fd);
|
||||
check(stop_audiod_tmo(5000) != 0, "wedged audiod exits on SIGTERM within 5s");
|
||||
|
||||
/* ---- T7：慢/不读客户端下行丢弃（server→client 写绝不堵 IO 循环）。
|
||||
* Z 狂发 QUERY 从不读响应 → server 下行缓冲灌满 → send_frame 单发即丢（txdrop++）。
|
||||
* 护栏断言：server 不卡死、新连接照常服务、txdrop 有计数。 ---- */
|
||||
printf("[T7 slow-client tx drop]\n");
|
||||
unlink(TSOCK);
|
||||
g_audiod_pid = start_audiod_ex("null", NULL, NULL);
|
||||
usleep(400000);
|
||||
struct cli z7;
|
||||
check(cli_connect(&z7) == 0, "connect Z7");
|
||||
int serr7 = 0, sent7 = 0;
|
||||
for (int i = 0; i < 2000; i++) {
|
||||
if (sf(z7.fd, MSG_AUDIO_QUERY, 9, NULL, 0) != 0) { serr7 = 1; break; }
|
||||
sent7++;
|
||||
}
|
||||
check(!serr7 && sent7 == 2000, "query spam sent (client never reads)");
|
||||
struct cli f7;
|
||||
f7.fd = -1;
|
||||
char q7[1024];
|
||||
int alive7 = 0;
|
||||
if (cli_connect(&f7) == 0) {
|
||||
if (do_query(&f7, q7, sizeof q7) == 0 && strstr(q7, "route_mode=")) alive7 = 1;
|
||||
close(f7.fd);
|
||||
}
|
||||
check(alive7, "server responsive after tx drops");
|
||||
check(alive7 && strstr(q7, "txdrop=") != NULL && atoi(strstr(q7, "txdrop=") + 7) > 0,
|
||||
"txdrop counted");
|
||||
close(z7.fd);
|
||||
stop_audiod();
|
||||
|
||||
printf("=== SUMMARY: pass=%d fail=%d ===\n", g_pass, g_fail);
|
||||
return g_fail ? 1 : 0;
|
||||
}
|
||||
Reference in new issue
Block a user