fix(reader): txt编码误判(UTF-8前窗截断判GB18030致乱码) + build_index无持久化(整本重排慢/UI阻塞卡) — ①rd_utf8_sample_ok结尾不完整序列按合法(中间非法仍拒,GBK不误判,新增t13回归) ②索引落盘MeowData/os.meow.reader/index/*.idx(失效键size/mtime/font/fs/ls/wh/enc,命中秒开,原子写+强自检弃坏缓存,编码/字号/行距重排重写) 重开<0.3s/首开建cache; test_reader_core 72-0, 0warning

This commit is contained in:
AskaEth committed 2026-11-04 19:47:10 +08:00
1 parent 68710d7ed8
commit a64ae06eb6
5 files changed
+257 -4

No files matched your search

+35 -2
View File
@@ -35,6 +35,39 @@ int rd_utf8_validate(const uint8_t *buf, size_t len)
return 1;
}
/* 样本宽容 UTF-8 校验:与 rd_utf8_validate 同构,但对「缓冲末尾截断的多字节序列」按合法处理。
* 用途:rd_enc_detect 的样本是文件前 64KB 前窗,大 UTF-8 书几乎必然在该处把一个多字节汉字
* 切成两半;严格校验会因末尾半个字返回 0 → 误落 GB18030 → 全书乱码。此处仅放宽「结尾不完整
* 序列」,任何中间非法字节(overlong/代理/孤立续字节/非法首字节)仍返回 0,故 GBK/GB18030
* 样本不会因此被误判为 UTF-8(GBK 字节对在中途即因非法首/续字节被拒)。 */
static int rd_utf8_sample_ok(const uint8_t *buf, size_t len)
{
size_t i = 0;
while (i < len) {
uint8_t b = buf[i];
if (b < 0x80) { i++; continue; }
unsigned n, lo, hi;
if ((b & 0xE0) == 0xC0) { n = 2; lo = 0x80; hi = 0xBF; }
else if ((b & 0xF0) == 0xE0) { n = 3; lo = 0x80; hi = 0xBF; }
else if ((b & 0xF8) == 0xF0) { n = 4; lo = 0x80; hi = 0xBF; }
else return 0;
if (n == 2 && (b & 0xFE) == 0xC0) return 0; /* C0/C1 overlong */
if (n == 3 && b == 0xE0) lo = 0xA0; /* 防 overlong < U+0800 */
if (n == 3 && b == 0xED) hi = 0x9F; /* 防代理 U+D800..DFFF */
if (n == 4 && b == 0xF0) lo = 0x90; /* 防 overlong < U+10000 */
if (n == 4 && b == 0xF4) hi = 0x8F; /* 防 > U+10FFFF */
if (i + n > len) return 1; /* 末尾截断:宽容(前窗裁掉一半) */
for (unsigned k = 1; k < n; k++) {
uint8_t c = buf[i + k];
uint8_t cl = (k == 1) ? (uint8_t)lo : 0x80;
uint8_t ch = (k == 1) ? (uint8_t)hi : 0xBF;
if (c < cl || c > ch) return 0;
}
i += n;
}
return 1;
}
/* ================= GB18030 双字节 → UTF-8 ================= */
static inline int gb_trail_ok(unsigned b)
{
@@ -186,8 +219,8 @@ rd_enc_t rd_enc_detect(const uint8_t *buf, size_t len, rd_enc_stats *st)
if (len >= 2 && buf[0] == 0xFF && buf[1] == 0xFE) return RD_ENC_UTF16LE;
if (len >= 2 && buf[0] == 0xFE && buf[1] == 0xFF) return RD_ENC_UTF16BE;
/* 2) 严格 UTF-8 */
if (rd_utf8_validate(buf, len)) {
/* 2) 严格 UTF-8(样本宽容末尾截断,见 rd_utf8_sample_ok 注释) */
if (rd_utf8_sample_ok(buf, len)) {
/* 纯 ASCII 判 ASCII(信息性;UTF-8 子集) */
int ascii = 1;
for (size_t i = 0; i < len; i++) if (buf[i] >= 0x80) { ascii = 0; break; }