fix(reader): txt编码误判(UTF-8前窗截断判GB18030致乱码) + build_index无持久化(整本重排慢/UI阻塞卡) — ①rd_utf8_sample_ok结尾不完整序列按合法(中间非法仍拒,GBK不误判,新增t13回归) ②索引落盘MeowData/os.meow.reader/index/*.idx(失效键size/mtime/font/fs/ls/wh/enc,命中秒开,原子写+强自检弃坏缓存,编码/字号/行距重排重写) 重开<0.3s/首开建cache; test_reader_core 72-0, 0warning
This commit is contained in:
1 parent
68710d7ed8
commit
a64ae06eb6
5 files changed
+257
-4
No files matched your search
@@ -35,6 +35,39 @@ int rd_utf8_validate(const uint8_t *buf, size_t len)
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* 样本宽容 UTF-8 校验:与 rd_utf8_validate 同构,但对「缓冲末尾截断的多字节序列」按合法处理。
|
||||
* 用途:rd_enc_detect 的样本是文件前 64KB 前窗,大 UTF-8 书几乎必然在该处把一个多字节汉字
|
||||
* 切成两半;严格校验会因末尾半个字返回 0 → 误落 GB18030 → 全书乱码。此处仅放宽「结尾不完整
|
||||
* 序列」,任何中间非法字节(overlong/代理/孤立续字节/非法首字节)仍返回 0,故 GBK/GB18030
|
||||
* 样本不会因此被误判为 UTF-8(GBK 字节对在中途即因非法首/续字节被拒)。 */
|
||||
static int rd_utf8_sample_ok(const uint8_t *buf, size_t len)
|
||||
{
|
||||
size_t i = 0;
|
||||
while (i < len) {
|
||||
uint8_t b = buf[i];
|
||||
if (b < 0x80) { i++; continue; }
|
||||
unsigned n, lo, hi;
|
||||
if ((b & 0xE0) == 0xC0) { n = 2; lo = 0x80; hi = 0xBF; }
|
||||
else if ((b & 0xF0) == 0xE0) { n = 3; lo = 0x80; hi = 0xBF; }
|
||||
else if ((b & 0xF8) == 0xF0) { n = 4; lo = 0x80; hi = 0xBF; }
|
||||
else return 0;
|
||||
if (n == 2 && (b & 0xFE) == 0xC0) return 0; /* C0/C1 overlong */
|
||||
if (n == 3 && b == 0xE0) lo = 0xA0; /* 防 overlong < U+0800 */
|
||||
if (n == 3 && b == 0xED) hi = 0x9F; /* 防代理 U+D800..DFFF */
|
||||
if (n == 4 && b == 0xF0) lo = 0x90; /* 防 overlong < U+10000 */
|
||||
if (n == 4 && b == 0xF4) hi = 0x8F; /* 防 > U+10FFFF */
|
||||
if (i + n > len) return 1; /* 末尾截断:宽容(前窗裁掉一半) */
|
||||
for (unsigned k = 1; k < n; k++) {
|
||||
uint8_t c = buf[i + k];
|
||||
uint8_t cl = (k == 1) ? (uint8_t)lo : 0x80;
|
||||
uint8_t ch = (k == 1) ? (uint8_t)hi : 0xBF;
|
||||
if (c < cl || c > ch) return 0;
|
||||
}
|
||||
i += n;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* ================= GB18030 双字节 → UTF-8 ================= */
|
||||
static inline int gb_trail_ok(unsigned b)
|
||||
{
|
||||
@@ -186,8 +219,8 @@ rd_enc_t rd_enc_detect(const uint8_t *buf, size_t len, rd_enc_stats *st)
|
||||
if (len >= 2 && buf[0] == 0xFF && buf[1] == 0xFE) return RD_ENC_UTF16LE;
|
||||
if (len >= 2 && buf[0] == 0xFE && buf[1] == 0xFF) return RD_ENC_UTF16BE;
|
||||
|
||||
/* 2) 严格 UTF-8 */
|
||||
if (rd_utf8_validate(buf, len)) {
|
||||
/* 2) 严格 UTF-8(样本宽容末尾截断,见 rd_utf8_sample_ok 注释) */
|
||||
if (rd_utf8_sample_ok(buf, len)) {
|
||||
/* 纯 ASCII 判 ASCII(信息性;UTF-8 子集) */
|
||||
int ascii = 1;
|
||||
for (size_t i = 0; i < len; i++) if (buf[i] >= 0x80) { ascii = 0; break; }
|
||||
|
||||
Reference in new issue
Block a user