fix(reader): txt编码误判(UTF-8前窗截断判GB18030致乱码) + build_index无持久化(整本重排慢/UI阻塞卡) — ①rd_utf8_sample_ok结尾不完整序列按合法(中间非法仍拒,GBK不误判,新增t13回归) ②索引落盘MeowData/os.meow.reader/index/*.idx(失效键size/mtime/font/fs/ls/wh/enc,命中秒开,原子写+强自检弃坏缓存,编码/字号/行距重排重写) 重开<0.3s/首开建cache; test_reader_core 72-0, 0warning
This commit is contained in:
1 parent
68710d7ed8
commit
a64ae06eb6
5 files changed
+257
-4
No files matched your search
@@ -14,6 +14,7 @@
|
|||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
|
||||||
typedef struct seg {
|
typedef struct seg {
|
||||||
int page_start; /* 全局页表起点 */
|
int page_start; /* 全局页表起点 */
|
||||||
@@ -45,6 +46,42 @@ struct rd_book {
|
|||||||
int seg_cache_idx;
|
int seg_cache_idx;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/* ---- 页索引持久化(MeowData;见 rd_book_set_index_dir) ----
|
||||||
|
* build_index 对整书逐段 reflow 建 flat 页表,是开大书最贵的步骤(无持久化时每次打开都重跑)。
|
||||||
|
* 宿主快 ≠ 真机快:同一书 MIPS 上仍需数秒。这里把 (segs + pages) 落盘成 {path 哈希}.idx,
|
||||||
|
* 以 (file_size, file_mtime, font path+mt, fs, ls, content_w, content_h, enc) 为失效键——
|
||||||
|
* 书/字号/行距/版面/编码/字体任一变了即自动重排并重写,其余情况命中缓存秒开。
|
||||||
|
* 纯 libc(无 SDK):reader.c 用 meow_config_dir 推导 MeowData dir 注入;冒烟/无卡时 dir 为空 → 直排不持久。 */
|
||||||
|
|
||||||
|
static char g_index_dir[400]; /* 索引缓存目录; 空 = 禁用 */
|
||||||
|
void rd_book_set_index_dir(const char *dir)
|
||||||
|
{
|
||||||
|
if (dir && dir[0]) { strncpy(g_index_dir, dir, sizeof g_index_dir - 1); g_index_dir[sizeof g_index_dir - 1] = 0; }
|
||||||
|
else g_index_dir[0] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
#define IDX_MAGIC0 'R'
|
||||||
|
#define IDX_MAGIC1 'I'
|
||||||
|
#define IDX_MAGIC2 'D'
|
||||||
|
#define IDX_MAGIC3 'X'
|
||||||
|
#define IDX_VER 1u
|
||||||
|
|
||||||
|
static int fio_rd(FILE *f, void *p, size_t n) { return fread(p, 1, n, f) == n; }
|
||||||
|
static int fio_wr(FILE *f, const void *p, size_t n) { return fwrite(p, 1, n, f) == n; }
|
||||||
|
|
||||||
|
static uint32_t idx_fnv1a(const char *s)
|
||||||
|
{
|
||||||
|
uint32_t h = 2166136261u;
|
||||||
|
for (const unsigned char *p = (const unsigned char *)s; *p; p++) { h ^= *p; h *= 16777619u; }
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
static int index_cache_path(const rd_book *b, char *out, int outsz)
|
||||||
|
{
|
||||||
|
if (!g_index_dir[0]) return -1;
|
||||||
|
if (snprintf(out, (size_t)outsz, "%s/%08x.idx", g_index_dir, idx_fnv1a(b->path)) >= outsz) return -1;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
/* 编码枚举 → 可读名(R3:Big5 正确解码后名实相符)。 */
|
/* 编码枚举 → 可读名(R3:Big5 正确解码后名实相符)。 */
|
||||||
static const char *enc_to_name(rd_enc_t enc)
|
static const char *enc_to_name(rd_enc_t enc)
|
||||||
{
|
{
|
||||||
@@ -71,6 +108,135 @@ static int book_meas(uint32_t cp, int fs, int *adv, void *ctx)
|
|||||||
return rdft_advance(b->ft, cp, adv);
|
return rdft_advance(b->ft, cp, adv);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* 失效键:书 + 版面 + 编码 + 字体。任一变化 → 缓存不命中(重排)。 */
|
||||||
|
typedef struct {
|
||||||
|
uint64_t file_size;
|
||||||
|
int64_t file_mtime;
|
||||||
|
int64_t font_mtime; /* 字体文件 mtime;无字体(近似测宽) = 0 */
|
||||||
|
int32_t fs, ls, cw, ch, enc;
|
||||||
|
char font[320];
|
||||||
|
} idx_key;
|
||||||
|
|
||||||
|
static int index_key_fill(rd_book *b, idx_key *k)
|
||||||
|
{
|
||||||
|
memset(k, 0, sizeof *k);
|
||||||
|
struct stat st;
|
||||||
|
if (stat(b->path, &st) != 0) { k->file_size = 0; k->file_mtime = 0; }
|
||||||
|
else { k->file_size = (uint64_t)st.st_size; k->file_mtime = (int64_t)st.st_mtime; }
|
||||||
|
if (b->font[0]) {
|
||||||
|
/* b->font[400] 已 NUL 结尾(open 时 <400 才收);memcpy 319B 恒在界内且 k 已清零 → 余下为 0 */
|
||||||
|
memcpy(k->font, b->font, sizeof k->font - 1);
|
||||||
|
struct stat fs2;
|
||||||
|
if (stat(b->font, &fs2) == 0) k->font_mtime = (int64_t)fs2.st_mtime;
|
||||||
|
}
|
||||||
|
k->fs = b->fs; k->ls = b->ls; k->cw = b->content_w; k->ch = b->content_h;
|
||||||
|
k->enc = (int32_t)b->enc;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int index_key_write(const idx_key *k, FILE *f)
|
||||||
|
{
|
||||||
|
const uint8_t magic[4] = { IDX_MAGIC0, IDX_MAGIC1, IDX_MAGIC2, IDX_MAGIC3 };
|
||||||
|
uint16_t ver = IDX_VER;
|
||||||
|
return fio_wr(f, magic, sizeof magic) && fio_wr(f, &ver, sizeof ver) &&
|
||||||
|
fio_wr(f, &k->file_size, sizeof k->file_size) &&
|
||||||
|
fio_wr(f, &k->file_mtime, sizeof k->file_mtime) && fio_wr(f, &k->font_mtime, sizeof k->font_mtime) &&
|
||||||
|
fio_wr(f, &k->fs, sizeof k->fs) && fio_wr(f, &k->ls, sizeof k->ls) &&
|
||||||
|
fio_wr(f, &k->cw, sizeof k->cw) && fio_wr(f, &k->ch, sizeof k->ch) &&
|
||||||
|
fio_wr(f, &k->enc, sizeof k->enc) && fio_wr(f, k->font, sizeof k->font);
|
||||||
|
}
|
||||||
|
static int index_key_read(idx_key *k, FILE *f)
|
||||||
|
{
|
||||||
|
uint8_t magic[4];
|
||||||
|
if (!fio_rd(f, magic, sizeof magic)) return 0;
|
||||||
|
if (magic[0] != IDX_MAGIC0 || magic[1] != IDX_MAGIC1 || magic[2] != IDX_MAGIC2 || magic[3] != IDX_MAGIC3) return 0;
|
||||||
|
uint16_t ver = 0;
|
||||||
|
if (!fio_rd(f, &ver, sizeof ver) || ver != IDX_VER) return 0;
|
||||||
|
if (!fio_rd(f, &k->file_size, sizeof k->file_size) || !fio_rd(f, &k->file_mtime, sizeof k->file_mtime) ||
|
||||||
|
!fio_rd(f, &k->font_mtime, sizeof k->font_mtime) || !fio_rd(f, &k->fs, sizeof k->fs) ||
|
||||||
|
!fio_rd(f, &k->ls, sizeof k->ls) || !fio_rd(f, &k->cw, sizeof k->cw) ||
|
||||||
|
!fio_rd(f, &k->ch, sizeof k->ch) || !fio_rd(f, &k->enc, sizeof k->enc) ||
|
||||||
|
!fio_rd(f, k->font, sizeof k->font)) return 0;
|
||||||
|
k->font[sizeof k->font - 1] = 0;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
static int index_key_eq(const idx_key *a, const idx_key *b)
|
||||||
|
{
|
||||||
|
return a->file_size == b->file_size && a->file_mtime == b->file_mtime &&
|
||||||
|
a->font_mtime == b->font_mtime && a->fs == b->fs && a->ls == b->ls &&
|
||||||
|
a->cw == b->cw && a->ch == b->ch && a->enc == b->enc &&
|
||||||
|
strncmp(a->font, b->font, sizeof a->font) == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* 读缓存:头部与当前键一致则载入 (segs,pages) 并自检一致;返回 1=命中,0=未命中/损坏(调用方重排)。 */
|
||||||
|
static int index_cache_load(rd_book *b)
|
||||||
|
{
|
||||||
|
char path[440];
|
||||||
|
if (index_cache_path(b, path, sizeof path) != 0) return 0;
|
||||||
|
FILE *f = fopen(path, "rb");
|
||||||
|
if (!f) return 0;
|
||||||
|
idx_key key, cur;
|
||||||
|
int rc = 0;
|
||||||
|
if (index_key_read(&key, f)) {
|
||||||
|
index_key_fill(b, &cur);
|
||||||
|
if (index_key_eq(&key, &cur)) {
|
||||||
|
int32_t nsegs = 0, npages = 0;
|
||||||
|
if (fio_rd(f, &nsegs, sizeof nsegs) && fio_rd(f, &npages, sizeof npages) &&
|
||||||
|
nsegs >= 1 && npages >= 1 && nsegs <= 1000000 && npages <= 8000000 &&
|
||||||
|
(uint64_t)npages <= cur.file_size + 1 && (uint64_t)nsegs <= cur.file_size + 1) {
|
||||||
|
seg_t *segs = (seg_t *)malloc((size_t)nsegs * sizeof(seg_t));
|
||||||
|
rd_book_page *pages = (rd_book_page *)malloc((size_t)npages * sizeof(rd_book_page));
|
||||||
|
if (segs && pages &&
|
||||||
|
fio_rd(f, segs, (size_t)nsegs * sizeof(seg_t)) &&
|
||||||
|
fio_rd(f, pages, (size_t)npages * sizeof(rd_book_page))) {
|
||||||
|
/* 自检:page_count 求和 == npages 且 page_start 链正确;每页 seg 落在所属段区间。 */
|
||||||
|
int ok = 1, run = 0;
|
||||||
|
for (int32_t i = 0; i < nsegs && ok; i++) {
|
||||||
|
if (segs[i].page_count < 0 || segs[i].page_start != run) ok = 0;
|
||||||
|
else run += segs[i].page_count;
|
||||||
|
}
|
||||||
|
if (ok && run == npages) {
|
||||||
|
for (int32_t p = 0; p < npages && ok; p++) {
|
||||||
|
int32_t sg = pages[p].seg;
|
||||||
|
if (sg < 0 || sg >= nsegs || pages[p].seg_char < 0 ||
|
||||||
|
p < segs[sg].page_start || p >= segs[sg].page_start + segs[sg].page_count) ok = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (ok) {
|
||||||
|
b->segs = segs; b->pages = pages;
|
||||||
|
b->nsegs = (int)nsegs; b->npages = (int)npages;
|
||||||
|
rc = 1; /* 命中:调用方免 build_index */
|
||||||
|
} else { free(segs); free(pages); }
|
||||||
|
} else { free(segs); free(pages); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fclose(f);
|
||||||
|
return rc;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* 写缓存:先写 .tmp 再 rename(原子替换);失败静默(下次重排)。 */
|
||||||
|
static void index_cache_save(rd_book *b)
|
||||||
|
{
|
||||||
|
char path[440];
|
||||||
|
if (b->nsegs <= 0 || b->npages <= 0) return; /* 空书无页表可缓存 */
|
||||||
|
if (index_cache_path(b, path, sizeof path) != 0) return;
|
||||||
|
/* 目录由 reader.c 在 set_index_dir 前建好;这里 fopen 失败(无目录/只读)→ 静默跳过(下次重排)。 */
|
||||||
|
char tmp[480];
|
||||||
|
if (snprintf(tmp, sizeof tmp, "%s.tmp", path) >= (int)sizeof tmp) return;
|
||||||
|
FILE *f = fopen(tmp, "wb");
|
||||||
|
if (!f) return;
|
||||||
|
idx_key key;
|
||||||
|
int ok = 1;
|
||||||
|
index_key_fill(b, &key);
|
||||||
|
ok = ok && index_key_write(&key, f);
|
||||||
|
ok = ok && fio_wr(f, &b->nsegs, sizeof b->nsegs) && fio_wr(f, &b->npages, sizeof b->npages);
|
||||||
|
if (ok && b->nsegs > 0) ok = fio_wr(f, b->segs, (size_t)b->nsegs * sizeof(seg_t));
|
||||||
|
if (ok && b->npages > 0) ok = fio_wr(f, b->pages, (size_t)b->npages * sizeof(rd_book_page));
|
||||||
|
fclose(f);
|
||||||
|
if (ok) { rename(tmp, path); } else { remove(tmp); }
|
||||||
|
}
|
||||||
|
|
||||||
/* ---- 文本:建索引(逐段 layout) ---- */
|
/* ---- 文本:建索引(逐段 layout) ---- */
|
||||||
static int build_index(rd_book *b)
|
static int build_index(rd_book *b)
|
||||||
{
|
{
|
||||||
@@ -182,7 +348,11 @@ int rd_book_open(rd_book **b, const char *path, const char *font_path,
|
|||||||
if (font_path && rdft_load_font(x->ft, x->font) != 0) { rdft_free(x->ft); x->ft = NULL; goto close; }
|
if (font_path && rdft_load_font(x->ft, x->font) != 0) { rdft_free(x->ft); x->ft = NULL; goto close; }
|
||||||
if (x->ft) rdft_set_px(x->ft, fs);
|
if (x->ft) rdft_set_px(x->ft, fs);
|
||||||
x->seg_cache_idx = -1;
|
x->seg_cache_idx = -1;
|
||||||
if (build_index(x) != 0) goto close;
|
/* 页索引缓存命中(书+字号+版面+编码+字体未变)→ 免整书重排,秒开;未命中 → 重排并落盘。 */
|
||||||
|
if (index_cache_load(x) != 1) {
|
||||||
|
if (build_index(x) != 0) goto close;
|
||||||
|
index_cache_save(x);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
int pages = rd_src_pages(x->src);
|
int pages = rd_src_pages(x->src);
|
||||||
x->npages = pages > 0 ? pages : 0;
|
x->npages = pages > 0 ? pages : 0;
|
||||||
@@ -250,6 +420,7 @@ int rd_book_set_enc(rd_book *b, rd_enc_t enc)
|
|||||||
free(b->segs); free(b->pages); b->segs = NULL; b->pages = NULL;
|
free(b->segs); free(b->pages); b->segs = NULL; b->pages = NULL;
|
||||||
b->nsegs = b->npages = 0;
|
b->nsegs = b->npages = 0;
|
||||||
if (build_index(b) != 0) return -3; /* 新 src 有效但索引空 → 渲染返回 -1,安全降级 */
|
if (build_index(b) != 0) return -3; /* 新 src 有效但索引空 → 渲染返回 -1,安全降级 */
|
||||||
|
index_cache_save(b); /* 编码切换后重新落盘(键含 enc,与旧缓存并存不冲突) */
|
||||||
return rd_book_jump_to_anchor(b, seg, ch, b->fs, b->ls);
|
return rd_book_jump_to_anchor(b, seg, ch, b->fs, b->ls);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -374,6 +545,7 @@ int rd_book_jump_to_anchor(rd_book *b, int seg, int32_t ch, int fs, int ls)
|
|||||||
b->segs = NULL; b->pages = NULL; b->nsegs = b->npages = 0;
|
b->segs = NULL; b->pages = NULL; b->nsegs = b->npages = 0;
|
||||||
if (build_index(b) != 0) { b->segs = nos; b->pages = nop; b->nsegs = n_oldsegs; b->npages = n_oldpages; return -5; }
|
if (build_index(b) != 0) { b->segs = nos; b->pages = nop; b->nsegs = n_oldsegs; b->npages = n_oldpages; return -5; }
|
||||||
free(nos); free(nop);
|
free(nos); free(nop);
|
||||||
|
index_cache_save(b); /* 字号/行距改变后重新落盘(键含 fs/ls) */
|
||||||
}
|
}
|
||||||
int target = 0;
|
int target = 0;
|
||||||
for (int p = 0; p < b->npages; p++) {
|
for (int p = 0; p < b->npages; p++) {
|
||||||
|
|||||||
@@ -64,6 +64,10 @@ const char *rd_book_path(const rd_book *b); /* 原始路径 */
|
|||||||
* 用于「编码」菜单(Big5 繁体书手动选定)。 */
|
* 用于「编码」菜单(Big5 繁体书手动选定)。 */
|
||||||
int rd_book_set_enc(rd_book *b, rd_enc_t enc);
|
int rd_book_set_enc(rd_book *b, rd_enc_t enc);
|
||||||
|
|
||||||
|
/* R3.1:页索引持久化缓存目录(MeowData/os.meow.reader/index)。dir 为空 → 禁持久化(直排,不落盘)。
|
||||||
|
* 由 reader.c 在 open 前建目录并注入;书/字号/行距/版面/编码/字体任一变化时缓存自动失效重排。 */
|
||||||
|
void rd_book_set_index_dir(const char *dir);
|
||||||
|
|
||||||
/* 字体/几何 */
|
/* 字体/几何 */
|
||||||
int rd_book_font(rd_book *b); /* 当前字号 */
|
int rd_book_font(rd_book *b); /* 当前字号 */
|
||||||
int rd_book_ls(rd_book *b); /* 行距% */
|
int rd_book_ls(rd_book *b); /* 行距% */
|
||||||
|
|||||||
@@ -633,6 +633,28 @@ static void reader_scrub_begin(int x, int y)
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* ---------- 打开 ---------- */
|
/* ---------- 打开 ---------- */
|
||||||
|
/* R3.1:页索引缓存目录 = <TF>/MeowData/os.meow.reader/index。由 meow_config_dir
|
||||||
|
* (<TF>/MeowData/<pkg>/data) 剥尾 "/data" → <TF>/MeowData/<pkg>,接 "index",mkdir -p 后注入。 */
|
||||||
|
static void reader_setup_index_dir(void)
|
||||||
|
{
|
||||||
|
const char *data = meow_config_dir();
|
||||||
|
if (!data || !data[0]) return; /* 无卡/宿主:不持久(book.c 直排) */
|
||||||
|
char base[420];
|
||||||
|
size_t n = strlen(data);
|
||||||
|
const char *suf = "/data"; size_t sl = strlen(suf);
|
||||||
|
size_t keep = (n >= sl && strcmp(data + n - sl, suf) == 0) ? n - sl : n;
|
||||||
|
if (keep >= sizeof base) keep = sizeof base - 1;
|
||||||
|
memcpy(base, data, keep); base[keep] = 0;
|
||||||
|
char dir[440];
|
||||||
|
if (snprintf(dir, sizeof dir, "%s/index", base) >= (int)sizeof dir) return;
|
||||||
|
/* mkdir -p 幂等(progress 同款) */
|
||||||
|
char t[440];
|
||||||
|
strncpy(t, dir, sizeof t - 1); t[sizeof t - 1] = 0;
|
||||||
|
for (char *c = t + 1; *c; c++) if (*c == '/') { *c = 0; mkdir(t, 0755); *c = '/'; }
|
||||||
|
mkdir(t, 0755);
|
||||||
|
rd_book_set_index_dir(dir);
|
||||||
|
}
|
||||||
|
|
||||||
int reader_open(const char *path)
|
int reader_open(const char *path)
|
||||||
{
|
{
|
||||||
if (!path || !path[0]) return -1;
|
if (!path || !path[0]) return -1;
|
||||||
@@ -661,6 +683,9 @@ int reader_open(const char *path)
|
|||||||
/* 阅读页底/字色:rd.bg(默认纯黑,B)→ 覆盖 th->surface */
|
/* 阅读页底/字色:rd.bg(默认纯黑,B)→ 覆盖 th->surface */
|
||||||
reader_resolve_bg(&g_page_bg, &g_page_fg);
|
reader_resolve_bg(&g_page_bg, &g_page_fg);
|
||||||
|
|
||||||
|
/* 页索引缓存(建好目录并注入;命中则开书免整书重排,秒开) */
|
||||||
|
reader_setup_index_dir();
|
||||||
|
|
||||||
/* 开书(FreeType 路径带齐几何;无字体则 g_has_font=0 → atlas 兜底) */
|
/* 开书(FreeType 路径带齐几何;无字体则 g_has_font=0 → atlas 兜底) */
|
||||||
int rc = rd_book_open(&g_book, path, fp, g_fs, g_ls, g_pw, g_ph, g_mx, g_my, g_top);
|
int rc = rd_book_open(&g_book, path, fp, g_fs, g_ls, g_pw, g_ph, g_mx, g_my, g_top);
|
||||||
if (rc != 0) {
|
if (rc != 0) {
|
||||||
|
|||||||
@@ -35,6 +35,39 @@ int rd_utf8_validate(const uint8_t *buf, size_t len)
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* 样本宽容 UTF-8 校验:与 rd_utf8_validate 同构,但对「缓冲末尾截断的多字节序列」按合法处理。
|
||||||
|
* 用途:rd_enc_detect 的样本是文件前 64KB 前窗,大 UTF-8 书几乎必然在该处把一个多字节汉字
|
||||||
|
* 切成两半;严格校验会因末尾半个字返回 0 → 误落 GB18030 → 全书乱码。此处仅放宽「结尾不完整
|
||||||
|
* 序列」,任何中间非法字节(overlong/代理/孤立续字节/非法首字节)仍返回 0,故 GBK/GB18030
|
||||||
|
* 样本不会因此被误判为 UTF-8(GBK 字节对在中途即因非法首/续字节被拒)。 */
|
||||||
|
static int rd_utf8_sample_ok(const uint8_t *buf, size_t len)
|
||||||
|
{
|
||||||
|
size_t i = 0;
|
||||||
|
while (i < len) {
|
||||||
|
uint8_t b = buf[i];
|
||||||
|
if (b < 0x80) { i++; continue; }
|
||||||
|
unsigned n, lo, hi;
|
||||||
|
if ((b & 0xE0) == 0xC0) { n = 2; lo = 0x80; hi = 0xBF; }
|
||||||
|
else if ((b & 0xF0) == 0xE0) { n = 3; lo = 0x80; hi = 0xBF; }
|
||||||
|
else if ((b & 0xF8) == 0xF0) { n = 4; lo = 0x80; hi = 0xBF; }
|
||||||
|
else return 0;
|
||||||
|
if (n == 2 && (b & 0xFE) == 0xC0) return 0; /* C0/C1 overlong */
|
||||||
|
if (n == 3 && b == 0xE0) lo = 0xA0; /* 防 overlong < U+0800 */
|
||||||
|
if (n == 3 && b == 0xED) hi = 0x9F; /* 防代理 U+D800..DFFF */
|
||||||
|
if (n == 4 && b == 0xF0) lo = 0x90; /* 防 overlong < U+10000 */
|
||||||
|
if (n == 4 && b == 0xF4) hi = 0x8F; /* 防 > U+10FFFF */
|
||||||
|
if (i + n > len) return 1; /* 末尾截断:宽容(前窗裁掉一半) */
|
||||||
|
for (unsigned k = 1; k < n; k++) {
|
||||||
|
uint8_t c = buf[i + k];
|
||||||
|
uint8_t cl = (k == 1) ? (uint8_t)lo : 0x80;
|
||||||
|
uint8_t ch = (k == 1) ? (uint8_t)hi : 0xBF;
|
||||||
|
if (c < cl || c > ch) return 0;
|
||||||
|
}
|
||||||
|
i += n;
|
||||||
|
}
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
/* ================= GB18030 双字节 → UTF-8 ================= */
|
/* ================= GB18030 双字节 → UTF-8 ================= */
|
||||||
static inline int gb_trail_ok(unsigned b)
|
static inline int gb_trail_ok(unsigned b)
|
||||||
{
|
{
|
||||||
@@ -186,8 +219,8 @@ rd_enc_t rd_enc_detect(const uint8_t *buf, size_t len, rd_enc_stats *st)
|
|||||||
if (len >= 2 && buf[0] == 0xFF && buf[1] == 0xFE) return RD_ENC_UTF16LE;
|
if (len >= 2 && buf[0] == 0xFF && buf[1] == 0xFE) return RD_ENC_UTF16LE;
|
||||||
if (len >= 2 && buf[0] == 0xFE && buf[1] == 0xFF) return RD_ENC_UTF16BE;
|
if (len >= 2 && buf[0] == 0xFE && buf[1] == 0xFF) return RD_ENC_UTF16BE;
|
||||||
|
|
||||||
/* 2) 严格 UTF-8 */
|
/* 2) 严格 UTF-8(样本宽容末尾截断,见 rd_utf8_sample_ok 注释) */
|
||||||
if (rd_utf8_validate(buf, len)) {
|
if (rd_utf8_sample_ok(buf, len)) {
|
||||||
/* 纯 ASCII 判 ASCII(信息性;UTF-8 子集) */
|
/* 纯 ASCII 判 ASCII(信息性;UTF-8 子集) */
|
||||||
int ascii = 1;
|
int ascii = 1;
|
||||||
for (size_t i = 0; i < len; i++) if (buf[i] >= 0x80) { ascii = 0; break; }
|
for (size_t i = 0; i < len; i++) if (buf[i] >= 0x80) { ascii = 0; break; }
|
||||||
|
|||||||
@@ -322,10 +322,29 @@ static void t12(void)
|
|||||||
remove(path);
|
remove(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* t13 — 大 UTF-8 书编码检测:前 64KB 窗口恰在多字节字符中间截断时仍判 UTF-8。
|
||||||
|
* 复现 M6.42《这游戏》乱码:rdtxt_probe 读前 64KB → rd_enc_detect 严格 UTF-8 校验因末尾
|
||||||
|
* 半个汉字失败 → 落入 GB18030 兜底 → 全书按 GB18030 解码 → 乱码。此处必须判回 UTF-8。 */
|
||||||
|
static void t13(void)
|
||||||
|
{
|
||||||
|
uint8_t buf[65536];
|
||||||
|
const uint8_t cjk[] = { 0xE4, 0xBD, 0xA0 }; /* 「你」= 3 字节 CJK */
|
||||||
|
size_t o = 0;
|
||||||
|
while (o + 3 <= 65535) { memcpy(buf + o, cjk, 3); o += 3; } /* o == 65535 */
|
||||||
|
buf[o] = 0xE4; /* 末尾 1 字节 = 半个汉字(3 字节序列被 64KB 窗口切断) */
|
||||||
|
size_t len = o + 1; /* 65536:rdtxt_probe 读满 64KB */
|
||||||
|
CHECK(len == 65536, "t13 len");
|
||||||
|
CHECK(rd_utf8_validate(buf, len) == 0, "t13 strict rejects trailing truncation");
|
||||||
|
CHECK(rd_enc_detect(buf, len, NULL) == RD_ENC_UTF8, "t13 detect utf8 despite mid-char cut");
|
||||||
|
/* 对照:GBK 仍须判 GB18030(样本宽容不得误伤 GBK) */
|
||||||
|
const uint8_t gbk[] = { 0xC4, 0xE3, 0xBA, 0xC3 };
|
||||||
|
CHECK(rd_enc_detect(gbk, sizeof gbk, NULL) == RD_ENC_GB18030, "t13 gbk still gb18030");
|
||||||
|
}
|
||||||
|
|
||||||
int main(void)
|
int main(void)
|
||||||
{
|
{
|
||||||
t1(); t2(); t3(); t4(); t5();
|
t1(); t2(); t3(); t4(); t5();
|
||||||
t6(); t7(); t8(); t9(); t10(); t11(); t12();
|
t6(); t7(); t8(); t9(); t10(); t11(); t12(); t13();
|
||||||
printf("test_reader_core: %d pass, %d fail\n", g_pass, g_fail);
|
printf("test_reader_core: %d pass, %d fail\n", g_pass, g_fail);
|
||||||
return g_fail ? 1 : 0;
|
return g_fail ? 1 : 0;
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user