#!/usr/bin/env python3 """gen_gb18030_table.py — 生成 GB18030 双字节 → Unicode BMP 码位表(源码生成物,入仓)。 输出:core/gb18030_table.h 的 `static const uint16_t gb_2byte[126][190]`。 为什么这样生成(对齐 devdocs/plan/READER_ENGINE_NOTES.md §3.4): - 用 Python 内置 `gb18030` codec 枚举全部 2 字节码位(GB18030 向后兼容 GBK, 故本表覆盖 GBK 全部合法双字节对 + 部分扩展)。 - 表大小 = 126 引导字节(0x81..0xFE)× 190 尾字节 = 23,940 项 × 2B = 47,880 B。 - 0 值 = 未映射;解码器查表得码位,命中 0 → U+FFFD。 - 尾字节列布局 = [0x40..0x7E] 紧接 [0x80..0xFE](跳过 0x7F);索引 idx(b): b in 0x40..0x7F -> b-0x40 ;b in 0x80..0xFE -> b - 0x41 (即 b - 0x40 - (b>=0x80))。 用法:`python3 tools/gen_gb18030_table.py > core/gb18030_table.h` (可用 env GB18030_TABLE_H 覆写输出路径。) """ import sys LEADS = list(range(0x81, 0xFF)) # 0x81..0xFE → 126 个 TRAILS = list(range(0x40, 0x7F)) + list(range(0x80, 0xFF)) # 0x40..0x7E,0x80..0xFE → 190 个 INDEX_OF = {b: i for i, b in enumerate(TRAILS)} # 尾字节 → 列号 def build(): g = [0] * (len(LEADS) * len(TRAILS)) for k, lead in enumerate(LEADS): for j, tr in enumerate(TRAILS): try: s = bytes([lead, tr]).decode('gb18030') if len(s) == 1 and ord(s) <= 0xFFFF: g[k * len(TRAILS) + j] = ord(s) except UnicodeDecodeError: pass return g def emit(g): rows, cols = len(LEADS), len(TRAILS) out = sys.stdout out.write("/* gb18030_table.h — GB18030 双字节 → Unicode BMP 码位表(生成物,勿手改;\n") out.write(" * 由 tools/gen_gb18030_table.py 用 python3 `gb18030` codec 生成)。\n") out.write(" * 布局:gb_2byte[lead-0x81][trail_col];列号 idx(b)= b-0x40-(b>=0x80);\n") out.write(" * 值 0 = 未映射(解码器输出 U+FFFD)。46 bytes 行格式可读。*/\n") out.write("#ifndef RD_GB18030_TABLE_H\n#define RD_GB18030_TABLE_H\n\n") out.write("#include \n\n") out.write("#define RD_GB_LEADS %d\n" % rows) out.write("#define RD_GB_TRAILS %d\n" % cols) out.write("#define RD_GB_LEAD0 0x81\n") out.write("#define RD_GB_TRAIL0 0x40\n\n") out.write("/* gb_2byte[126][190]:rdgb_lookup(lead, trail) 返回 BMP 码位或 0。 */\n") out.write("static const uint16_t gb_2byte[%d][%d] = {\n" % (rows, cols)) for k in range(rows): out.write(" {") for j in range(cols): v = g[k * cols + j] if j % 8 == 0: out.write("\n " if j else "") out.write("0x%04X,%s" % (v, "" if j == cols - 1 else "")) out.write("\n }%s\n" % ("," if k != rows - 1 else "")) out.write("};\n\n#endif /* RD_GB18030_TABLE_H */\n") if __name__ == '__main__': emit(build())