cctokenizer.py

"""Tokenization classes for ChineseCharTokenizer."""

from typing import Optional, Tuple, Union
from transformers import BertTokenizer
import numpy as np
import os
import re
import shutil

# https://www.ling.upenn.edu/courses/Spring_2003/ling538/UnicodeRanges.html
# https://www.microfocus.com/documentation/idol/IDOL/Servers/IDOLServer/11.2/Guides/html/English/expert/Content/IDOLExpert/Languages/Script_Ranges.htm
# https://www.ssec.wisc.edu/~tomw/java/unicode.html
# https://character-table.netlify.app/
# https://character-table.netlify.app/french/
# https://www.compart.com/en/unicode/U+31CE 重要：看各个unicode的编码语义信息，属于哪个编码段(block/plane)

# 联合国六个官方语言：阿拉伯文、中文、英文、法文、俄文、西班牙文
# [U_LAT] 拉丁语系的文字，包括英语、法语、西班牙语、德语等
# [U_RUS] 俄语等
# [U_ARA] 表示阿拉伯语
# [U_JAP] 日语
# [U_KOR] 韩语
# [U_LAN] 某种语言的文字，藏语, 泰语等等
# [U_CHI] 未知中文

# [U_PHO] 注音文字，音标等
# [U_RAD] 部首，笔画
# [U_PUN] 标点

# [U_GRE] 希腊字母
# [U_SYM] 各种各样的符号
# [U_COM] 组合符号，包括上下标
# [U_NUM] 序号，包括VIII,(1),(一)等，也包含一些数字
# [U_MAT] 数学符号
# [U_EMO] 表情
# ¤ 货币符号

unicode_map = [
    {'token': '[U_LAT]', 'range': (0x0000, 0x007F), 'meaning': 'Basic Latin', },
    {'token': '[U_LAT]', 'range': (0x0080, 0x00FF), 'meaning': 'C1 Controls and Latin-1 Supplement', },  # 拉丁字符，有德语、法语
    {'token': '[U_LAT]', 'range': (0x0100, 0x017F), 'meaning': 'Latin Extended-A', },  # 有法语，例如œ
    {'token': '[U_LAT]', 'range': (0x0180, 0x024F), 'meaning': 'Latin Extended-B', },  # 有法语，例如Ÿ
    {'token': '[U_PHO]', 'range': (0x0250, 0x02AF), 'meaning': 'IPA Extensions', },  # 国际音标，例如ə
    {'token': '[U_PHO]', 'range': (0x02B0, 0x02FF), 'meaning': 'Spacing Modifier Letters', },  # 国际音标，例如ʳ
    {'token': '[U_PHO]', 'range': (0x0300, 0x036F), 'meaning': 'Combining Diacritical Marks', },  # 变音符号
    {'token': '[U_GRE]', 'range': (0x0370, 0x03FF), 'meaning': 'Greek/Coptic', },  # 希腊字符，例如α
    {'token': '[U_RUS]', 'range': (0x0400, 0x04FF), 'meaning': 'Cyrillic', },  # 有俄语
    {'token': '[U_RUS]', 'range': (0x0500, 0x052F), 'meaning': 'Cyrillic Supplement', },
    {'token': '[U_LAN]', 'range': (0x0530, 0x058F), 'meaning': 'Armenian', },  # 亚美尼亚语，属于印欧语系
    {'token': '[U_LAN]', 'range': (0x0590, 0x05FF), 'meaning': 'Hebrew', },  # 希伯来语，犹太族用，有以色列，属于亚非语系
    {'token': '[U_ARA]', 'range': (0x0600, 0x06FF), 'meaning': 'Arabic', },  # 阿拉伯语，联合国用，有沙特，属于亚非语系
    {'token': '[U_LAN]', 'range': (0x0700, 0x074F), 'meaning': 'Syriac', },  # 古叙利亚语
    {'token': '[U_ARA]', 'range': (0x0750, 0x077F), 'meaning': 'Undefined -> Arabic', },
    {'token': '[U_LAN]', 'range': (0x0780, 0x07BF), 'meaning': 'Thaana', },  # 它拿字母, 马尔代夫用，属于印欧语系
    {'token': '[U_ARA]', 'range': (0x07C0, 0x08FF), 'meaning': 'Undefined -> Arabic', },
    {'token': '[U_LAN]', 'range': (0x0900, 0x097F), 'meaning': 'Devanagari', },  # 梵语, 印度宗教用，属于印欧语系
    {'token': '[U_LAN]', 'range': (0x0980, 0x09FF), 'meaning': 'Bengali/Assamese', },  # 孟加拉语
    {'token': '[U_LAN]', 'range': (0x0A00, 0x0A7F), 'meaning': 'Gurmukhi', },  # 古木基文，彭加语，印度用
    {'token': '[U_LAN]', 'range': (0x0A80, 0x0AFF), 'meaning': 'Gujarati', },  # 古吉拉特语，印度用
    {'token': '[U_LAN]', 'range': (0x0B00, 0x0B7F), 'meaning': 'Oriya', },  # 印度用
    {'token': '[U_LAN]', 'range': (0x0B80, 0x0BFF), 'meaning': 'Tamil', },  # 印度用
    {'token': '[U_LAN]', 'range': (0x0C00, 0x0C7F), 'meaning': 'Telugu', },  # 印度用
    {'token': '[U_LAN]', 'range': (0x0C80, 0x0CFF), 'meaning': 'Kannada', },  # 印度用
    {'token': '[U_LAN]', 'range': (0x0D00, 0x0DFF), 'meaning': 'Malayalam', },  # 印度用
    # {'token': '[U_LAN]', 'range': (0x0D80, 0x0DFF), 'meaning': 'Sinhala', },  # 僧伽罗语，斯里兰卡用，近印度
    {'token': '[U_LAN]', 'range': (0x0E00, 0x0E7F), 'meaning': 'Thai', },  # 泰语
    {'token': '[U_LAN]', 'range': (0x0E80, 0x0EFF), 'meaning': 'Lao', },  # 老挝语
    {'token': '[U_LAN]', 'range': (0x0F00, 0x0FFF), 'meaning': 'Tibetan', },  # 藏语 NOTE: 藏语是否需要单独列出？
    {'token': '[U_LAN]', 'range': (0x1000, 0x109F), 'meaning': 'Myanmar', },  # 缅甸语
    {'token': '[U_LAN]', 'range': (0x10A0, 0x10FF), 'meaning': 'Georgian', },  # 格鲁吉亚语，伊朗也用
    {'token': '[U_KOR]', 'range': (0x1100, 0x11FF), 'meaning': 'Hangul Jamo', },  # 谚文，有古韩语
    {'token': '[U_LAN]', 'range': (0x1200, 0x137F), 'meaning': 'Ethiopic', },  # 埃塞俄比亚语，非洲，仅次于阿拉伯语
    {'token': '[U_LAN]', 'range': (0x1380, 0x139F), 'meaning': 'Undefined -> Ethiopic', },
    {'token': '[U_LAN]', 'range': (0x13A0, 0x13FF), 'meaning': 'Cherokee', },  # 切罗基语，北美原住民
    {'token': '[U_LAN]', 'range': (0x1400, 0x167F), 'meaning': 'Unified Canadian Aboriginal Syllabics', },  # 加拿大澳大利亚音标
    {'token': '[U_LAN]', 'range': (0x1680, 0x169F), 'meaning': 'Ogham', },  # 欧甘字母，古爱尔兰用
    {'token': '[U_LAN]', 'range': (0x16A0, 0x16FF), 'meaning': 'Runic', },  # 卢恩字母，古北欧用
    {'token': '[U_LAN]', 'range': (0x1700, 0x171F), 'meaning': 'Tagalog', },  # 他加禄语，菲律宾及东南亚用
    {'token': '[U_LAN]', 'range': (0x1720, 0x173F), 'meaning': 'Hanunoo', },  # 菲律宾用
    {'token': '[U_LAN]', 'range': (0x1740, 0x175F), 'meaning': 'Buhid', },  # 菲律宾用
    {'token': '[U_LAN]', 'range': (0x1760, 0x177F), 'meaning': 'Tagbanwa', },  # 菲律宾用
    {'token': '[U_LAN]', 'range': (0x1780, 0x17FF), 'meaning': 'Khmer', },  # 高棉语，柬埔寨用
    {'token': '[U_LAN]', 'range': (0x1800, 0x18AF), 'meaning': 'Mongolian', },  # 蒙古语
    {'token': '[U_LAN]', 'range': (0x18B0, 0x18FF), 'meaning': 'Undefined -> Unified Canadian Aboriginal Syllabics', },
    {'token': '[U_LAN]', 'range': (0x1900, 0x194F), 'meaning': 'Limbu', },  # 林布语，尼泊尔用
    {'token': '[U_LAN]', 'range': (0x1950, 0x197F), 'meaning': 'Tai Le', },  # 傣语，云南用
    {'token': '[U_LAN]', 'range': (0x1980, 0x19DF), 'meaning': 'Undefined -> New Tai Lue', },  # 新傣语
    {'token': '[U_LAN]', 'range': (0x19E0, 0x19FF), 'meaning': 'Khmer Symbols', },  # 高棉标点，柬埔寨用
    {'token': '[U_LAN]', 'range': (0x1A00, 0x1CFF), 'meaning': 'Undefined -> Ol Chiki', },  # 桑塔利语，印度用
    {'token': '[U_PHO]', 'range': (0x1D00, 0x1D7F), 'meaning': 'Phonetic Extensions', },  # 音标，例如法语的ᵈ
    {'token': '[U_PHO]', 'range': (0x1D80, 0x1DFF), 'meaning': 'Undefined -> Phonetic Extensions Supplement', },
    {'token': '[U_LAT]', 'range': (0x1E00, 0x1EFF), 'meaning': 'Latin Extended Additional', },  # 拉丁带修饰符号，例如ṡ
    {'token': '[U_GRE]', 'range': (0x1F00, 0x1FFF), 'meaning': 'Greek Extended', },  # 希腊字符带修饰，例如Ᾱ
    {'token': '[U_SYM]', 'range': (0x2000, 0x206F), 'meaning': 'General Punctuation', },  # 各种符号，例如千分之‰
    {'token': '[U_COM]', 'range': (0x2070, 0x209F), 'meaning': 'Superscripts and Subscripts', },  # 上下标，例如₂
    {'token': '¤', 'range': (0x20A0, 0x20CF), 'meaning': 'Currency Symbols', },  # 货币符号，例如欧元€
    {'token': '[U_COM]', 'range': (0x20D0, 0x20FF), 'meaning': 'Combining Diacritical Marks for Symbols', },  # 奇怪的可组合符号
    {'token': '[U_SYM]', 'range': (0x2100, 0x214F), 'meaning': 'Letterlike Symbols', },  # 符号，例如普朗克常数ℎ，摄氏度℃
    {'token': '[U_NUM]', 'range': (0x2150, 0x218F), 'meaning': 'Number Forms', },  # 特殊形式数字，例如三分之一⅓，八Ⅷ
    {'token': '[U_SYM]', 'range': (0x2190, 0x21FF), 'meaning': 'Arrows', },  # 箭头，例如→
    {'token': '[U_MAT]', 'range': (0x2200, 0x22FF), 'meaning': 'Mathematical Operators', },  # 数学符号，例如减号−，不属于∉
    {'token': '[U_SYM]', 'range': (0x2300, 0x23FF), 'meaning': 'Miscellaneous Technical', },  # 杂乱的符号，例如放大倍数⌀，右上角⌝
    {'token': '[U_SYM]', 'range': (0x2400, 0x243F), 'meaning': 'Control Pictures', },  # 描述符：表达控制的符号，例如空␀，退出␛
    {'token': '[U_SYM]', 'range': (0x2440, 0x245F), 'meaning': 'Optical Character Recognition', },  # OCR符号，例如⑀，⑂
    {'token': '[U_NUM]', 'range': (0x2460, 0x24FF), 'meaning': 'Enclosed Alphanumerics', },  # 带框序号，例如①，⒆
    {'token': '[U_SYM]', 'range': (0x2500, 0x257F), 'meaning': 'Box Drawing', },  # 画盒子符，例如┋，┓
    {'token': '[U_SYM]', 'range': (0x2580, 0x259F), 'meaning': 'Block Elements', },  # 画块符，例如▉，▀
    {'token': '[U_SYM]', 'range': (0x25A0, 0x25FF), 'meaning': 'Geometric Shapes', },  # 几何形状，例如△
    {'token': '[U_SYM]', 'range': (0x2600, 0x26FF), 'meaning': 'Miscellaneous Symbols', },  # 杂乱的符号，例如多云☁，女♀
    {'token': '[U_SYM]', 'range': (0x2700, 0x27BF), 'meaning': 'Dingbats', },  # 杂乱的符号✈➓
    {'token': '[U_MAT]', 'range': (0x27C0, 0x27EF), 'meaning': 'Miscellaneous Mathematical Symbols-A', },  # 杂乱的数学符号⟂⟘
    {'token': '[U_SYM]', 'range': (0x27F0, 0x27FF), 'meaning': 'Supplemental Arrows-A', },  # 补充箭头⟹
    {'token': '[U_LAN]', 'range': (0x2800, 0x28FF), 'meaning': 'Braille Patterns', },  # 盲文，点字文，⠝⠟
    {'token': '[U_SYM]', 'range': (0x2900, 0x297F), 'meaning': 'Supplemental Arrows-B', },  # ⥬
    {'token': '[U_MAT]', 'range': (0x2980, 0x29FF), 'meaning': 'Miscellaneous Mathematical Symbols-B', },  # ⭬
    {'token': '[U_MAT]', 'range': (0x2A00, 0x2AFF), 'meaning': 'Supplemental Mathematical Operators', },  # ⪆
    {'token': '[U_SYM]', 'range': (0x2B00, 0x2BFF), 'meaning': 'Miscellaneous Symbols and Arrows'},  # ⭬
    {'token': '[U_LAN]', 'range': (0x2C00, 0x2E7F), 'meaning': 'Undefined -> Coptic', },  # 科普特语，埃塞俄比亚语
    {'token': '[U_RAD]', 'range': (0x2E80, 0x2EFF), 'meaning': 'CJK Radicals Supplement', },  # CJK中日韩统一表意文字，部首，例如⺘
    {'token': '[U_RAD]', 'range': (0x2F00, 0x2FDF), 'meaning': 'Kangxi Radicals', },  # 康熙字典部首
    {'token': '[U_SYM]', 'range': (0x2FE0, 0x2FEF), 'meaning': 'Undefined -> Symbol', },
    {'token': '[U_SYM]', 'range': (0x2FF0, 0x2FFF), 'meaning': 'Ideographic Description Characters', },  # 描述符：表意文字结构，例如上下结构，左右结构，半包围结构
    {'token': '[U_PUN]', 'range': (0x3000, 0x303F), 'meaning': 'CJK Symbols and Punctuation', },  # 中文标点 。
    {'token': '[U_JAP]', 'range': (0x3040, 0x309F), 'meaning': 'Hiragana', },  # 日语平假名
    {'token': '[U_JAP]', 'range': (0x30A0, 0x30FF), 'meaning': 'Katakana', },  # 日语片假名
    {'token': '[U_PHO]', 'range': (0x3100, 0x312F), 'meaning': 'Bopomofo', },  # 汉语拼音字，例如 ㄠㄎ
    {'token': '[U_KOR]', 'range': (0x3130, 0x318F), 'meaning': 'Hangul Compatibility Jamo', },  # 韩文
    {'token': '[U_JAP]', 'range': (0x3190, 0x319F), 'meaning': 'Kanbun (Kunten)', },  # 汉文，日本用
    {'token': '[U_PHO]', 'range': (0x31A0, 0x31BF), 'meaning': 'Bopomofo Extended', },  # 汉语拼音字，例如ㆠ
    {'token': '[U_RAD]', 'range': (0x31C0, 0x31EF), 'meaning': 'Undefined -> CJK Strokes', },  # 汉字笔画
    {'token': '[U_JAP]', 'range': (0x31F0, 0x31FF), 'meaning': 'Katakana Phonetic Extensions', },  # 片假名音标
    {'token': '[U_NUM]', 'range': (0x3200, 0x32FF), 'meaning': 'Enclosed CJK Letters and Months', },  # 汉字序号，例如㈠
    {'token': '[U_SYM]', 'range': (0x3300, 0x33FF), 'meaning': 'CJK Compatibility', },  # 单字符表达单位 平方厘米㎠，毫克㎎，23点㍯
    {'token': '[U_CHI]', 'range': (0x3400, 0x4DBF), 'meaning': 'CJK Unified Ideographs Extension A', },  # 中文罕见字
    {'token': '[U_SYM]', 'range': (0x4DC0, 0x4DFF), 'meaning': 'Yijing Hexagram Symbols', },  # 易经六十四卦，䷁	䷖
    {'token': '[U_CHI]', 'range': (0x4E00, 0x9FAF), 'meaning': 'CJK Unified Ideographs', },  # 中文
    {'token': '[U_CHI]', 'range': (0x9FB0, 0x9FFF), 'meaning': 'Undefined -> CJK Unified Ideographs', },  # 中文
    {'token': '[U_LAN]', 'range': (0xA000, 0xA48F), 'meaning': 'Yi Syllables', },  # 彝文字符，凉山彝族用
    {'token': '[U_LAN]', 'range': (0xA490, 0xA4CF), 'meaning': 'Yi Radicals', },  # 彝文部首
    {'token': '[U_LAN]', 'range': (0xA4D0, 0xABFF), 'meaning': 'Undefined -> Cherokee'},  # 彻罗基族, 北美印第安人
    {'token': '[U_KOR]', 'range': (0xAC00, 0xD7AF), 'meaning': 'Hangul Syllables', },  # 韩语音节
    {'token': '[U_KOR]', 'range': (0xD7B0, 0xD7FF), 'meaning': 'Undefined -> Hangul Jamo Extended-B', },
    {'range': (0xD800, 0xDBFF), 'meaning': 'High Surrogate Area', 'token': '[UNK]'},
    {'range': (0xDC00, 0xDFFF), 'meaning': 'Low Surrogate Area', 'token': '[UNK]'},
    {'range': (0xE000, 0xF8FF), 'meaning': 'Private Use Area', 'token': '[UNK]'},  # NOTE 统计字频时，有很多落无法显示的字符落到了这一区域
    {'token': '[U_CHI]', 'range': (0xF900, 0xFAFF), 'meaning': 'CJK Compatibility Ideographs', },  # 中文
    {'token': '[U_LAT]', 'range': (0xFB00, 0xFB4F), 'meaning': 'Alphabetic Presentation Forms', },  # 拉丁、希伯来语字母排版符号
    {'token': '[U_ARA]', 'range': (0xFB50, 0xFDFF), 'meaning': 'Arabic Presentation Forms-A', },  # 阿拉伯语排版符号
    {'token': '[U_SYM]', 'range': (0xFE00, 0xFE0F), 'meaning': 'Variation Selectors', },  # 描述符: 表示选择第几个
    {'token': '[U_PUN]', 'range': (0xFE10, 0xFE1F), 'meaning': 'Undefined -> Vertical Forms', },  # 竖排标点
    {'token': '[U_COM]', 'range': (0xFE20, 0xFE2F), 'meaning': 'Combining Half Marks', },  # 可组合符号 例如$︡a
    {'token': '[U_PUN]', 'range': (0xFE30, 0xFE4F), 'meaning': 'CJK Compatibility Forms', },  # 中文排版符号，例如竖排的符号︻︼
    {'token': '[U_PUN]', 'range': (0xFE50, 0xFE6F), 'meaning': 'Small Form Variants', },  # 小符号，小逗号﹐，小问号﹖
    {'token': '[U_ARA]', 'range': (0xFE70, 0xFEFF), 'meaning': 'Arabic Presentation Forms-B', },  # 阿拉伯语排版符号
    {'token': '[U_LAT]', 'range': (0xFF00, 0xFFEF), 'meaning': 'Halfwidth and Fullwidth Forms', },  # 中文全角字符０ｃ, TODO: 这里视为拉丁文，应该映射回拉丁的半角字符
    {'token': '[U_SYM]', 'range': (0xFFF0, 0xFFFF), 'meaning': 'Specials', },  # 描述符: 可表示替换等操作
    {'token': '[U_LAN]', 'range': (0x10000, 0x1007F), 'meaning': 'Linear B Syllabary', },  # 线形文字，像象形文字
    {'token': '[U_LAN]', 'range': (0x10080, 0x100FF), 'meaning': 'Linear B Ideograms', },
    {'token': '[U_LAN]', 'range': (0x10100, 0x1013F), 'meaning': 'Aegean Numbers', },  # 爱琴海象形数字
    {'token': '[U_LAN]', 'range': (0x10140, 0x102FF), 'meaning': 'Undefined -> Carian', },
    {'token': '[U_LAN]', 'range': (0x10300, 0x1032F), 'meaning': 'Old Italic', },  # 古意大利
    {'token': '[U_LAN]', 'range': (0x10330, 0x1034F), 'meaning': 'Gothic', },  # 哥特语
    {'range': (0x10350, 0x1037F), 'meaning': 'Undefined', 'token': '[UNK]'},
    {'token': '[U_LAN]', 'range': (0x10380, 0x1039F), 'meaning': 'Ugaritic', },  # 古文字在叙利亚发现
    {'token': '[U_LAN]', 'range': (0x103A0, 0x103FF), 'meaning': 'Undefined -> Old Persian', },  # 古波斯语
    {'token': '[U_LAN]', 'range': (0x10400, 0x1044F), 'meaning': 'Deseret', },  # 北美原住民用
    {'token': '[U_PHO]', 'range': (0x10450, 0x1047F), 'meaning': 'Shavian', },  # 萧伯纳的音标
    {'token': '[U_LAN]', 'range': (0x10480, 0x104AF), 'meaning': 'Osmanya', },  # 索马里曾经用
    {'token': '[U_LAN]', 'range': (0x104B0, 0x107FF), 'meaning': 'Undefined -> Osage', },  # 欧塞奇语
    {'token': '[U_LAN]', 'range': (0x10800, 0x1083F), 'meaning': 'Cypriot Syllabary', },  # 塞浦路斯，地中海岛国
    {'token': '[U_LAN]', 'range': (0x10840, 0x1CFFF), 'meaning': 'Undefined -> Cuneiform, Chakma, Kharoshthi...', },  # 许多种语言，包括楔形文字, 和一些符号𓆝 1319d 𓆟 1319f
    {'token': '[U_LAN]', 'range': (0x1D000, 0x1D0FF), 'meaning': 'Byzantine Musical Symbols', },  # 拜占庭人音乐符号
    {'token': '[U_SYM]', 'range': (0x1D100, 0x1D1FF), 'meaning': 'Musical Symbols', },  # 音乐符号
    {'range': (0x1D200, 0x1D2FF), 'meaning': 'Undefined', 'token': '[UNK]'},
    {'token': '[U_SYM]', 'range': (0x1D300, 0x1D35F), 'meaning': 'Tai Xuan Jing Symbols', },  # 太玄经符号
    {'range': (0x1D360, 0x1D3FF), 'meaning': 'Undefined', 'token': '[UNK]'},
    {'token': '[U_MAT]', 'range': (0x1D400, 0x1D7FF), 'meaning': 'Mathematical Alphanumeric Symbols', },  # 数学中用的字母，例如斜体的字母
    {'token': '[U_LAN]', 'range': (0x1D800, 0x1F003), 'meaning': 'Undefined -> Adlam', },  # 阿德拉姆字母，用于西非
    {'token': '[U_EMO]', 'range': (0x1F004, 0x1FAF8), 'meaning': 'Undefined -> Emoji', },  # emoji
    {'token': '[U_SYM]', 'range': (0x1FAF9, 0x1FFFF), 'meaning': 'Undefined -> Symbols for Legacy Computing', },
    {'token': '[U_CHI]', 'range': (0x20000, 0x2A6DF), 'meaning': 'CJK Unified Ideographs Extension B', },  # 中文罕见字
    {'token': '[U_CHI]', 'range': (0x2A6E0, 0x2F7FF), 'meaning': 'Undefined -> CJK Unified Ideographs Extension F...', },  # 中文罕见字
    {'token': '[U_CHI]', 'range': (0x2F800, 0x2FA1F), 'meaning': 'CJK Compatibility Ideographs Supplement', },  # 中文罕见字
    {'range': (0x2FA20, 0x2FAAF), 'meaning': 'Undefined', 'token': '[UNK]'},
    {'range': (0x2FAB0, 0x2FFFF), 'meaning': 'Unused', 'token': '[UNK]'},
    {'token': '[U_CHI]', 'range': (0x30000, 0x3134F), 'meaning': 'Unused -> CJK Unified Ideographs Extension G (unassigned)', },  # 未使用的中文编码区间
    {'range': (0x31350, 0xDFFFF), 'meaning': 'Unused', 'token': '[UNK]'},
    {'token': '[U_SYM]', 'range': (0xE0000, 0xE007F), 'meaning': 'Tags', },  # 描述符: 表示标签
    {'range': (0xE0080, 0xE00FF), 'meaning': 'Unused', 'token': '[UNK]'},
    {'token': '[U_SYM]', 'range': (0xE0100, 0xE01EF), 'meaning': 'Variation Selectors Supplement', },  # 描述符: 表示选择
    {'range': (0xE01F0, 0xEFFFF), 'meaning': 'Unused', 'token': '[UNK]'},
    {'range': (0xF0000, 0xFFFFD), 'meaning': 'Supplementary Private Use Area-A', 'token': '[UNK]'},
    {'range': (0xFFFFE, 0xFFFFF), 'meaning': 'Unused', 'token': '[UNK]'},
    {'range': (0x100000, 0x10FFFD), 'meaning': 'Supplementary Private Use Area-B', 'token': '[UNK]'},
]


def get_unicode_ranges():
    # 检查区间是否连续
    left_bounds = [m['range'][0] for m in unicode_map]
    right_bounds = [m['range'][1] for m in unicode_map]
    for right, left in zip(right_bounds[:-1], left_bounds[1:]):
        assert right+1 == left
    return np.array(right_bounds)


def _is_chinese_char(cp):
    # copied from transformers.models.bert.tokenization_bert.BasicTokenizer._is_chinese_char
    """Checks whether CP is the codepoint of a CJK character."""
    # This defines a "chinese character" as anything in the CJK Unicode block:
    #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
    #
    # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
    # despite its name. The modern Korean Hangul alphabet is a different block,
    # as is Japanese Hiragana and Katakana. Those alphabets are used to write
    # space-separated words, so they are not treated specially and handled
    # like the all of the other languages.
    if (
        (cp >= 0x4E00 and cp <= 0x9FFF)
        or (cp >= 0x3400 and cp <= 0x4DBF)  #
        or (cp >= 0x20000 and cp <= 0x2A6DF)  #
        or (cp >= 0x2A700 and cp <= 0x2B73F)  #
        or (cp >= 0x2B740 and cp <= 0x2B81F)  #
        or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
        or (cp >= 0xF900 and cp <= 0xFAFF)
        or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
    ):  #
        return True

    return False


def show_unicode(start=0x1F004, end=0x1FAF8):
    # emoji 文件 https://www.unicode.org/Public/emoji/latest/emoji-sequences.txt
    # https://github.com/hidehalo/emoji/issues/3
    # https://apps.timwhitlock.info/emoji/tables/unicode
    for i in range(start, end):
        print(chr(i), end=' ')
    print()


def load_json(file):
    import json
    with open(file, 'r', encoding='utf-8') as f:
        obj = json.load(f)
    return obj


class ChineseCharTokenizer(BertTokenizer):
    vocab_files_names = {"vocab_file": "vocab.txt", 'mapping_file': "replace.json"}

    def __init__(self, vocab_file, *args, **kwargs):
        super(ChineseCharTokenizer, self).__init__(vocab_file, *args, **kwargs)
        self.unicoder_ranges = get_unicode_ranges()
        self.enclosed_tokens = {token for token in self.vocab if token[0] == '[' and token[-1] == ']' and 'unused' not in token}
        self.enclosed_tokens_by_len = [
            [token for token in self.enclosed_tokens if len(token) == 5],
            [token for token in self.enclosed_tokens if len(token) == 6],
            [token for token in self.enclosed_tokens if len(token) == 7]
        ]
        self.dir = os.path.join(os.path.dirname(vocab_file))
        self.replace_map = load_json(os.path.join(self.dir, 'replace.json'))

    # # [EOS]相当于逗号、换行，不用看作special token
    def convert_token_to_representative(self, token: str) -> str:
        token = self.replace_map.get(token, token)  # 异体字转换，繁简转换，全半角转换，大小写转换等
        if token in self.vocab:
            return token
        else:
            assert len(token) == 1, token
            if re.match(r'\s', token):  # 匹配\u2003, \t等
                return ' '
            v = ord(token)
            if _is_chinese_char(v):
                return '[U_CHI]'
            elif v <= 0x10FFFD:
                i = np.searchsorted(self.unicoder_ranges, v)  # 找到插入位置 ranges[i-1] < v <= ranges[i]
                return unicode_map[i]['token']
            else:
                return '[UNK]'

    # bert的tokenize会加上CLS?
    def _tokenize(self, text):
        # 如果没有人为加的特殊赋好，可以不用这个tokenize，list(text)就是tokenize的结果
        split_tokens = []
        i = 0
        while i < len(text):
            if text[i:i+5] in self.enclosed_tokens_by_len[0]:
                split_tokens.append(text[i:i+5])
                i += 5
            elif text[i:i+6] == '[MASK]':
                split_tokens.append('[MASK]')
                i += 6
            elif text[i:i+7] in self.enclosed_tokens_by_len[2]:
                split_tokens.append(text[i:i+7])
                i += 7
            else:
                split_tokens.append(self.convert_token_to_representative(text[i]))
                i += 1
        return split_tokens

    def _convert_token_to_id(self, token):
        return self.vocab.get(self.convert_token_to_representative(token), self.vocab.get(self.unk_token))  # BUG: convert_token_to_representative 不是 id!

    def convert_tokens_to_string(self, tokens):
        return ''.join(tokens)

    def save_pretrained(self, save_directory: Union[str, os.PathLike], legacy_format: Optional[bool] = None, filename_prefix: Optional[str] = None, push_to_hub: bool = False, **kwargs) -> Tuple[str]:
        ret = super().save_pretrained(save_directory, legacy_format, filename_prefix, push_to_hub, **kwargs)
        shutil.copyfile(os.path.join(self.dir, 'replace.json'), f'{save_directory}/replace.json')
        shutil.copyfile(os.path.join(self.dir, 'cctokenizer.py'), f'{save_directory}/cctokenizer.py')
        return ret