|
|
|
from codecs import BOM_UTF8, BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE |
|
from encodings.aliases import aliases |
|
from re import IGNORECASE, compile as re_compile |
|
from typing import Dict, List, Set, Union |
|
|
|
|
|
ENCODING_MARKS: Dict[str, Union[bytes, List[bytes]]] = { |
|
"utf_8": BOM_UTF8, |
|
"utf_7": [ |
|
b"\x2b\x2f\x76\x38", |
|
b"\x2b\x2f\x76\x39", |
|
b"\x2b\x2f\x76\x2b", |
|
b"\x2b\x2f\x76\x2f", |
|
b"\x2b\x2f\x76\x38\x2d", |
|
], |
|
"gb18030": b"\x84\x31\x95\x33", |
|
"utf_32": [BOM_UTF32_BE, BOM_UTF32_LE], |
|
"utf_16": [BOM_UTF16_BE, BOM_UTF16_LE], |
|
} |
|
|
|
TOO_SMALL_SEQUENCE: int = 32 |
|
TOO_BIG_SEQUENCE: int = int(10e6) |
|
|
|
UTF8_MAXIMAL_ALLOCATION: int = 1_112_064 |
|
|
|
|
|
UNICODE_RANGES_COMBINED: Dict[str, range] = { |
|
"Control character": range(32), |
|
"Basic Latin": range(32, 128), |
|
"Latin-1 Supplement": range(128, 256), |
|
"Latin Extended-A": range(256, 384), |
|
"Latin Extended-B": range(384, 592), |
|
"IPA Extensions": range(592, 688), |
|
"Spacing Modifier Letters": range(688, 768), |
|
"Combining Diacritical Marks": range(768, 880), |
|
"Greek and Coptic": range(880, 1024), |
|
"Cyrillic": range(1024, 1280), |
|
"Cyrillic Supplement": range(1280, 1328), |
|
"Armenian": range(1328, 1424), |
|
"Hebrew": range(1424, 1536), |
|
"Arabic": range(1536, 1792), |
|
"Syriac": range(1792, 1872), |
|
"Arabic Supplement": range(1872, 1920), |
|
"Thaana": range(1920, 1984), |
|
"NKo": range(1984, 2048), |
|
"Samaritan": range(2048, 2112), |
|
"Mandaic": range(2112, 2144), |
|
"Syriac Supplement": range(2144, 2160), |
|
"Arabic Extended-B": range(2160, 2208), |
|
"Arabic Extended-A": range(2208, 2304), |
|
"Devanagari": range(2304, 2432), |
|
"Bengali": range(2432, 2560), |
|
"Gurmukhi": range(2560, 2688), |
|
"Gujarati": range(2688, 2816), |
|
"Oriya": range(2816, 2944), |
|
"Tamil": range(2944, 3072), |
|
"Telugu": range(3072, 3200), |
|
"Kannada": range(3200, 3328), |
|
"Malayalam": range(3328, 3456), |
|
"Sinhala": range(3456, 3584), |
|
"Thai": range(3584, 3712), |
|
"Lao": range(3712, 3840), |
|
"Tibetan": range(3840, 4096), |
|
"Myanmar": range(4096, 4256), |
|
"Georgian": range(4256, 4352), |
|
"Hangul Jamo": range(4352, 4608), |
|
"Ethiopic": range(4608, 4992), |
|
"Ethiopic Supplement": range(4992, 5024), |
|
"Cherokee": range(5024, 5120), |
|
"Unified Canadian Aboriginal Syllabics": range(5120, 5760), |
|
"Ogham": range(5760, 5792), |
|
"Runic": range(5792, 5888), |
|
"Tagalog": range(5888, 5920), |
|
"Hanunoo": range(5920, 5952), |
|
"Buhid": range(5952, 5984), |
|
"Tagbanwa": range(5984, 6016), |
|
"Khmer": range(6016, 6144), |
|
"Mongolian": range(6144, 6320), |
|
"Unified Canadian Aboriginal Syllabics Extended": range(6320, 6400), |
|
"Limbu": range(6400, 6480), |
|
"Tai Le": range(6480, 6528), |
|
"New Tai Lue": range(6528, 6624), |
|
"Khmer Symbols": range(6624, 6656), |
|
"Buginese": range(6656, 6688), |
|
"Tai Tham": range(6688, 6832), |
|
"Combining Diacritical Marks Extended": range(6832, 6912), |
|
"Balinese": range(6912, 7040), |
|
"Sundanese": range(7040, 7104), |
|
"Batak": range(7104, 7168), |
|
"Lepcha": range(7168, 7248), |
|
"Ol Chiki": range(7248, 7296), |
|
"Cyrillic Extended-C": range(7296, 7312), |
|
"Georgian Extended": range(7312, 7360), |
|
"Sundanese Supplement": range(7360, 7376), |
|
"Vedic Extensions": range(7376, 7424), |
|
"Phonetic Extensions": range(7424, 7552), |
|
"Phonetic Extensions Supplement": range(7552, 7616), |
|
"Combining Diacritical Marks Supplement": range(7616, 7680), |
|
"Latin Extended Additional": range(7680, 7936), |
|
"Greek Extended": range(7936, 8192), |
|
"General Punctuation": range(8192, 8304), |
|
"Superscripts and Subscripts": range(8304, 8352), |
|
"Currency Symbols": range(8352, 8400), |
|
"Combining Diacritical Marks for Symbols": range(8400, 8448), |
|
"Letterlike Symbols": range(8448, 8528), |
|
"Number Forms": range(8528, 8592), |
|
"Arrows": range(8592, 8704), |
|
"Mathematical Operators": range(8704, 8960), |
|
"Miscellaneous Technical": range(8960, 9216), |
|
"Control Pictures": range(9216, 9280), |
|
"Optical Character Recognition": range(9280, 9312), |
|
"Enclosed Alphanumerics": range(9312, 9472), |
|
"Box Drawing": range(9472, 9600), |
|
"Block Elements": range(9600, 9632), |
|
"Geometric Shapes": range(9632, 9728), |
|
"Miscellaneous Symbols": range(9728, 9984), |
|
"Dingbats": range(9984, 10176), |
|
"Miscellaneous Mathematical Symbols-A": range(10176, 10224), |
|
"Supplemental Arrows-A": range(10224, 10240), |
|
"Braille Patterns": range(10240, 10496), |
|
"Supplemental Arrows-B": range(10496, 10624), |
|
"Miscellaneous Mathematical Symbols-B": range(10624, 10752), |
|
"Supplemental Mathematical Operators": range(10752, 11008), |
|
"Miscellaneous Symbols and Arrows": range(11008, 11264), |
|
"Glagolitic": range(11264, 11360), |
|
"Latin Extended-C": range(11360, 11392), |
|
"Coptic": range(11392, 11520), |
|
"Georgian Supplement": range(11520, 11568), |
|
"Tifinagh": range(11568, 11648), |
|
"Ethiopic Extended": range(11648, 11744), |
|
"Cyrillic Extended-A": range(11744, 11776), |
|
"Supplemental Punctuation": range(11776, 11904), |
|
"CJK Radicals Supplement": range(11904, 12032), |
|
"Kangxi Radicals": range(12032, 12256), |
|
"Ideographic Description Characters": range(12272, 12288), |
|
"CJK Symbols and Punctuation": range(12288, 12352), |
|
"Hiragana": range(12352, 12448), |
|
"Katakana": range(12448, 12544), |
|
"Bopomofo": range(12544, 12592), |
|
"Hangul Compatibility Jamo": range(12592, 12688), |
|
"Kanbun": range(12688, 12704), |
|
"Bopomofo Extended": range(12704, 12736), |
|
"CJK Strokes": range(12736, 12784), |
|
"Katakana Phonetic Extensions": range(12784, 12800), |
|
"Enclosed CJK Letters and Months": range(12800, 13056), |
|
"CJK Compatibility": range(13056, 13312), |
|
"CJK Unified Ideographs Extension A": range(13312, 19904), |
|
"Yijing Hexagram Symbols": range(19904, 19968), |
|
"CJK Unified Ideographs": range(19968, 40960), |
|
"Yi Syllables": range(40960, 42128), |
|
"Yi Radicals": range(42128, 42192), |
|
"Lisu": range(42192, 42240), |
|
"Vai": range(42240, 42560), |
|
"Cyrillic Extended-B": range(42560, 42656), |
|
"Bamum": range(42656, 42752), |
|
"Modifier Tone Letters": range(42752, 42784), |
|
"Latin Extended-D": range(42784, 43008), |
|
"Syloti Nagri": range(43008, 43056), |
|
"Common Indic Number Forms": range(43056, 43072), |
|
"Phags-pa": range(43072, 43136), |
|
"Saurashtra": range(43136, 43232), |
|
"Devanagari Extended": range(43232, 43264), |
|
"Kayah Li": range(43264, 43312), |
|
"Rejang": range(43312, 43360), |
|
"Hangul Jamo Extended-A": range(43360, 43392), |
|
"Javanese": range(43392, 43488), |
|
"Myanmar Extended-B": range(43488, 43520), |
|
"Cham": range(43520, 43616), |
|
"Myanmar Extended-A": range(43616, 43648), |
|
"Tai Viet": range(43648, 43744), |
|
"Meetei Mayek Extensions": range(43744, 43776), |
|
"Ethiopic Extended-A": range(43776, 43824), |
|
"Latin Extended-E": range(43824, 43888), |
|
"Cherokee Supplement": range(43888, 43968), |
|
"Meetei Mayek": range(43968, 44032), |
|
"Hangul Syllables": range(44032, 55216), |
|
"Hangul Jamo Extended-B": range(55216, 55296), |
|
"High Surrogates": range(55296, 56192), |
|
"High Private Use Surrogates": range(56192, 56320), |
|
"Low Surrogates": range(56320, 57344), |
|
"Private Use Area": range(57344, 63744), |
|
"CJK Compatibility Ideographs": range(63744, 64256), |
|
"Alphabetic Presentation Forms": range(64256, 64336), |
|
"Arabic Presentation Forms-A": range(64336, 65024), |
|
"Variation Selectors": range(65024, 65040), |
|
"Vertical Forms": range(65040, 65056), |
|
"Combining Half Marks": range(65056, 65072), |
|
"CJK Compatibility Forms": range(65072, 65104), |
|
"Small Form Variants": range(65104, 65136), |
|
"Arabic Presentation Forms-B": range(65136, 65280), |
|
"Halfwidth and Fullwidth Forms": range(65280, 65520), |
|
"Specials": range(65520, 65536), |
|
"Linear B Syllabary": range(65536, 65664), |
|
"Linear B Ideograms": range(65664, 65792), |
|
"Aegean Numbers": range(65792, 65856), |
|
"Ancient Greek Numbers": range(65856, 65936), |
|
"Ancient Symbols": range(65936, 66000), |
|
"Phaistos Disc": range(66000, 66048), |
|
"Lycian": range(66176, 66208), |
|
"Carian": range(66208, 66272), |
|
"Coptic Epact Numbers": range(66272, 66304), |
|
"Old Italic": range(66304, 66352), |
|
"Gothic": range(66352, 66384), |
|
"Old Permic": range(66384, 66432), |
|
"Ugaritic": range(66432, 66464), |
|
"Old Persian": range(66464, 66528), |
|
"Deseret": range(66560, 66640), |
|
"Shavian": range(66640, 66688), |
|
"Osmanya": range(66688, 66736), |
|
"Osage": range(66736, 66816), |
|
"Elbasan": range(66816, 66864), |
|
"Caucasian Albanian": range(66864, 66928), |
|
"Vithkuqi": range(66928, 67008), |
|
"Linear A": range(67072, 67456), |
|
"Latin Extended-F": range(67456, 67520), |
|
"Cypriot Syllabary": range(67584, 67648), |
|
"Imperial Aramaic": range(67648, 67680), |
|
"Palmyrene": range(67680, 67712), |
|
"Nabataean": range(67712, 67760), |
|
"Hatran": range(67808, 67840), |
|
"Phoenician": range(67840, 67872), |
|
"Lydian": range(67872, 67904), |
|
"Meroitic Hieroglyphs": range(67968, 68000), |
|
"Meroitic Cursive": range(68000, 68096), |
|
"Kharoshthi": range(68096, 68192), |
|
"Old South Arabian": range(68192, 68224), |
|
"Old North Arabian": range(68224, 68256), |
|
"Manichaean": range(68288, 68352), |
|
"Avestan": range(68352, 68416), |
|
"Inscriptional Parthian": range(68416, 68448), |
|
"Inscriptional Pahlavi": range(68448, 68480), |
|
"Psalter Pahlavi": range(68480, 68528), |
|
"Old Turkic": range(68608, 68688), |
|
"Old Hungarian": range(68736, 68864), |
|
"Hanifi Rohingya": range(68864, 68928), |
|
"Rumi Numeral Symbols": range(69216, 69248), |
|
"Yezidi": range(69248, 69312), |
|
"Arabic Extended-C": range(69312, 69376), |
|
"Old Sogdian": range(69376, 69424), |
|
"Sogdian": range(69424, 69488), |
|
"Old Uyghur": range(69488, 69552), |
|
"Chorasmian": range(69552, 69600), |
|
"Elymaic": range(69600, 69632), |
|
"Brahmi": range(69632, 69760), |
|
"Kaithi": range(69760, 69840), |
|
"Sora Sompeng": range(69840, 69888), |
|
"Chakma": range(69888, 69968), |
|
"Mahajani": range(69968, 70016), |
|
"Sharada": range(70016, 70112), |
|
"Sinhala Archaic Numbers": range(70112, 70144), |
|
"Khojki": range(70144, 70224), |
|
"Multani": range(70272, 70320), |
|
"Khudawadi": range(70320, 70400), |
|
"Grantha": range(70400, 70528), |
|
"Newa": range(70656, 70784), |
|
"Tirhuta": range(70784, 70880), |
|
"Siddham": range(71040, 71168), |
|
"Modi": range(71168, 71264), |
|
"Mongolian Supplement": range(71264, 71296), |
|
"Takri": range(71296, 71376), |
|
"Ahom": range(71424, 71504), |
|
"Dogra": range(71680, 71760), |
|
"Warang Citi": range(71840, 71936), |
|
"Dives Akuru": range(71936, 72032), |
|
"Nandinagari": range(72096, 72192), |
|
"Zanabazar Square": range(72192, 72272), |
|
"Soyombo": range(72272, 72368), |
|
"Unified Canadian Aboriginal Syllabics Extended-A": range(72368, 72384), |
|
"Pau Cin Hau": range(72384, 72448), |
|
"Devanagari Extended-A": range(72448, 72544), |
|
"Bhaiksuki": range(72704, 72816), |
|
"Marchen": range(72816, 72896), |
|
"Masaram Gondi": range(72960, 73056), |
|
"Gunjala Gondi": range(73056, 73136), |
|
"Makasar": range(73440, 73472), |
|
"Kawi": range(73472, 73568), |
|
"Lisu Supplement": range(73648, 73664), |
|
"Tamil Supplement": range(73664, 73728), |
|
"Cuneiform": range(73728, 74752), |
|
"Cuneiform Numbers and Punctuation": range(74752, 74880), |
|
"Early Dynastic Cuneiform": range(74880, 75088), |
|
"Cypro-Minoan": range(77712, 77824), |
|
"Egyptian Hieroglyphs": range(77824, 78896), |
|
"Egyptian Hieroglyph Format Controls": range(78896, 78944), |
|
"Anatolian Hieroglyphs": range(82944, 83584), |
|
"Bamum Supplement": range(92160, 92736), |
|
"Mro": range(92736, 92784), |
|
"Tangsa": range(92784, 92880), |
|
"Bassa Vah": range(92880, 92928), |
|
"Pahawh Hmong": range(92928, 93072), |
|
"Medefaidrin": range(93760, 93856), |
|
"Miao": range(93952, 94112), |
|
"Ideographic Symbols and Punctuation": range(94176, 94208), |
|
"Tangut": range(94208, 100352), |
|
"Tangut Components": range(100352, 101120), |
|
"Khitan Small Script": range(101120, 101632), |
|
"Tangut Supplement": range(101632, 101760), |
|
"Kana Extended-B": range(110576, 110592), |
|
"Kana Supplement": range(110592, 110848), |
|
"Kana Extended-A": range(110848, 110896), |
|
"Small Kana Extension": range(110896, 110960), |
|
"Nushu": range(110960, 111360), |
|
"Duployan": range(113664, 113824), |
|
"Shorthand Format Controls": range(113824, 113840), |
|
"Znamenny Musical Notation": range(118528, 118736), |
|
"Byzantine Musical Symbols": range(118784, 119040), |
|
"Musical Symbols": range(119040, 119296), |
|
"Ancient Greek Musical Notation": range(119296, 119376), |
|
"Kaktovik Numerals": range(119488, 119520), |
|
"Mayan Numerals": range(119520, 119552), |
|
"Tai Xuan Jing Symbols": range(119552, 119648), |
|
"Counting Rod Numerals": range(119648, 119680), |
|
"Mathematical Alphanumeric Symbols": range(119808, 120832), |
|
"Sutton SignWriting": range(120832, 121520), |
|
"Latin Extended-G": range(122624, 122880), |
|
"Glagolitic Supplement": range(122880, 122928), |
|
"Cyrillic Extended-D": range(122928, 123024), |
|
"Nyiakeng Puachue Hmong": range(123136, 123216), |
|
"Toto": range(123536, 123584), |
|
"Wancho": range(123584, 123648), |
|
"Nag Mundari": range(124112, 124160), |
|
"Ethiopic Extended-B": range(124896, 124928), |
|
"Mende Kikakui": range(124928, 125152), |
|
"Adlam": range(125184, 125280), |
|
"Indic Siyaq Numbers": range(126064, 126144), |
|
"Ottoman Siyaq Numbers": range(126208, 126288), |
|
"Arabic Mathematical Alphabetic Symbols": range(126464, 126720), |
|
"Mahjong Tiles": range(126976, 127024), |
|
"Domino Tiles": range(127024, 127136), |
|
"Playing Cards": range(127136, 127232), |
|
"Enclosed Alphanumeric Supplement": range(127232, 127488), |
|
"Enclosed Ideographic Supplement": range(127488, 127744), |
|
"Miscellaneous Symbols and Pictographs": range(127744, 128512), |
|
"Emoticons range(Emoji)": range(128512, 128592), |
|
"Ornamental Dingbats": range(128592, 128640), |
|
"Transport and Map Symbols": range(128640, 128768), |
|
"Alchemical Symbols": range(128768, 128896), |
|
"Geometric Shapes Extended": range(128896, 129024), |
|
"Supplemental Arrows-C": range(129024, 129280), |
|
"Supplemental Symbols and Pictographs": range(129280, 129536), |
|
"Chess Symbols": range(129536, 129648), |
|
"Symbols and Pictographs Extended-A": range(129648, 129792), |
|
"Symbols for Legacy Computing": range(129792, 130048), |
|
"CJK Unified Ideographs Extension B": range(131072, 173792), |
|
"CJK Unified Ideographs Extension C": range(173824, 177984), |
|
"CJK Unified Ideographs Extension D": range(177984, 178208), |
|
"CJK Unified Ideographs Extension E": range(178208, 183984), |
|
"CJK Unified Ideographs Extension F": range(183984, 191472), |
|
"CJK Compatibility Ideographs Supplement": range(194560, 195104), |
|
"CJK Unified Ideographs Extension G": range(196608, 201552), |
|
"CJK Unified Ideographs Extension H": range(201552, 205744), |
|
"Tags": range(917504, 917632), |
|
"Variation Selectors Supplement": range(917760, 918000), |
|
"Supplementary Private Use Area-A": range(983040, 1048576), |
|
"Supplementary Private Use Area-B": range(1048576, 1114112), |
|
} |
|
|
|
|
|
UNICODE_SECONDARY_RANGE_KEYWORD: List[str] = [ |
|
"Supplement", |
|
"Extended", |
|
"Extensions", |
|
"Modifier", |
|
"Marks", |
|
"Punctuation", |
|
"Symbols", |
|
"Forms", |
|
"Operators", |
|
"Miscellaneous", |
|
"Drawing", |
|
"Block", |
|
"Shapes", |
|
"Supplemental", |
|
"Tags", |
|
] |
|
|
|
RE_POSSIBLE_ENCODING_INDICATION = re_compile( |
|
r"(?:(?:encoding)|(?:charset)|(?:coding))(?:[\:= ]{1,10})(?:[\"\']?)([a-zA-Z0-9\-_]+)(?:[\"\']?)", |
|
IGNORECASE, |
|
) |
|
|
|
IANA_NO_ALIASES = [ |
|
"cp720", |
|
"cp737", |
|
"cp856", |
|
"cp874", |
|
"cp875", |
|
"cp1006", |
|
"koi8_r", |
|
"koi8_t", |
|
"koi8_u", |
|
] |
|
|
|
IANA_SUPPORTED: List[str] = sorted( |
|
filter( |
|
lambda x: x.endswith("_codec") is False |
|
and x not in {"rot_13", "tactis", "mbcs"}, |
|
list(set(aliases.values())) + IANA_NO_ALIASES, |
|
) |
|
) |
|
|
|
IANA_SUPPORTED_COUNT: int = len(IANA_SUPPORTED) |
|
|
|
|
|
IANA_SUPPORTED_SIMILAR: Dict[str, List[str]] = { |
|
"cp037": ["cp1026", "cp1140", "cp273", "cp500"], |
|
"cp1026": ["cp037", "cp1140", "cp273", "cp500"], |
|
"cp1125": ["cp866"], |
|
"cp1140": ["cp037", "cp1026", "cp273", "cp500"], |
|
"cp1250": ["iso8859_2"], |
|
"cp1251": ["kz1048", "ptcp154"], |
|
"cp1252": ["iso8859_15", "iso8859_9", "latin_1"], |
|
"cp1253": ["iso8859_7"], |
|
"cp1254": ["iso8859_15", "iso8859_9", "latin_1"], |
|
"cp1257": ["iso8859_13"], |
|
"cp273": ["cp037", "cp1026", "cp1140", "cp500"], |
|
"cp437": ["cp850", "cp858", "cp860", "cp861", "cp862", "cp863", "cp865"], |
|
"cp500": ["cp037", "cp1026", "cp1140", "cp273"], |
|
"cp850": ["cp437", "cp857", "cp858", "cp865"], |
|
"cp857": ["cp850", "cp858", "cp865"], |
|
"cp858": ["cp437", "cp850", "cp857", "cp865"], |
|
"cp860": ["cp437", "cp861", "cp862", "cp863", "cp865"], |
|
"cp861": ["cp437", "cp860", "cp862", "cp863", "cp865"], |
|
"cp862": ["cp437", "cp860", "cp861", "cp863", "cp865"], |
|
"cp863": ["cp437", "cp860", "cp861", "cp862", "cp865"], |
|
"cp865": ["cp437", "cp850", "cp857", "cp858", "cp860", "cp861", "cp862", "cp863"], |
|
"cp866": ["cp1125"], |
|
"iso8859_10": ["iso8859_14", "iso8859_15", "iso8859_4", "iso8859_9", "latin_1"], |
|
"iso8859_11": ["tis_620"], |
|
"iso8859_13": ["cp1257"], |
|
"iso8859_14": [ |
|
"iso8859_10", |
|
"iso8859_15", |
|
"iso8859_16", |
|
"iso8859_3", |
|
"iso8859_9", |
|
"latin_1", |
|
], |
|
"iso8859_15": [ |
|
"cp1252", |
|
"cp1254", |
|
"iso8859_10", |
|
"iso8859_14", |
|
"iso8859_16", |
|
"iso8859_3", |
|
"iso8859_9", |
|
"latin_1", |
|
], |
|
"iso8859_16": [ |
|
"iso8859_14", |
|
"iso8859_15", |
|
"iso8859_2", |
|
"iso8859_3", |
|
"iso8859_9", |
|
"latin_1", |
|
], |
|
"iso8859_2": ["cp1250", "iso8859_16", "iso8859_4"], |
|
"iso8859_3": ["iso8859_14", "iso8859_15", "iso8859_16", "iso8859_9", "latin_1"], |
|
"iso8859_4": ["iso8859_10", "iso8859_2", "iso8859_9", "latin_1"], |
|
"iso8859_7": ["cp1253"], |
|
"iso8859_9": [ |
|
"cp1252", |
|
"cp1254", |
|
"cp1258", |
|
"iso8859_10", |
|
"iso8859_14", |
|
"iso8859_15", |
|
"iso8859_16", |
|
"iso8859_3", |
|
"iso8859_4", |
|
"latin_1", |
|
], |
|
"kz1048": ["cp1251", "ptcp154"], |
|
"latin_1": [ |
|
"cp1252", |
|
"cp1254", |
|
"cp1258", |
|
"iso8859_10", |
|
"iso8859_14", |
|
"iso8859_15", |
|
"iso8859_16", |
|
"iso8859_3", |
|
"iso8859_4", |
|
"iso8859_9", |
|
], |
|
"mac_iceland": ["mac_roman", "mac_turkish"], |
|
"mac_roman": ["mac_iceland", "mac_turkish"], |
|
"mac_turkish": ["mac_iceland", "mac_roman"], |
|
"ptcp154": ["cp1251", "kz1048"], |
|
"tis_620": ["iso8859_11"], |
|
} |
|
|
|
|
|
CHARDET_CORRESPONDENCE: Dict[str, str] = { |
|
"iso2022_kr": "ISO-2022-KR", |
|
"iso2022_jp": "ISO-2022-JP", |
|
"euc_kr": "EUC-KR", |
|
"tis_620": "TIS-620", |
|
"utf_32": "UTF-32", |
|
"euc_jp": "EUC-JP", |
|
"koi8_r": "KOI8-R", |
|
"iso8859_1": "ISO-8859-1", |
|
"iso8859_2": "ISO-8859-2", |
|
"iso8859_5": "ISO-8859-5", |
|
"iso8859_6": "ISO-8859-6", |
|
"iso8859_7": "ISO-8859-7", |
|
"iso8859_8": "ISO-8859-8", |
|
"utf_16": "UTF-16", |
|
"cp855": "IBM855", |
|
"mac_cyrillic": "MacCyrillic", |
|
"gb2312": "GB2312", |
|
"gb18030": "GB18030", |
|
"cp932": "CP932", |
|
"cp866": "IBM866", |
|
"utf_8": "utf-8", |
|
"utf_8_sig": "UTF-8-SIG", |
|
"shift_jis": "SHIFT_JIS", |
|
"big5": "Big5", |
|
"cp1250": "windows-1250", |
|
"cp1251": "windows-1251", |
|
"cp1252": "Windows-1252", |
|
"cp1253": "windows-1253", |
|
"cp1255": "windows-1255", |
|
"cp1256": "windows-1256", |
|
"cp1254": "Windows-1254", |
|
"cp949": "CP949", |
|
} |
|
|
|
|
|
COMMON_SAFE_ASCII_CHARACTERS: Set[str] = { |
|
"<", |
|
">", |
|
"=", |
|
":", |
|
"/", |
|
"&", |
|
";", |
|
"{", |
|
"}", |
|
"[", |
|
"]", |
|
",", |
|
"|", |
|
'"', |
|
"-", |
|
} |
|
|
|
|
|
KO_NAMES: Set[str] = {"johab", "cp949", "euc_kr"} |
|
ZH_NAMES: Set[str] = {"big5", "cp950", "big5hkscs", "hz"} |
|
|
|
|
|
TRACE: int = 5 |
|
|
|
|
|
|
|
|
|
FREQUENCIES: Dict[str, List[str]] = { |
|
"English": [ |
|
"e", |
|
"a", |
|
"t", |
|
"i", |
|
"o", |
|
"n", |
|
"s", |
|
"r", |
|
"h", |
|
"l", |
|
"d", |
|
"c", |
|
"u", |
|
"m", |
|
"f", |
|
"p", |
|
"g", |
|
"w", |
|
"y", |
|
"b", |
|
"v", |
|
"k", |
|
"x", |
|
"j", |
|
"z", |
|
"q", |
|
], |
|
"English—": [ |
|
"e", |
|
"a", |
|
"t", |
|
"i", |
|
"o", |
|
"n", |
|
"s", |
|
"r", |
|
"h", |
|
"l", |
|
"d", |
|
"c", |
|
"m", |
|
"u", |
|
"f", |
|
"p", |
|
"g", |
|
"w", |
|
"b", |
|
"y", |
|
"v", |
|
"k", |
|
"j", |
|
"x", |
|
"z", |
|
"q", |
|
], |
|
"German": [ |
|
"e", |
|
"n", |
|
"i", |
|
"r", |
|
"s", |
|
"t", |
|
"a", |
|
"d", |
|
"h", |
|
"u", |
|
"l", |
|
"g", |
|
"o", |
|
"c", |
|
"m", |
|
"b", |
|
"f", |
|
"k", |
|
"w", |
|
"z", |
|
"p", |
|
"v", |
|
"ü", |
|
"ä", |
|
"ö", |
|
"j", |
|
], |
|
"French": [ |
|
"e", |
|
"a", |
|
"s", |
|
"n", |
|
"i", |
|
"t", |
|
"r", |
|
"l", |
|
"u", |
|
"o", |
|
"d", |
|
"c", |
|
"p", |
|
"m", |
|
"é", |
|
"v", |
|
"g", |
|
"f", |
|
"b", |
|
"h", |
|
"q", |
|
"à", |
|
"x", |
|
"è", |
|
"y", |
|
"j", |
|
], |
|
"Dutch": [ |
|
"e", |
|
"n", |
|
"a", |
|
"i", |
|
"r", |
|
"t", |
|
"o", |
|
"d", |
|
"s", |
|
"l", |
|
"g", |
|
"h", |
|
"v", |
|
"m", |
|
"u", |
|
"k", |
|
"c", |
|
"p", |
|
"b", |
|
"w", |
|
"j", |
|
"z", |
|
"f", |
|
"y", |
|
"x", |
|
"ë", |
|
], |
|
"Italian": [ |
|
"e", |
|
"i", |
|
"a", |
|
"o", |
|
"n", |
|
"l", |
|
"t", |
|
"r", |
|
"s", |
|
"c", |
|
"d", |
|
"u", |
|
"p", |
|
"m", |
|
"g", |
|
"v", |
|
"f", |
|
"b", |
|
"z", |
|
"h", |
|
"q", |
|
"è", |
|
"à", |
|
"k", |
|
"y", |
|
"ò", |
|
], |
|
"Polish": [ |
|
"a", |
|
"i", |
|
"o", |
|
"e", |
|
"n", |
|
"r", |
|
"z", |
|
"w", |
|
"s", |
|
"c", |
|
"t", |
|
"k", |
|
"y", |
|
"d", |
|
"p", |
|
"m", |
|
"u", |
|
"l", |
|
"j", |
|
"ł", |
|
"g", |
|
"b", |
|
"h", |
|
"ą", |
|
"ę", |
|
"ó", |
|
], |
|
"Spanish": [ |
|
"e", |
|
"a", |
|
"o", |
|
"n", |
|
"s", |
|
"r", |
|
"i", |
|
"l", |
|
"d", |
|
"t", |
|
"c", |
|
"u", |
|
"m", |
|
"p", |
|
"b", |
|
"g", |
|
"v", |
|
"f", |
|
"y", |
|
"ó", |
|
"h", |
|
"q", |
|
"í", |
|
"j", |
|
"z", |
|
"á", |
|
], |
|
"Russian": [ |
|
"о", |
|
"а", |
|
"е", |
|
"и", |
|
"н", |
|
"с", |
|
"т", |
|
"р", |
|
"в", |
|
"л", |
|
"к", |
|
"м", |
|
"д", |
|
"п", |
|
"у", |
|
"г", |
|
"я", |
|
"ы", |
|
"з", |
|
"б", |
|
"й", |
|
"ь", |
|
"ч", |
|
"х", |
|
"ж", |
|
"ц", |
|
], |
|
|
|
"Japanese": [ |
|
"人", |
|
"一", |
|
"大", |
|
"亅", |
|
"丁", |
|
"丨", |
|
"竹", |
|
"笑", |
|
"口", |
|
"日", |
|
"今", |
|
"二", |
|
"彳", |
|
"行", |
|
"十", |
|
"土", |
|
"丶", |
|
"寸", |
|
"寺", |
|
"時", |
|
"乙", |
|
"丿", |
|
"乂", |
|
"气", |
|
"気", |
|
"冂", |
|
"巾", |
|
"亠", |
|
"市", |
|
"目", |
|
"儿", |
|
"見", |
|
"八", |
|
"小", |
|
"凵", |
|
"県", |
|
"月", |
|
"彐", |
|
"門", |
|
"間", |
|
"木", |
|
"東", |
|
"山", |
|
"出", |
|
"本", |
|
"中", |
|
"刀", |
|
"分", |
|
"耳", |
|
"又", |
|
"取", |
|
"最", |
|
"言", |
|
"田", |
|
"心", |
|
"思", |
|
"刂", |
|
"前", |
|
"京", |
|
"尹", |
|
"事", |
|
"生", |
|
"厶", |
|
"云", |
|
"会", |
|
"未", |
|
"来", |
|
"白", |
|
"冫", |
|
"楽", |
|
"灬", |
|
"馬", |
|
"尸", |
|
"尺", |
|
"駅", |
|
"明", |
|
"耂", |
|
"者", |
|
"了", |
|
"阝", |
|
"都", |
|
"高", |
|
"卜", |
|
"占", |
|
"厂", |
|
"广", |
|
"店", |
|
"子", |
|
"申", |
|
"奄", |
|
"亻", |
|
"俺", |
|
"上", |
|
"方", |
|
"冖", |
|
"学", |
|
"衣", |
|
"艮", |
|
"食", |
|
"自", |
|
], |
|
|
|
"Japanese—": [ |
|
"ー", |
|
"ン", |
|
"ス", |
|
"・", |
|
"ル", |
|
"ト", |
|
"リ", |
|
"イ", |
|
"ア", |
|
"ラ", |
|
"ッ", |
|
"ク", |
|
"ド", |
|
"シ", |
|
"レ", |
|
"ジ", |
|
"タ", |
|
"フ", |
|
"ロ", |
|
"カ", |
|
"テ", |
|
"マ", |
|
"ィ", |
|
"グ", |
|
"バ", |
|
"ム", |
|
"プ", |
|
"オ", |
|
"コ", |
|
"デ", |
|
"ニ", |
|
"ウ", |
|
"メ", |
|
"サ", |
|
"ビ", |
|
"ナ", |
|
"ブ", |
|
"ャ", |
|
"エ", |
|
"ュ", |
|
"チ", |
|
"キ", |
|
"ズ", |
|
"ダ", |
|
"パ", |
|
"ミ", |
|
"ェ", |
|
"ョ", |
|
"ハ", |
|
"セ", |
|
"ベ", |
|
"ガ", |
|
"モ", |
|
"ツ", |
|
"ネ", |
|
"ボ", |
|
"ソ", |
|
"ノ", |
|
"ァ", |
|
"ヴ", |
|
"ワ", |
|
"ポ", |
|
"ペ", |
|
"ピ", |
|
"ケ", |
|
"ゴ", |
|
"ギ", |
|
"ザ", |
|
"ホ", |
|
"ゲ", |
|
"ォ", |
|
"ヤ", |
|
"ヒ", |
|
"ユ", |
|
"ヨ", |
|
"ヘ", |
|
"ゼ", |
|
"ヌ", |
|
"ゥ", |
|
"ゾ", |
|
"ヶ", |
|
"ヂ", |
|
"ヲ", |
|
"ヅ", |
|
"ヵ", |
|
"ヱ", |
|
"ヰ", |
|
"ヮ", |
|
"ヽ", |
|
"゠", |
|
"ヾ", |
|
"ヷ", |
|
"ヿ", |
|
"ヸ", |
|
"ヹ", |
|
"ヺ", |
|
], |
|
|
|
"Japanese——": [ |
|
"の", |
|
"に", |
|
"る", |
|
"た", |
|
"と", |
|
"は", |
|
"し", |
|
"い", |
|
"を", |
|
"で", |
|
"て", |
|
"が", |
|
"な", |
|
"れ", |
|
"か", |
|
"ら", |
|
"さ", |
|
"っ", |
|
"り", |
|
"す", |
|
"あ", |
|
"も", |
|
"こ", |
|
"ま", |
|
"う", |
|
"く", |
|
"よ", |
|
"き", |
|
"ん", |
|
"め", |
|
"お", |
|
"け", |
|
"そ", |
|
"つ", |
|
"だ", |
|
"や", |
|
"え", |
|
"ど", |
|
"わ", |
|
"ち", |
|
"み", |
|
"せ", |
|
"じ", |
|
"ば", |
|
"へ", |
|
"び", |
|
"ず", |
|
"ろ", |
|
"ほ", |
|
"げ", |
|
"む", |
|
"べ", |
|
"ひ", |
|
"ょ", |
|
"ゆ", |
|
"ぶ", |
|
"ご", |
|
"ゃ", |
|
"ね", |
|
"ふ", |
|
"ぐ", |
|
"ぎ", |
|
"ぼ", |
|
"ゅ", |
|
"づ", |
|
"ざ", |
|
"ぞ", |
|
"ぬ", |
|
"ぜ", |
|
"ぱ", |
|
"ぽ", |
|
"ぷ", |
|
"ぴ", |
|
"ぃ", |
|
"ぁ", |
|
"ぇ", |
|
"ぺ", |
|
"ゞ", |
|
"ぢ", |
|
"ぉ", |
|
"ぅ", |
|
"ゐ", |
|
"ゝ", |
|
"ゑ", |
|
"゛", |
|
"゜", |
|
"ゎ", |
|
"ゔ", |
|
"゚", |
|
"ゟ", |
|
"゙", |
|
"ゕ", |
|
"ゖ", |
|
], |
|
"Portuguese": [ |
|
"a", |
|
"e", |
|
"o", |
|
"s", |
|
"i", |
|
"r", |
|
"d", |
|
"n", |
|
"t", |
|
"m", |
|
"u", |
|
"c", |
|
"l", |
|
"p", |
|
"g", |
|
"v", |
|
"b", |
|
"f", |
|
"h", |
|
"ã", |
|
"q", |
|
"é", |
|
"ç", |
|
"á", |
|
"z", |
|
"í", |
|
], |
|
"Swedish": [ |
|
"e", |
|
"a", |
|
"n", |
|
"r", |
|
"t", |
|
"s", |
|
"i", |
|
"l", |
|
"d", |
|
"o", |
|
"m", |
|
"k", |
|
"g", |
|
"v", |
|
"h", |
|
"f", |
|
"u", |
|
"p", |
|
"ä", |
|
"c", |
|
"b", |
|
"ö", |
|
"å", |
|
"y", |
|
"j", |
|
"x", |
|
], |
|
"Chinese": [ |
|
"的", |
|
"一", |
|
"是", |
|
"不", |
|
"了", |
|
"在", |
|
"人", |
|
"有", |
|
"我", |
|
"他", |
|
"这", |
|
"个", |
|
"们", |
|
"中", |
|
"来", |
|
"上", |
|
"大", |
|
"为", |
|
"和", |
|
"国", |
|
"地", |
|
"到", |
|
"以", |
|
"说", |
|
"时", |
|
"要", |
|
"就", |
|
"出", |
|
"会", |
|
"可", |
|
"也", |
|
"你", |
|
"对", |
|
"生", |
|
"能", |
|
"而", |
|
"子", |
|
"那", |
|
"得", |
|
"于", |
|
"着", |
|
"下", |
|
"自", |
|
"之", |
|
"年", |
|
"过", |
|
"发", |
|
"后", |
|
"作", |
|
"里", |
|
"用", |
|
"道", |
|
"行", |
|
"所", |
|
"然", |
|
"家", |
|
"种", |
|
"事", |
|
"成", |
|
"方", |
|
"多", |
|
"经", |
|
"么", |
|
"去", |
|
"法", |
|
"学", |
|
"如", |
|
"都", |
|
"同", |
|
"现", |
|
"当", |
|
"没", |
|
"动", |
|
"面", |
|
"起", |
|
"看", |
|
"定", |
|
"天", |
|
"分", |
|
"还", |
|
"进", |
|
"好", |
|
"小", |
|
"部", |
|
"其", |
|
"些", |
|
"主", |
|
"样", |
|
"理", |
|
"心", |
|
"她", |
|
"本", |
|
"前", |
|
"开", |
|
"但", |
|
"因", |
|
"只", |
|
"从", |
|
"想", |
|
"实", |
|
], |
|
"Ukrainian": [ |
|
"о", |
|
"а", |
|
"н", |
|
"і", |
|
"и", |
|
"р", |
|
"в", |
|
"т", |
|
"е", |
|
"с", |
|
"к", |
|
"л", |
|
"у", |
|
"д", |
|
"м", |
|
"п", |
|
"з", |
|
"я", |
|
"ь", |
|
"б", |
|
"г", |
|
"й", |
|
"ч", |
|
"х", |
|
"ц", |
|
"ї", |
|
], |
|
"Norwegian": [ |
|
"e", |
|
"r", |
|
"n", |
|
"t", |
|
"a", |
|
"s", |
|
"i", |
|
"o", |
|
"l", |
|
"d", |
|
"g", |
|
"k", |
|
"m", |
|
"v", |
|
"f", |
|
"p", |
|
"u", |
|
"b", |
|
"h", |
|
"å", |
|
"y", |
|
"j", |
|
"ø", |
|
"c", |
|
"æ", |
|
"w", |
|
], |
|
"Finnish": [ |
|
"a", |
|
"i", |
|
"n", |
|
"t", |
|
"e", |
|
"s", |
|
"l", |
|
"o", |
|
"u", |
|
"k", |
|
"ä", |
|
"m", |
|
"r", |
|
"v", |
|
"j", |
|
"h", |
|
"p", |
|
"y", |
|
"d", |
|
"ö", |
|
"g", |
|
"c", |
|
"b", |
|
"f", |
|
"w", |
|
"z", |
|
], |
|
"Vietnamese": [ |
|
"n", |
|
"h", |
|
"t", |
|
"i", |
|
"c", |
|
"g", |
|
"a", |
|
"o", |
|
"u", |
|
"m", |
|
"l", |
|
"r", |
|
"à", |
|
"đ", |
|
"s", |
|
"e", |
|
"v", |
|
"p", |
|
"b", |
|
"y", |
|
"ư", |
|
"d", |
|
"á", |
|
"k", |
|
"ộ", |
|
"ế", |
|
], |
|
"Czech": [ |
|
"o", |
|
"e", |
|
"a", |
|
"n", |
|
"t", |
|
"s", |
|
"i", |
|
"l", |
|
"v", |
|
"r", |
|
"k", |
|
"d", |
|
"u", |
|
"m", |
|
"p", |
|
"í", |
|
"c", |
|
"h", |
|
"z", |
|
"á", |
|
"y", |
|
"j", |
|
"b", |
|
"ě", |
|
"é", |
|
"ř", |
|
], |
|
"Hungarian": [ |
|
"e", |
|
"a", |
|
"t", |
|
"l", |
|
"s", |
|
"n", |
|
"k", |
|
"r", |
|
"i", |
|
"o", |
|
"z", |
|
"á", |
|
"é", |
|
"g", |
|
"m", |
|
"b", |
|
"y", |
|
"v", |
|
"d", |
|
"h", |
|
"u", |
|
"p", |
|
"j", |
|
"ö", |
|
"f", |
|
"c", |
|
], |
|
"Korean": [ |
|
"이", |
|
"다", |
|
"에", |
|
"의", |
|
"는", |
|
"로", |
|
"하", |
|
"을", |
|
"가", |
|
"고", |
|
"지", |
|
"서", |
|
"한", |
|
"은", |
|
"기", |
|
"으", |
|
"년", |
|
"대", |
|
"사", |
|
"시", |
|
"를", |
|
"리", |
|
"도", |
|
"인", |
|
"스", |
|
"일", |
|
], |
|
"Indonesian": [ |
|
"a", |
|
"n", |
|
"e", |
|
"i", |
|
"r", |
|
"t", |
|
"u", |
|
"s", |
|
"d", |
|
"k", |
|
"m", |
|
"l", |
|
"g", |
|
"p", |
|
"b", |
|
"o", |
|
"h", |
|
"y", |
|
"j", |
|
"c", |
|
"w", |
|
"f", |
|
"v", |
|
"z", |
|
"x", |
|
"q", |
|
], |
|
"Turkish": [ |
|
"a", |
|
"e", |
|
"i", |
|
"n", |
|
"r", |
|
"l", |
|
"ı", |
|
"k", |
|
"d", |
|
"t", |
|
"s", |
|
"m", |
|
"y", |
|
"u", |
|
"o", |
|
"b", |
|
"ü", |
|
"ş", |
|
"v", |
|
"g", |
|
"z", |
|
"h", |
|
"c", |
|
"p", |
|
"ç", |
|
"ğ", |
|
], |
|
"Romanian": [ |
|
"e", |
|
"i", |
|
"a", |
|
"r", |
|
"n", |
|
"t", |
|
"u", |
|
"l", |
|
"o", |
|
"c", |
|
"s", |
|
"d", |
|
"p", |
|
"m", |
|
"ă", |
|
"f", |
|
"v", |
|
"î", |
|
"g", |
|
"b", |
|
"ș", |
|
"ț", |
|
"z", |
|
"h", |
|
"â", |
|
"j", |
|
], |
|
"Farsi": [ |
|
"ا", |
|
"ی", |
|
"ر", |
|
"د", |
|
"ن", |
|
"ه", |
|
"و", |
|
"م", |
|
"ت", |
|
"ب", |
|
"س", |
|
"ل", |
|
"ک", |
|
"ش", |
|
"ز", |
|
"ف", |
|
"گ", |
|
"ع", |
|
"خ", |
|
"ق", |
|
"ج", |
|
"آ", |
|
"پ", |
|
"ح", |
|
"ط", |
|
"ص", |
|
], |
|
"Arabic": [ |
|
"ا", |
|
"ل", |
|
"ي", |
|
"م", |
|
"و", |
|
"ن", |
|
"ر", |
|
"ت", |
|
"ب", |
|
"ة", |
|
"ع", |
|
"د", |
|
"س", |
|
"ف", |
|
"ه", |
|
"ك", |
|
"ق", |
|
"أ", |
|
"ح", |
|
"ج", |
|
"ش", |
|
"ط", |
|
"ص", |
|
"ى", |
|
"خ", |
|
"إ", |
|
], |
|
"Danish": [ |
|
"e", |
|
"r", |
|
"n", |
|
"t", |
|
"a", |
|
"i", |
|
"s", |
|
"d", |
|
"l", |
|
"o", |
|
"g", |
|
"m", |
|
"k", |
|
"f", |
|
"v", |
|
"u", |
|
"b", |
|
"h", |
|
"p", |
|
"å", |
|
"y", |
|
"ø", |
|
"æ", |
|
"c", |
|
"j", |
|
"w", |
|
], |
|
"Serbian": [ |
|
"а", |
|
"и", |
|
"о", |
|
"е", |
|
"н", |
|
"р", |
|
"с", |
|
"у", |
|
"т", |
|
"к", |
|
"ј", |
|
"в", |
|
"д", |
|
"м", |
|
"п", |
|
"л", |
|
"г", |
|
"з", |
|
"б", |
|
"a", |
|
"i", |
|
"e", |
|
"o", |
|
"n", |
|
"ц", |
|
"ш", |
|
], |
|
"Lithuanian": [ |
|
"i", |
|
"a", |
|
"s", |
|
"o", |
|
"r", |
|
"e", |
|
"t", |
|
"n", |
|
"u", |
|
"k", |
|
"m", |
|
"l", |
|
"p", |
|
"v", |
|
"d", |
|
"j", |
|
"g", |
|
"ė", |
|
"b", |
|
"y", |
|
"ų", |
|
"š", |
|
"ž", |
|
"c", |
|
"ą", |
|
"į", |
|
], |
|
"Slovene": [ |
|
"e", |
|
"a", |
|
"i", |
|
"o", |
|
"n", |
|
"r", |
|
"s", |
|
"l", |
|
"t", |
|
"j", |
|
"v", |
|
"k", |
|
"d", |
|
"p", |
|
"m", |
|
"u", |
|
"z", |
|
"b", |
|
"g", |
|
"h", |
|
"č", |
|
"c", |
|
"š", |
|
"ž", |
|
"f", |
|
"y", |
|
], |
|
"Slovak": [ |
|
"o", |
|
"a", |
|
"e", |
|
"n", |
|
"i", |
|
"r", |
|
"v", |
|
"t", |
|
"s", |
|
"l", |
|
"k", |
|
"d", |
|
"m", |
|
"p", |
|
"u", |
|
"c", |
|
"h", |
|
"j", |
|
"b", |
|
"z", |
|
"á", |
|
"y", |
|
"ý", |
|
"í", |
|
"č", |
|
"é", |
|
], |
|
"Hebrew": [ |
|
"י", |
|
"ו", |
|
"ה", |
|
"ל", |
|
"ר", |
|
"ב", |
|
"ת", |
|
"מ", |
|
"א", |
|
"ש", |
|
"נ", |
|
"ע", |
|
"ם", |
|
"ד", |
|
"ק", |
|
"ח", |
|
"פ", |
|
"ס", |
|
"כ", |
|
"ג", |
|
"ט", |
|
"צ", |
|
"ן", |
|
"ז", |
|
"ך", |
|
], |
|
"Bulgarian": [ |
|
"а", |
|
"и", |
|
"о", |
|
"е", |
|
"н", |
|
"т", |
|
"р", |
|
"с", |
|
"в", |
|
"л", |
|
"к", |
|
"д", |
|
"п", |
|
"м", |
|
"з", |
|
"г", |
|
"я", |
|
"ъ", |
|
"у", |
|
"б", |
|
"ч", |
|
"ц", |
|
"й", |
|
"ж", |
|
"щ", |
|
"х", |
|
], |
|
"Croatian": [ |
|
"a", |
|
"i", |
|
"o", |
|
"e", |
|
"n", |
|
"r", |
|
"j", |
|
"s", |
|
"t", |
|
"u", |
|
"k", |
|
"l", |
|
"v", |
|
"d", |
|
"m", |
|
"p", |
|
"g", |
|
"z", |
|
"b", |
|
"c", |
|
"č", |
|
"h", |
|
"š", |
|
"ž", |
|
"ć", |
|
"f", |
|
], |
|
"Hindi": [ |
|
"क", |
|
"र", |
|
"स", |
|
"न", |
|
"त", |
|
"म", |
|
"ह", |
|
"प", |
|
"य", |
|
"ल", |
|
"व", |
|
"ज", |
|
"द", |
|
"ग", |
|
"ब", |
|
"श", |
|
"ट", |
|
"अ", |
|
"ए", |
|
"थ", |
|
"भ", |
|
"ड", |
|
"च", |
|
"ध", |
|
"ष", |
|
"इ", |
|
], |
|
"Estonian": [ |
|
"a", |
|
"i", |
|
"e", |
|
"s", |
|
"t", |
|
"l", |
|
"u", |
|
"n", |
|
"o", |
|
"k", |
|
"r", |
|
"d", |
|
"m", |
|
"v", |
|
"g", |
|
"p", |
|
"j", |
|
"h", |
|
"ä", |
|
"b", |
|
"õ", |
|
"ü", |
|
"f", |
|
"c", |
|
"ö", |
|
"y", |
|
], |
|
"Thai": [ |
|
"า", |
|
"น", |
|
"ร", |
|
"อ", |
|
"ก", |
|
"เ", |
|
"ง", |
|
"ม", |
|
"ย", |
|
"ล", |
|
"ว", |
|
"ด", |
|
"ท", |
|
"ส", |
|
"ต", |
|
"ะ", |
|
"ป", |
|
"บ", |
|
"ค", |
|
"ห", |
|
"แ", |
|
"จ", |
|
"พ", |
|
"ช", |
|
"ข", |
|
"ใ", |
|
], |
|
"Greek": [ |
|
"α", |
|
"τ", |
|
"ο", |
|
"ι", |
|
"ε", |
|
"ν", |
|
"ρ", |
|
"σ", |
|
"κ", |
|
"η", |
|
"π", |
|
"ς", |
|
"υ", |
|
"μ", |
|
"λ", |
|
"ί", |
|
"ό", |
|
"ά", |
|
"γ", |
|
"έ", |
|
"δ", |
|
"ή", |
|
"ω", |
|
"χ", |
|
"θ", |
|
"ύ", |
|
], |
|
"Tamil": [ |
|
"க", |
|
"த", |
|
"ப", |
|
"ட", |
|
"ர", |
|
"ம", |
|
"ல", |
|
"ன", |
|
"வ", |
|
"ற", |
|
"ய", |
|
"ள", |
|
"ச", |
|
"ந", |
|
"இ", |
|
"ண", |
|
"அ", |
|
"ஆ", |
|
"ழ", |
|
"ங", |
|
"எ", |
|
"உ", |
|
"ஒ", |
|
"ஸ", |
|
], |
|
"Kazakh": [ |
|
"а", |
|
"ы", |
|
"е", |
|
"н", |
|
"т", |
|
"р", |
|
"л", |
|
"і", |
|
"д", |
|
"с", |
|
"м", |
|
"қ", |
|
"к", |
|
"о", |
|
"б", |
|
"и", |
|
"у", |
|
"ғ", |
|
"ж", |
|
"ң", |
|
"з", |
|
"ш", |
|
"й", |
|
"п", |
|
"г", |
|
"ө", |
|
], |
|
} |
|
|
|
LANGUAGE_SUPPORTED_COUNT: int = len(FREQUENCIES) |
|
|
