Commit
·
e11875c
1
Parent(s):
56eb2dc
Upload tokenizer
Browse files- merges.txt +94 -0
- special_tokens_map.json +5 -0
- tokenizer.json +491 -0
- tokenizer_config.json +9 -0
- vocab.json +1 -0
merges.txt
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version: 0.2
|
| 2 |
+
i n
|
| 3 |
+
i s
|
| 4 |
+
t e
|
| 5 |
+
Ġ o
|
| 6 |
+
Ġ is
|
| 7 |
+
a l
|
| 8 |
+
o n
|
| 9 |
+
Ġ a
|
| 10 |
+
I n
|
| 11 |
+
S u
|
| 12 |
+
a n
|
| 13 |
+
a t
|
| 14 |
+
a y
|
| 15 |
+
a te
|
| 16 |
+
b s
|
| 17 |
+
b u
|
| 18 |
+
c h
|
| 19 |
+
c o
|
| 20 |
+
c t
|
| 21 |
+
d y
|
| 22 |
+
e d
|
| 23 |
+
e n
|
| 24 |
+
e s
|
| 25 |
+
e y
|
| 26 |
+
e ct
|
| 27 |
+
f o
|
| 28 |
+
g s
|
| 29 |
+
g ate
|
| 30 |
+
h i
|
| 31 |
+
h in
|
| 32 |
+
i on
|
| 33 |
+
i dy
|
| 34 |
+
i en
|
| 35 |
+
j o
|
| 36 |
+
j ect
|
| 37 |
+
k ed
|
| 38 |
+
l in
|
| 39 |
+
m on
|
| 40 |
+
m an
|
| 41 |
+
n t
|
| 42 |
+
n at
|
| 43 |
+
o ject
|
| 44 |
+
p r
|
| 45 |
+
r y
|
| 46 |
+
r ien
|
| 47 |
+
r lin
|
| 48 |
+
s in
|
| 49 |
+
s gate
|
| 50 |
+
t hin
|
| 51 |
+
u e
|
| 52 |
+
u nt
|
| 53 |
+
v al
|
| 54 |
+
w ay
|
| 55 |
+
w hi
|
| 56 |
+
Ġ in
|
| 57 |
+
Ġ bu
|
| 58 |
+
Ġ co
|
| 59 |
+
Ġ fo
|
| 60 |
+
Ġ jo
|
| 61 |
+
Ġ mon
|
| 62 |
+
Ġ nat
|
| 63 |
+
Ġ pr
|
| 64 |
+
Ġ thin
|
| 65 |
+
Ġ val
|
| 66 |
+
Ġ whi
|
| 67 |
+
in t
|
| 68 |
+
te d
|
| 69 |
+
te rlin
|
| 70 |
+
Ġo f
|
| 71 |
+
Ġo man
|
| 72 |
+
Ġo rien
|
| 73 |
+
al s
|
| 74 |
+
Su bs
|
| 75 |
+
es sgate
|
| 76 |
+
ion als
|
| 77 |
+
sin essgate
|
| 78 |
+
unt ry
|
| 79 |
+
way s
|
| 80 |
+
Ġin terlin
|
| 81 |
+
Ġbu sinessgate
|
| 82 |
+
Ġco untry
|
| 83 |
+
Ġfo r
|
| 84 |
+
Ġjo int
|
| 85 |
+
Ġmon ey
|
| 86 |
+
Ġnat ionals
|
| 87 |
+
Ġpr oject
|
| 88 |
+
Ġthin gs
|
| 89 |
+
Ġval ue
|
| 90 |
+
Ġwhi ch
|
| 91 |
+
Ġorien ted
|
| 92 |
+
Subs idy
|
| 93 |
+
Ġinterlin ked
|
| 94 |
+
Ġbusinessgate ways
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": "<|endoftext|>",
|
| 3 |
+
"eos_token": "<|endoftext|>",
|
| 4 |
+
"unk_token": "<|endoftext|>"
|
| 5 |
+
}
|
tokenizer.json
ADDED
|
@@ -0,0 +1,491 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": "1.0",
|
| 3 |
+
"truncation": null,
|
| 4 |
+
"padding": null,
|
| 5 |
+
"added_tokens": [
|
| 6 |
+
{
|
| 7 |
+
"id": 0,
|
| 8 |
+
"content": "<|endoftext|>",
|
| 9 |
+
"single_word": false,
|
| 10 |
+
"lstrip": false,
|
| 11 |
+
"rstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"special": true
|
| 14 |
+
}
|
| 15 |
+
],
|
| 16 |
+
"normalizer": null,
|
| 17 |
+
"pre_tokenizer": {
|
| 18 |
+
"type": "ByteLevel",
|
| 19 |
+
"add_prefix_space": false,
|
| 20 |
+
"trim_offsets": true,
|
| 21 |
+
"use_regex": true
|
| 22 |
+
},
|
| 23 |
+
"post_processor": {
|
| 24 |
+
"type": "ByteLevel",
|
| 25 |
+
"add_prefix_space": true,
|
| 26 |
+
"trim_offsets": false,
|
| 27 |
+
"use_regex": true
|
| 28 |
+
},
|
| 29 |
+
"decoder": {
|
| 30 |
+
"type": "ByteLevel",
|
| 31 |
+
"add_prefix_space": true,
|
| 32 |
+
"trim_offsets": true,
|
| 33 |
+
"use_regex": true
|
| 34 |
+
},
|
| 35 |
+
"model": {
|
| 36 |
+
"type": "BPE",
|
| 37 |
+
"dropout": null,
|
| 38 |
+
"unk_token": null,
|
| 39 |
+
"continuing_subword_prefix": "",
|
| 40 |
+
"end_of_word_suffix": "",
|
| 41 |
+
"fuse_unk": false,
|
| 42 |
+
"byte_fallback": false,
|
| 43 |
+
"vocab": {
|
| 44 |
+
"<|endoftext|>": 0,
|
| 45 |
+
"!": 1,
|
| 46 |
+
"\"": 2,
|
| 47 |
+
"#": 3,
|
| 48 |
+
"$": 4,
|
| 49 |
+
"%": 5,
|
| 50 |
+
"&": 6,
|
| 51 |
+
"'": 7,
|
| 52 |
+
"(": 8,
|
| 53 |
+
")": 9,
|
| 54 |
+
"*": 10,
|
| 55 |
+
"+": 11,
|
| 56 |
+
",": 12,
|
| 57 |
+
"-": 13,
|
| 58 |
+
".": 14,
|
| 59 |
+
"/": 15,
|
| 60 |
+
"0": 16,
|
| 61 |
+
"1": 17,
|
| 62 |
+
"2": 18,
|
| 63 |
+
"3": 19,
|
| 64 |
+
"4": 20,
|
| 65 |
+
"5": 21,
|
| 66 |
+
"6": 22,
|
| 67 |
+
"7": 23,
|
| 68 |
+
"8": 24,
|
| 69 |
+
"9": 25,
|
| 70 |
+
":": 26,
|
| 71 |
+
";": 27,
|
| 72 |
+
"<": 28,
|
| 73 |
+
"=": 29,
|
| 74 |
+
">": 30,
|
| 75 |
+
"?": 31,
|
| 76 |
+
"@": 32,
|
| 77 |
+
"A": 33,
|
| 78 |
+
"B": 34,
|
| 79 |
+
"C": 35,
|
| 80 |
+
"D": 36,
|
| 81 |
+
"E": 37,
|
| 82 |
+
"F": 38,
|
| 83 |
+
"G": 39,
|
| 84 |
+
"H": 40,
|
| 85 |
+
"I": 41,
|
| 86 |
+
"J": 42,
|
| 87 |
+
"K": 43,
|
| 88 |
+
"L": 44,
|
| 89 |
+
"M": 45,
|
| 90 |
+
"N": 46,
|
| 91 |
+
"O": 47,
|
| 92 |
+
"P": 48,
|
| 93 |
+
"Q": 49,
|
| 94 |
+
"R": 50,
|
| 95 |
+
"S": 51,
|
| 96 |
+
"T": 52,
|
| 97 |
+
"U": 53,
|
| 98 |
+
"V": 54,
|
| 99 |
+
"W": 55,
|
| 100 |
+
"X": 56,
|
| 101 |
+
"Y": 57,
|
| 102 |
+
"Z": 58,
|
| 103 |
+
"[": 59,
|
| 104 |
+
"\\": 60,
|
| 105 |
+
"]": 61,
|
| 106 |
+
"^": 62,
|
| 107 |
+
"_": 63,
|
| 108 |
+
"`": 64,
|
| 109 |
+
"a": 65,
|
| 110 |
+
"b": 66,
|
| 111 |
+
"c": 67,
|
| 112 |
+
"d": 68,
|
| 113 |
+
"e": 69,
|
| 114 |
+
"f": 70,
|
| 115 |
+
"g": 71,
|
| 116 |
+
"h": 72,
|
| 117 |
+
"i": 73,
|
| 118 |
+
"j": 74,
|
| 119 |
+
"k": 75,
|
| 120 |
+
"l": 76,
|
| 121 |
+
"m": 77,
|
| 122 |
+
"n": 78,
|
| 123 |
+
"o": 79,
|
| 124 |
+
"p": 80,
|
| 125 |
+
"q": 81,
|
| 126 |
+
"r": 82,
|
| 127 |
+
"s": 83,
|
| 128 |
+
"t": 84,
|
| 129 |
+
"u": 85,
|
| 130 |
+
"v": 86,
|
| 131 |
+
"w": 87,
|
| 132 |
+
"x": 88,
|
| 133 |
+
"y": 89,
|
| 134 |
+
"z": 90,
|
| 135 |
+
"{": 91,
|
| 136 |
+
"|": 92,
|
| 137 |
+
"}": 93,
|
| 138 |
+
"~": 94,
|
| 139 |
+
"¡": 95,
|
| 140 |
+
"¢": 96,
|
| 141 |
+
"£": 97,
|
| 142 |
+
"¤": 98,
|
| 143 |
+
"¥": 99,
|
| 144 |
+
"¦": 100,
|
| 145 |
+
"§": 101,
|
| 146 |
+
"¨": 102,
|
| 147 |
+
"©": 103,
|
| 148 |
+
"ª": 104,
|
| 149 |
+
"«": 105,
|
| 150 |
+
"¬": 106,
|
| 151 |
+
"®": 107,
|
| 152 |
+
"¯": 108,
|
| 153 |
+
"°": 109,
|
| 154 |
+
"±": 110,
|
| 155 |
+
"²": 111,
|
| 156 |
+
"³": 112,
|
| 157 |
+
"´": 113,
|
| 158 |
+
"µ": 114,
|
| 159 |
+
"¶": 115,
|
| 160 |
+
"·": 116,
|
| 161 |
+
"¸": 117,
|
| 162 |
+
"¹": 118,
|
| 163 |
+
"º": 119,
|
| 164 |
+
"»": 120,
|
| 165 |
+
"¼": 121,
|
| 166 |
+
"½": 122,
|
| 167 |
+
"¾": 123,
|
| 168 |
+
"¿": 124,
|
| 169 |
+
"À": 125,
|
| 170 |
+
"Á": 126,
|
| 171 |
+
"Â": 127,
|
| 172 |
+
"Ã": 128,
|
| 173 |
+
"Ä": 129,
|
| 174 |
+
"Å": 130,
|
| 175 |
+
"Æ": 131,
|
| 176 |
+
"Ç": 132,
|
| 177 |
+
"È": 133,
|
| 178 |
+
"É": 134,
|
| 179 |
+
"Ê": 135,
|
| 180 |
+
"Ë": 136,
|
| 181 |
+
"Ì": 137,
|
| 182 |
+
"Í": 138,
|
| 183 |
+
"Î": 139,
|
| 184 |
+
"Ï": 140,
|
| 185 |
+
"Ð": 141,
|
| 186 |
+
"Ñ": 142,
|
| 187 |
+
"Ò": 143,
|
| 188 |
+
"Ó": 144,
|
| 189 |
+
"Ô": 145,
|
| 190 |
+
"Õ": 146,
|
| 191 |
+
"Ö": 147,
|
| 192 |
+
"×": 148,
|
| 193 |
+
"Ø": 149,
|
| 194 |
+
"Ù": 150,
|
| 195 |
+
"Ú": 151,
|
| 196 |
+
"Û": 152,
|
| 197 |
+
"Ü": 153,
|
| 198 |
+
"Ý": 154,
|
| 199 |
+
"Þ": 155,
|
| 200 |
+
"ß": 156,
|
| 201 |
+
"à": 157,
|
| 202 |
+
"á": 158,
|
| 203 |
+
"â": 159,
|
| 204 |
+
"ã": 160,
|
| 205 |
+
"ä": 161,
|
| 206 |
+
"å": 162,
|
| 207 |
+
"æ": 163,
|
| 208 |
+
"ç": 164,
|
| 209 |
+
"è": 165,
|
| 210 |
+
"é": 166,
|
| 211 |
+
"ê": 167,
|
| 212 |
+
"ë": 168,
|
| 213 |
+
"ì": 169,
|
| 214 |
+
"í": 170,
|
| 215 |
+
"î": 171,
|
| 216 |
+
"ï": 172,
|
| 217 |
+
"ð": 173,
|
| 218 |
+
"ñ": 174,
|
| 219 |
+
"ò": 175,
|
| 220 |
+
"ó": 176,
|
| 221 |
+
"ô": 177,
|
| 222 |
+
"õ": 178,
|
| 223 |
+
"ö": 179,
|
| 224 |
+
"÷": 180,
|
| 225 |
+
"ø": 181,
|
| 226 |
+
"ù": 182,
|
| 227 |
+
"ú": 183,
|
| 228 |
+
"û": 184,
|
| 229 |
+
"ü": 185,
|
| 230 |
+
"ý": 186,
|
| 231 |
+
"þ": 187,
|
| 232 |
+
"ÿ": 188,
|
| 233 |
+
"Ā": 189,
|
| 234 |
+
"ā": 190,
|
| 235 |
+
"Ă": 191,
|
| 236 |
+
"ă": 192,
|
| 237 |
+
"Ą": 193,
|
| 238 |
+
"ą": 194,
|
| 239 |
+
"Ć": 195,
|
| 240 |
+
"ć": 196,
|
| 241 |
+
"Ĉ": 197,
|
| 242 |
+
"ĉ": 198,
|
| 243 |
+
"Ċ": 199,
|
| 244 |
+
"ċ": 200,
|
| 245 |
+
"Č": 201,
|
| 246 |
+
"č": 202,
|
| 247 |
+
"Ď": 203,
|
| 248 |
+
"ď": 204,
|
| 249 |
+
"Đ": 205,
|
| 250 |
+
"đ": 206,
|
| 251 |
+
"Ē": 207,
|
| 252 |
+
"ē": 208,
|
| 253 |
+
"Ĕ": 209,
|
| 254 |
+
"ĕ": 210,
|
| 255 |
+
"Ė": 211,
|
| 256 |
+
"ė": 212,
|
| 257 |
+
"Ę": 213,
|
| 258 |
+
"ę": 214,
|
| 259 |
+
"Ě": 215,
|
| 260 |
+
"ě": 216,
|
| 261 |
+
"Ĝ": 217,
|
| 262 |
+
"ĝ": 218,
|
| 263 |
+
"Ğ": 219,
|
| 264 |
+
"ğ": 220,
|
| 265 |
+
"Ġ": 221,
|
| 266 |
+
"ġ": 222,
|
| 267 |
+
"Ģ": 223,
|
| 268 |
+
"ģ": 224,
|
| 269 |
+
"Ĥ": 225,
|
| 270 |
+
"ĥ": 226,
|
| 271 |
+
"Ħ": 227,
|
| 272 |
+
"ħ": 228,
|
| 273 |
+
"Ĩ": 229,
|
| 274 |
+
"ĩ": 230,
|
| 275 |
+
"Ī": 231,
|
| 276 |
+
"ī": 232,
|
| 277 |
+
"Ĭ": 233,
|
| 278 |
+
"ĭ": 234,
|
| 279 |
+
"Į": 235,
|
| 280 |
+
"į": 236,
|
| 281 |
+
"İ": 237,
|
| 282 |
+
"ı": 238,
|
| 283 |
+
"IJ": 239,
|
| 284 |
+
"ij": 240,
|
| 285 |
+
"Ĵ": 241,
|
| 286 |
+
"ĵ": 242,
|
| 287 |
+
"Ķ": 243,
|
| 288 |
+
"ķ": 244,
|
| 289 |
+
"ĸ": 245,
|
| 290 |
+
"Ĺ": 246,
|
| 291 |
+
"ĺ": 247,
|
| 292 |
+
"Ļ": 248,
|
| 293 |
+
"ļ": 249,
|
| 294 |
+
"Ľ": 250,
|
| 295 |
+
"ľ": 251,
|
| 296 |
+
"Ŀ": 252,
|
| 297 |
+
"ŀ": 253,
|
| 298 |
+
"Ł": 254,
|
| 299 |
+
"ł": 255,
|
| 300 |
+
"Ń": 256,
|
| 301 |
+
"in": 257,
|
| 302 |
+
"is": 258,
|
| 303 |
+
"te": 259,
|
| 304 |
+
"Ġo": 260,
|
| 305 |
+
"Ġis": 261,
|
| 306 |
+
"al": 262,
|
| 307 |
+
"on": 263,
|
| 308 |
+
"Ġa": 264,
|
| 309 |
+
"In": 265,
|
| 310 |
+
"Su": 266,
|
| 311 |
+
"an": 267,
|
| 312 |
+
"at": 268,
|
| 313 |
+
"ay": 269,
|
| 314 |
+
"ate": 270,
|
| 315 |
+
"bs": 271,
|
| 316 |
+
"bu": 272,
|
| 317 |
+
"ch": 273,
|
| 318 |
+
"co": 274,
|
| 319 |
+
"ct": 275,
|
| 320 |
+
"dy": 276,
|
| 321 |
+
"ed": 277,
|
| 322 |
+
"en": 278,
|
| 323 |
+
"es": 279,
|
| 324 |
+
"ey": 280,
|
| 325 |
+
"ect": 281,
|
| 326 |
+
"fo": 282,
|
| 327 |
+
"gs": 283,
|
| 328 |
+
"gate": 284,
|
| 329 |
+
"hi": 285,
|
| 330 |
+
"hin": 286,
|
| 331 |
+
"ion": 287,
|
| 332 |
+
"idy": 288,
|
| 333 |
+
"ien": 289,
|
| 334 |
+
"jo": 290,
|
| 335 |
+
"ject": 291,
|
| 336 |
+
"ked": 292,
|
| 337 |
+
"lin": 293,
|
| 338 |
+
"mon": 294,
|
| 339 |
+
"man": 295,
|
| 340 |
+
"nt": 296,
|
| 341 |
+
"nat": 297,
|
| 342 |
+
"oject": 298,
|
| 343 |
+
"pr": 299,
|
| 344 |
+
"ry": 300,
|
| 345 |
+
"rien": 301,
|
| 346 |
+
"rlin": 302,
|
| 347 |
+
"sin": 303,
|
| 348 |
+
"sgate": 304,
|
| 349 |
+
"thin": 305,
|
| 350 |
+
"ue": 306,
|
| 351 |
+
"unt": 307,
|
| 352 |
+
"val": 308,
|
| 353 |
+
"way": 309,
|
| 354 |
+
"whi": 310,
|
| 355 |
+
"Ġin": 311,
|
| 356 |
+
"Ġbu": 312,
|
| 357 |
+
"Ġco": 313,
|
| 358 |
+
"Ġfo": 314,
|
| 359 |
+
"Ġjo": 315,
|
| 360 |
+
"Ġmon": 316,
|
| 361 |
+
"Ġnat": 317,
|
| 362 |
+
"Ġpr": 318,
|
| 363 |
+
"Ġthin": 319,
|
| 364 |
+
"Ġval": 320,
|
| 365 |
+
"Ġwhi": 321,
|
| 366 |
+
"int": 322,
|
| 367 |
+
"ted": 323,
|
| 368 |
+
"terlin": 324,
|
| 369 |
+
"Ġof": 325,
|
| 370 |
+
"Ġoman": 326,
|
| 371 |
+
"Ġorien": 327,
|
| 372 |
+
"als": 328,
|
| 373 |
+
"Subs": 329,
|
| 374 |
+
"essgate": 330,
|
| 375 |
+
"ionals": 331,
|
| 376 |
+
"sinessgate": 332,
|
| 377 |
+
"untry": 333,
|
| 378 |
+
"ways": 334,
|
| 379 |
+
"Ġinterlin": 335,
|
| 380 |
+
"Ġbusinessgate": 336,
|
| 381 |
+
"Ġcountry": 337,
|
| 382 |
+
"Ġfor": 338,
|
| 383 |
+
"Ġjoint": 339,
|
| 384 |
+
"Ġmoney": 340,
|
| 385 |
+
"Ġnationals": 341,
|
| 386 |
+
"Ġproject": 342,
|
| 387 |
+
"Ġthings": 343,
|
| 388 |
+
"Ġvalue": 344,
|
| 389 |
+
"Ġwhich": 345,
|
| 390 |
+
"Ġoriented": 346,
|
| 391 |
+
"Subsidy": 347,
|
| 392 |
+
"Ġinterlinked": 348,
|
| 393 |
+
"Ġbusinessgateways": 349
|
| 394 |
+
},
|
| 395 |
+
"merges": [
|
| 396 |
+
"i n",
|
| 397 |
+
"i s",
|
| 398 |
+
"t e",
|
| 399 |
+
"Ġ o",
|
| 400 |
+
"Ġ is",
|
| 401 |
+
"a l",
|
| 402 |
+
"o n",
|
| 403 |
+
"Ġ a",
|
| 404 |
+
"I n",
|
| 405 |
+
"S u",
|
| 406 |
+
"a n",
|
| 407 |
+
"a t",
|
| 408 |
+
"a y",
|
| 409 |
+
"a te",
|
| 410 |
+
"b s",
|
| 411 |
+
"b u",
|
| 412 |
+
"c h",
|
| 413 |
+
"c o",
|
| 414 |
+
"c t",
|
| 415 |
+
"d y",
|
| 416 |
+
"e d",
|
| 417 |
+
"e n",
|
| 418 |
+
"e s",
|
| 419 |
+
"e y",
|
| 420 |
+
"e ct",
|
| 421 |
+
"f o",
|
| 422 |
+
"g s",
|
| 423 |
+
"g ate",
|
| 424 |
+
"h i",
|
| 425 |
+
"h in",
|
| 426 |
+
"i on",
|
| 427 |
+
"i dy",
|
| 428 |
+
"i en",
|
| 429 |
+
"j o",
|
| 430 |
+
"j ect",
|
| 431 |
+
"k ed",
|
| 432 |
+
"l in",
|
| 433 |
+
"m on",
|
| 434 |
+
"m an",
|
| 435 |
+
"n t",
|
| 436 |
+
"n at",
|
| 437 |
+
"o ject",
|
| 438 |
+
"p r",
|
| 439 |
+
"r y",
|
| 440 |
+
"r ien",
|
| 441 |
+
"r lin",
|
| 442 |
+
"s in",
|
| 443 |
+
"s gate",
|
| 444 |
+
"t hin",
|
| 445 |
+
"u e",
|
| 446 |
+
"u nt",
|
| 447 |
+
"v al",
|
| 448 |
+
"w ay",
|
| 449 |
+
"w hi",
|
| 450 |
+
"Ġ in",
|
| 451 |
+
"Ġ bu",
|
| 452 |
+
"Ġ co",
|
| 453 |
+
"Ġ fo",
|
| 454 |
+
"Ġ jo",
|
| 455 |
+
"Ġ mon",
|
| 456 |
+
"Ġ nat",
|
| 457 |
+
"Ġ pr",
|
| 458 |
+
"Ġ thin",
|
| 459 |
+
"Ġ val",
|
| 460 |
+
"Ġ whi",
|
| 461 |
+
"in t",
|
| 462 |
+
"te d",
|
| 463 |
+
"te rlin",
|
| 464 |
+
"Ġo f",
|
| 465 |
+
"Ġo man",
|
| 466 |
+
"Ġo rien",
|
| 467 |
+
"al s",
|
| 468 |
+
"Su bs",
|
| 469 |
+
"es sgate",
|
| 470 |
+
"ion als",
|
| 471 |
+
"sin essgate",
|
| 472 |
+
"unt ry",
|
| 473 |
+
"way s",
|
| 474 |
+
"Ġin terlin",
|
| 475 |
+
"Ġbu sinessgate",
|
| 476 |
+
"Ġco untry",
|
| 477 |
+
"Ġfo r",
|
| 478 |
+
"Ġjo int",
|
| 479 |
+
"Ġmon ey",
|
| 480 |
+
"Ġnat ionals",
|
| 481 |
+
"Ġpr oject",
|
| 482 |
+
"Ġthin gs",
|
| 483 |
+
"Ġval ue",
|
| 484 |
+
"Ġwhi ch",
|
| 485 |
+
"Ġorien ted",
|
| 486 |
+
"Subs idy",
|
| 487 |
+
"Ġinterlin ked",
|
| 488 |
+
"Ġbusinessgate ways"
|
| 489 |
+
]
|
| 490 |
+
}
|
| 491 |
+
}
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"bos_token": "<|endoftext|>",
|
| 4 |
+
"eos_token": "<|endoftext|>",
|
| 5 |
+
"model_max_length": 1024,
|
| 6 |
+
"special_tokens_map_file": null,
|
| 7 |
+
"tokenizer_class": "GPT2Tokenizer",
|
| 8 |
+
"unk_token": "<|endoftext|>"
|
| 9 |
+
}
|
vocab.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"<|endoftext|>":0,"!":1,"\"":2,"#":3,"$":4,"%":5,"&":6,"'":7,"(":8,")":9,"*":10,"+":11,",":12,"-":13,".":14,"/":15,"0":16,"1":17,"2":18,"3":19,"4":20,"5":21,"6":22,"7":23,"8":24,"9":25,":":26,";":27,"<":28,"=":29,">":30,"?":31,"@":32,"A":33,"B":34,"C":35,"D":36,"E":37,"F":38,"G":39,"H":40,"I":41,"J":42,"K":43,"L":44,"M":45,"N":46,"O":47,"P":48,"Q":49,"R":50,"S":51,"T":52,"U":53,"V":54,"W":55,"X":56,"Y":57,"Z":58,"[":59,"\\":60,"]":61,"^":62,"_":63,"`":64,"a":65,"b":66,"c":67,"d":68,"e":69,"f":70,"g":71,"h":72,"i":73,"j":74,"k":75,"l":76,"m":77,"n":78,"o":79,"p":80,"q":81,"r":82,"s":83,"t":84,"u":85,"v":86,"w":87,"x":88,"y":89,"z":90,"{":91,"|":92,"}":93,"~":94,"¡":95,"¢":96,"£":97,"¤":98,"¥":99,"¦":100,"§":101,"¨":102,"©":103,"ª":104,"«":105,"¬":106,"®":107,"¯":108,"°":109,"±":110,"²":111,"³":112,"´":113,"µ":114,"¶":115,"·":116,"¸":117,"¹":118,"º":119,"»":120,"¼":121,"½":122,"¾":123,"¿":124,"À":125,"Á":126,"Â":127,"Ã":128,"Ä":129,"Å":130,"Æ":131,"Ç":132,"È":133,"É":134,"Ê":135,"Ë":136,"Ì":137,"Í":138,"Î":139,"Ï":140,"Ð":141,"Ñ":142,"Ò":143,"Ó":144,"Ô":145,"Õ":146,"Ö":147,"×":148,"Ø":149,"Ù":150,"Ú":151,"Û":152,"Ü":153,"Ý":154,"Þ":155,"ß":156,"à":157,"á":158,"â":159,"ã":160,"ä":161,"å":162,"æ":163,"ç":164,"è":165,"é":166,"ê":167,"ë":168,"ì":169,"í":170,"î":171,"ï":172,"ð":173,"ñ":174,"ò":175,"ó":176,"ô":177,"õ":178,"ö":179,"÷":180,"ø":181,"ù":182,"ú":183,"û":184,"ü":185,"ý":186,"þ":187,"ÿ":188,"Ā":189,"ā":190,"Ă":191,"ă":192,"Ą":193,"ą":194,"Ć":195,"ć":196,"Ĉ":197,"ĉ":198,"Ċ":199,"ċ":200,"Č":201,"č":202,"Ď":203,"ď":204,"Đ":205,"đ":206,"Ē":207,"ē":208,"Ĕ":209,"ĕ":210,"Ė":211,"ė":212,"Ę":213,"ę":214,"Ě":215,"ě":216,"Ĝ":217,"ĝ":218,"Ğ":219,"ğ":220,"Ġ":221,"ġ":222,"Ģ":223,"ģ":224,"Ĥ":225,"ĥ":226,"Ħ":227,"ħ":228,"Ĩ":229,"ĩ":230,"Ī":231,"ī":232,"Ĭ":233,"ĭ":234,"Į":235,"į":236,"İ":237,"ı":238,"IJ":239,"ij":240,"Ĵ":241,"ĵ":242,"Ķ":243,"ķ":244,"ĸ":245,"Ĺ":246,"ĺ":247,"Ļ":248,"ļ":249,"Ľ":250,"ľ":251,"Ŀ":252,"ŀ":253,"Ł":254,"ł":255,"Ń":256,"in":257,"is":258,"te":259,"Ġo":260,"Ġis":261,"al":262,"on":263,"Ġa":264,"In":265,"Su":266,"an":267,"at":268,"ay":269,"ate":270,"bs":271,"bu":272,"ch":273,"co":274,"ct":275,"dy":276,"ed":277,"en":278,"es":279,"ey":280,"ect":281,"fo":282,"gs":283,"gate":284,"hi":285,"hin":286,"ion":287,"idy":288,"ien":289,"jo":290,"ject":291,"ked":292,"lin":293,"mon":294,"man":295,"nt":296,"nat":297,"oject":298,"pr":299,"ry":300,"rien":301,"rlin":302,"sin":303,"sgate":304,"thin":305,"ue":306,"unt":307,"val":308,"way":309,"whi":310,"Ġin":311,"Ġbu":312,"Ġco":313,"Ġfo":314,"Ġjo":315,"Ġmon":316,"Ġnat":317,"Ġpr":318,"Ġthin":319,"Ġval":320,"Ġwhi":321,"int":322,"ted":323,"terlin":324,"Ġof":325,"Ġoman":326,"Ġorien":327,"als":328,"Subs":329,"essgate":330,"ionals":331,"sinessgate":332,"untry":333,"ways":334,"Ġinterlin":335,"Ġbusinessgate":336,"Ġcountry":337,"Ġfor":338,"Ġjoint":339,"Ġmoney":340,"Ġnationals":341,"Ġproject":342,"Ġthings":343,"Ġvalue":344,"Ġwhich":345,"Ġoriented":346,"Subsidy":347,"Ġinterlinked":348,"Ġbusinessgateways":349}
|