Commit
•
f44a10f
1
Parent(s):
4753d6b
Upload tokenizer
Browse files- dict.txt +0 -0
- fairseq_dictionary.py +504 -0
- guoke_tokenizer.py +114 -0
- sentencepiece_bpe.py +62 -0
- sp.model +3 -0
- special_tokens_map.json +8 -0
- tokenization_fairseq_t5.py +162 -0
- tokenizer_config.json +20 -0
dict.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|
fairseq_dictionary.py
ADDED
@@ -0,0 +1,504 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
2 |
+
#
|
3 |
+
# This source code is ported from fairseq:
|
4 |
+
# https://github.com/facebookresearch/fairseq
|
5 |
+
#
|
6 |
+
# This source code is licensed under the MIT license found in the
|
7 |
+
# LICENSE file in the root directory of the fairseq repo
|
8 |
+
|
9 |
+
|
10 |
+
import os
|
11 |
+
import re
|
12 |
+
from collections import Counter
|
13 |
+
from multiprocessing import Pool
|
14 |
+
from typing import Iterable, List
|
15 |
+
|
16 |
+
import torch
|
17 |
+
|
18 |
+
|
19 |
+
def item(tensor):
|
20 |
+
# tpu-comment: making this a no-op for xla devices.
|
21 |
+
if torch.is_tensor(tensor) and tensor.device.type == "xla":
|
22 |
+
return tensor.detach()
|
23 |
+
if hasattr(tensor, "item"):
|
24 |
+
return tensor.item()
|
25 |
+
if hasattr(tensor, "__getitem__"):
|
26 |
+
return tensor[0]
|
27 |
+
return tensor
|
28 |
+
|
29 |
+
|
30 |
+
def post_process(sentence: str, symbol: str):
|
31 |
+
if symbol == "sentencepiece":
|
32 |
+
sentence = sentence.replace(" ", "").replace("\u2581", " ").strip()
|
33 |
+
elif symbol == "wordpiece":
|
34 |
+
sentence = sentence.replace(" ", "").replace("_", " ").strip()
|
35 |
+
elif symbol == "letter":
|
36 |
+
sentence = sentence.replace(" ", "").replace("|", " ").strip()
|
37 |
+
elif symbol == "silence":
|
38 |
+
import re
|
39 |
+
|
40 |
+
sentence = sentence.replace("<SIL>", "")
|
41 |
+
sentence = re.sub(" +", " ", sentence).strip()
|
42 |
+
elif symbol == "_EOW":
|
43 |
+
sentence = sentence.replace(" ", "").replace("_EOW", " ").strip()
|
44 |
+
elif symbol in {"subword_nmt", "@@ ", "@@"}:
|
45 |
+
if symbol == "subword_nmt":
|
46 |
+
symbol = "@@ "
|
47 |
+
sentence = (sentence + " ").replace(symbol, "").rstrip()
|
48 |
+
elif symbol == "none":
|
49 |
+
pass
|
50 |
+
elif symbol is not None:
|
51 |
+
raise NotImplementedError(f"Unknown post_process option: {symbol}")
|
52 |
+
return sentence
|
53 |
+
|
54 |
+
|
55 |
+
SPACE_NORMALIZER = re.compile(r"\s+")
|
56 |
+
|
57 |
+
|
58 |
+
def tokenize_line(line):
|
59 |
+
line = SPACE_NORMALIZER.sub(" ", line)
|
60 |
+
line = line.strip()
|
61 |
+
return line.split()
|
62 |
+
|
63 |
+
|
64 |
+
def _safe_readline(fd) -> str:
|
65 |
+
pos = fd.tell()
|
66 |
+
while True:
|
67 |
+
try:
|
68 |
+
return fd.readline()
|
69 |
+
except UnicodeDecodeError:
|
70 |
+
pos -= 1
|
71 |
+
fd.seek(pos) # search where this character begins
|
72 |
+
|
73 |
+
|
74 |
+
def find_offsets(filename: str, num_chunks: int) -> List[int]:
|
75 |
+
"""
|
76 |
+
given a file and a number of chuncks, find the offsets in the file
|
77 |
+
to be able to chunk around full lines.
|
78 |
+
"""
|
79 |
+
with open(filename, "r", encoding="utf-8") as f:
|
80 |
+
size = os.fstat(f.fileno()).st_size
|
81 |
+
chunk_size = size // num_chunks
|
82 |
+
offsets = [0 for _ in range(num_chunks + 1)]
|
83 |
+
for i in range(1, num_chunks):
|
84 |
+
f.seek(chunk_size * i)
|
85 |
+
_safe_readline(f)
|
86 |
+
offsets[i] = f.tell()
|
87 |
+
offsets[-1] = size
|
88 |
+
return offsets
|
89 |
+
|
90 |
+
|
91 |
+
class ChunkLineIterator:
|
92 |
+
"""
|
93 |
+
Iterator to properly iterate over lines of a file chunck.
|
94 |
+
"""
|
95 |
+
|
96 |
+
def __init__(self, fd, start_offset: int, end_offset: int):
|
97 |
+
self._fd = fd
|
98 |
+
self._start_offset = start_offset
|
99 |
+
self._end_offset = end_offset
|
100 |
+
|
101 |
+
def __iter__(self) -> Iterable[str]:
|
102 |
+
self._fd.seek(self._start_offset)
|
103 |
+
# next(f) breaks f.tell(), hence readline() must be used
|
104 |
+
line = _safe_readline(self._fd)
|
105 |
+
while line:
|
106 |
+
pos = self._fd.tell()
|
107 |
+
# f.tell() does not always give the byte position in the file
|
108 |
+
# sometimes it skips to a very large number
|
109 |
+
# it is unlikely that through a normal read we go from
|
110 |
+
# end bytes to end + 2**32 bytes (4 GB) and this makes it unlikely
|
111 |
+
# that the procedure breaks by the undeterministic behavior of
|
112 |
+
# f.tell()
|
113 |
+
if (
|
114 |
+
self._end_offset > 0
|
115 |
+
and pos > self._end_offset
|
116 |
+
and pos < self._end_offset + 2 ** 32
|
117 |
+
):
|
118 |
+
break
|
119 |
+
yield line
|
120 |
+
line = self._fd.readline()
|
121 |
+
|
122 |
+
|
123 |
+
class Chunker:
|
124 |
+
"""
|
125 |
+
contextmanager to read a chunck of a file line by line.
|
126 |
+
"""
|
127 |
+
|
128 |
+
def __init__(self, path: str, start_offset: int, end_offset: int):
|
129 |
+
self.path = path
|
130 |
+
self.start_offset = start_offset
|
131 |
+
self.end_offset = end_offset
|
132 |
+
|
133 |
+
def __enter__(self) -> ChunkLineIterator:
|
134 |
+
self.fd = open(self.path, "r", encoding="utf-8")
|
135 |
+
return ChunkLineIterator(self.fd, self.start_offset, self.end_offset)
|
136 |
+
|
137 |
+
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
138 |
+
self.fd.close()
|
139 |
+
|
140 |
+
|
141 |
+
class Dictionary:
|
142 |
+
"""A mapping from symbols to consecutive integers"""
|
143 |
+
|
144 |
+
def __init__(
|
145 |
+
self,
|
146 |
+
*, # begin keyword-only arguments
|
147 |
+
bos="<s>",
|
148 |
+
pad="<pad>",
|
149 |
+
eos="</s>",
|
150 |
+
unk="<unk>",
|
151 |
+
extra_special_symbols=None,
|
152 |
+
):
|
153 |
+
self.bos_word, self.unk_word, self.pad_word, self.eos_word = bos, unk, pad, eos
|
154 |
+
self.symbols = []
|
155 |
+
self.count = []
|
156 |
+
self.indices = {}
|
157 |
+
self.bos_index = self.add_symbol(bos)
|
158 |
+
self.pad_index = self.add_symbol(pad)
|
159 |
+
self.eos_index = self.add_symbol(eos)
|
160 |
+
self.unk_index = self.add_symbol(unk)
|
161 |
+
if extra_special_symbols:
|
162 |
+
for s in extra_special_symbols:
|
163 |
+
self.add_symbol(s)
|
164 |
+
self.nspecial = len(self.symbols)
|
165 |
+
|
166 |
+
def __eq__(self, other):
|
167 |
+
return self.indices == other.indices
|
168 |
+
|
169 |
+
def __getitem__(self, idx):
|
170 |
+
if idx < len(self.symbols):
|
171 |
+
return self.symbols[idx]
|
172 |
+
return self.unk_word
|
173 |
+
|
174 |
+
def get_count(self, idx):
|
175 |
+
return self.count[idx]
|
176 |
+
|
177 |
+
def __len__(self):
|
178 |
+
"""Returns the number of symbols in the dictionary"""
|
179 |
+
return len(self.symbols)
|
180 |
+
|
181 |
+
def __contains__(self, sym):
|
182 |
+
return sym in self.indices
|
183 |
+
|
184 |
+
def index(self, sym):
|
185 |
+
"""Returns the index of the specified symbol"""
|
186 |
+
assert isinstance(sym, str)
|
187 |
+
if sym in self.indices:
|
188 |
+
return self.indices[sym]
|
189 |
+
return self.unk_index
|
190 |
+
|
191 |
+
def string(
|
192 |
+
self,
|
193 |
+
tensor,
|
194 |
+
bpe_symbol=None,
|
195 |
+
escape_unk=False,
|
196 |
+
extra_symbols_to_ignore=None,
|
197 |
+
unk_string=None,
|
198 |
+
include_eos=False,
|
199 |
+
separator=" ",
|
200 |
+
):
|
201 |
+
"""Helper for converting a tensor of token indices to a string.
|
202 |
+
|
203 |
+
Can optionally remove BPE symbols or escape <unk> words.
|
204 |
+
"""
|
205 |
+
if torch.is_tensor(tensor) and tensor.dim() == 2:
|
206 |
+
return "\n".join(
|
207 |
+
self.string(
|
208 |
+
t,
|
209 |
+
bpe_symbol,
|
210 |
+
escape_unk,
|
211 |
+
extra_symbols_to_ignore,
|
212 |
+
include_eos=include_eos,
|
213 |
+
)
|
214 |
+
for t in tensor
|
215 |
+
)
|
216 |
+
|
217 |
+
extra_symbols_to_ignore = set(extra_symbols_to_ignore or [])
|
218 |
+
if not include_eos:
|
219 |
+
extra_symbols_to_ignore.add(self.eos())
|
220 |
+
|
221 |
+
def token_string(i):
|
222 |
+
if i == self.unk():
|
223 |
+
if unk_string is not None:
|
224 |
+
return unk_string
|
225 |
+
else:
|
226 |
+
return self.unk_string(escape_unk)
|
227 |
+
else:
|
228 |
+
return self[i]
|
229 |
+
|
230 |
+
if hasattr(self, "bos_index"):
|
231 |
+
extra_symbols_to_ignore.add(self.bos())
|
232 |
+
|
233 |
+
sent = separator.join(
|
234 |
+
token_string(i)
|
235 |
+
for i in tensor
|
236 |
+
if item(i) not in extra_symbols_to_ignore
|
237 |
+
)
|
238 |
+
|
239 |
+
return post_process(sent, bpe_symbol)
|
240 |
+
|
241 |
+
def unk_string(self, escape=False):
|
242 |
+
"""Return unknown string, optionally escaped as: <<unk>>"""
|
243 |
+
if escape:
|
244 |
+
return "<{}>".format(self.unk_word)
|
245 |
+
else:
|
246 |
+
return self.unk_word
|
247 |
+
|
248 |
+
def add_symbol(self, word, n=1, overwrite=False):
|
249 |
+
"""Adds a word to the dictionary"""
|
250 |
+
if word in self.indices and not overwrite:
|
251 |
+
idx = self.indices[word]
|
252 |
+
self.count[idx] = self.count[idx] + n
|
253 |
+
return idx
|
254 |
+
else:
|
255 |
+
idx = len(self.symbols)
|
256 |
+
self.indices[word] = idx
|
257 |
+
self.symbols.append(word)
|
258 |
+
self.count.append(n)
|
259 |
+
return idx
|
260 |
+
|
261 |
+
def update(self, new_dict):
|
262 |
+
"""Updates counts from new dictionary."""
|
263 |
+
for word in new_dict.symbols:
|
264 |
+
idx2 = new_dict.indices[word]
|
265 |
+
if word in self.indices:
|
266 |
+
idx = self.indices[word]
|
267 |
+
self.count[idx] = self.count[idx] + new_dict.count[idx2]
|
268 |
+
else:
|
269 |
+
idx = len(self.symbols)
|
270 |
+
self.indices[word] = idx
|
271 |
+
self.symbols.append(word)
|
272 |
+
self.count.append(new_dict.count[idx2])
|
273 |
+
|
274 |
+
def finalize(self, threshold=-1, nwords=-1, padding_factor=8):
|
275 |
+
"""Sort symbols by frequency in descending order, ignoring special ones.
|
276 |
+
|
277 |
+
Args:
|
278 |
+
- threshold defines the minimum word count
|
279 |
+
- nwords defines the total number of words in the final dictionary,
|
280 |
+
including special symbols
|
281 |
+
- padding_factor can be used to pad the dictionary size to be a
|
282 |
+
multiple of 8, which is important on some hardware (e.g., Nvidia
|
283 |
+
Tensor Cores).
|
284 |
+
"""
|
285 |
+
if nwords <= 0:
|
286 |
+
nwords = len(self)
|
287 |
+
|
288 |
+
new_indices = dict(zip(self.symbols[: self.nspecial], range(self.nspecial)))
|
289 |
+
new_symbols = self.symbols[: self.nspecial]
|
290 |
+
new_count = self.count[: self.nspecial]
|
291 |
+
|
292 |
+
c = Counter(
|
293 |
+
dict(
|
294 |
+
sorted(zip(self.symbols[self.nspecial:], self.count[self.nspecial:]))
|
295 |
+
)
|
296 |
+
)
|
297 |
+
for symbol, count in c.most_common(nwords - self.nspecial):
|
298 |
+
if count >= threshold:
|
299 |
+
new_indices[symbol] = len(new_symbols)
|
300 |
+
new_symbols.append(symbol)
|
301 |
+
new_count.append(count)
|
302 |
+
else:
|
303 |
+
break
|
304 |
+
|
305 |
+
assert len(new_symbols) == len(new_indices)
|
306 |
+
|
307 |
+
self.count = list(new_count)
|
308 |
+
self.symbols = list(new_symbols)
|
309 |
+
self.indices = new_indices
|
310 |
+
|
311 |
+
self.pad_to_multiple_(padding_factor)
|
312 |
+
|
313 |
+
def pad_to_multiple_(self, padding_factor):
|
314 |
+
"""Pad Dictionary size to be a multiple of *padding_factor*."""
|
315 |
+
if padding_factor > 1:
|
316 |
+
i = 0
|
317 |
+
while len(self) % padding_factor != 0:
|
318 |
+
symbol = "madeupword{:04d}".format(i)
|
319 |
+
self.add_symbol(symbol, n=0)
|
320 |
+
i += 1
|
321 |
+
|
322 |
+
def bos(self):
|
323 |
+
"""Helper to get index of beginning-of-sentence symbol"""
|
324 |
+
return self.bos_index
|
325 |
+
|
326 |
+
def pad(self):
|
327 |
+
"""Helper to get index of pad symbol"""
|
328 |
+
return self.pad_index
|
329 |
+
|
330 |
+
def eos(self):
|
331 |
+
"""Helper to get index of end-of-sentence symbol"""
|
332 |
+
return self.eos_index
|
333 |
+
|
334 |
+
def unk(self):
|
335 |
+
"""Helper to get index of unk symbol"""
|
336 |
+
return self.unk_index
|
337 |
+
|
338 |
+
@classmethod
|
339 |
+
def load(cls, f):
|
340 |
+
"""Loads the dictionary from a text file with the format:
|
341 |
+
|
342 |
+
```
|
343 |
+
<symbol0> <count0>
|
344 |
+
<symbol1> <count1>
|
345 |
+
...
|
346 |
+
```
|
347 |
+
"""
|
348 |
+
d = cls()
|
349 |
+
d.add_from_file(f)
|
350 |
+
return d
|
351 |
+
|
352 |
+
def add_from_file(self, f):
|
353 |
+
"""
|
354 |
+
Loads a pre-existing dictionary from a text file and adds its symbols
|
355 |
+
to this instance.
|
356 |
+
"""
|
357 |
+
if isinstance(f, str):
|
358 |
+
try:
|
359 |
+
with open(f, "r", encoding="utf-8") as fd:
|
360 |
+
self.add_from_file(fd)
|
361 |
+
except FileNotFoundError as fnfe:
|
362 |
+
raise fnfe
|
363 |
+
except UnicodeError:
|
364 |
+
raise Exception(
|
365 |
+
"Incorrect encoding detected in {}, please "
|
366 |
+
"rebuild the dataset".format(f)
|
367 |
+
)
|
368 |
+
return
|
369 |
+
|
370 |
+
lines = f.readlines()
|
371 |
+
indices_start_line = self._load_meta(lines)
|
372 |
+
|
373 |
+
for line in lines[indices_start_line:]:
|
374 |
+
try:
|
375 |
+
line, field = line.rstrip().rsplit(" ", 1)
|
376 |
+
if field == "#fairseq:overwrite":
|
377 |
+
overwrite = True
|
378 |
+
line, field = line.rsplit(" ", 1)
|
379 |
+
else:
|
380 |
+
overwrite = False
|
381 |
+
count = int(field)
|
382 |
+
word = line
|
383 |
+
if word in self and not overwrite:
|
384 |
+
raise RuntimeError(
|
385 |
+
"Duplicate word found when loading Dictionary: '{}'. "
|
386 |
+
"Duplicate words can overwrite earlier ones by adding the "
|
387 |
+
"#fairseq:overwrite flag at the end of the corresponding row "
|
388 |
+
"in the dictionary file. If using the Camembert model, please "
|
389 |
+
"download an updated copy of the model file.".format(word)
|
390 |
+
)
|
391 |
+
self.add_symbol(word, n=count, overwrite=overwrite)
|
392 |
+
except ValueError:
|
393 |
+
raise ValueError(
|
394 |
+
f"Incorrect dictionary format, expected '<token> <cnt> [flags]': \"{line}\""
|
395 |
+
)
|
396 |
+
|
397 |
+
def _save(self, f, kv_iterator):
|
398 |
+
if isinstance(f, str):
|
399 |
+
os.makedirs(os.path.dirname(f), exist_ok=True)
|
400 |
+
with open(f, "w", encoding="utf-8") as fd:
|
401 |
+
return self.save(fd)
|
402 |
+
for k, v in kv_iterator:
|
403 |
+
print("{} {}".format(k, v), file=f)
|
404 |
+
|
405 |
+
def _get_meta(self):
|
406 |
+
return [], []
|
407 |
+
|
408 |
+
def _load_meta(self, lines):
|
409 |
+
return 0
|
410 |
+
|
411 |
+
def save(self, f):
|
412 |
+
"""Stores dictionary into a text file"""
|
413 |
+
ex_keys, ex_vals = self._get_meta()
|
414 |
+
self._save(
|
415 |
+
f,
|
416 |
+
zip(
|
417 |
+
ex_keys + self.symbols[self.nspecial:],
|
418 |
+
ex_vals + self.count[self.nspecial:],
|
419 |
+
),
|
420 |
+
)
|
421 |
+
|
422 |
+
def dummy_sentence(self, length):
|
423 |
+
t = torch.Tensor(length).uniform_(self.nspecial + 1, len(self)).long()
|
424 |
+
t[-1] = self.eos()
|
425 |
+
return t
|
426 |
+
|
427 |
+
def encode_line(
|
428 |
+
self,
|
429 |
+
line,
|
430 |
+
line_tokenizer=tokenize_line,
|
431 |
+
add_if_not_exist=True,
|
432 |
+
consumer=None,
|
433 |
+
append_eos=True,
|
434 |
+
reverse_order=False,
|
435 |
+
) -> torch.IntTensor:
|
436 |
+
words = line_tokenizer(line)
|
437 |
+
if reverse_order:
|
438 |
+
words = list(reversed(words))
|
439 |
+
nwords = len(words)
|
440 |
+
ids = torch.IntTensor(nwords + 1 if append_eos else nwords)
|
441 |
+
|
442 |
+
for i, word in enumerate(words):
|
443 |
+
if add_if_not_exist:
|
444 |
+
idx = self.add_symbol(word)
|
445 |
+
else:
|
446 |
+
idx = self.index(word)
|
447 |
+
if consumer is not None:
|
448 |
+
consumer(word, idx)
|
449 |
+
ids[i] = idx
|
450 |
+
if append_eos:
|
451 |
+
ids[nwords] = self.eos_index
|
452 |
+
return ids
|
453 |
+
|
454 |
+
@staticmethod
|
455 |
+
def _add_file_to_dictionary_single_worker(
|
456 |
+
filename,
|
457 |
+
tokenize,
|
458 |
+
eos_word,
|
459 |
+
start_offset,
|
460 |
+
end_offset,
|
461 |
+
):
|
462 |
+
counter = Counter()
|
463 |
+
with Chunker(filename, start_offset, end_offset) as line_iterator:
|
464 |
+
for line in line_iterator:
|
465 |
+
for word in tokenize(line):
|
466 |
+
counter.update([word])
|
467 |
+
counter.update([eos_word])
|
468 |
+
return counter
|
469 |
+
|
470 |
+
@staticmethod
|
471 |
+
def add_file_to_dictionary(filename, dict, tokenize, num_workers):
|
472 |
+
def merge_result(counter):
|
473 |
+
for w, c in sorted(counter.items()):
|
474 |
+
dict.add_symbol(w, c)
|
475 |
+
|
476 |
+
local_file = filename
|
477 |
+
offsets = find_offsets(local_file, num_workers)
|
478 |
+
if num_workers > 1:
|
479 |
+
chunks = zip(offsets, offsets[1:])
|
480 |
+
pool = Pool(processes=num_workers)
|
481 |
+
results = []
|
482 |
+
for (start_offset, end_offset) in chunks:
|
483 |
+
results.append(
|
484 |
+
pool.apply_async(
|
485 |
+
Dictionary._add_file_to_dictionary_single_worker,
|
486 |
+
(
|
487 |
+
local_file,
|
488 |
+
tokenize,
|
489 |
+
dict.eos_word,
|
490 |
+
start_offset,
|
491 |
+
end_offset,
|
492 |
+
),
|
493 |
+
)
|
494 |
+
)
|
495 |
+
pool.close()
|
496 |
+
pool.join()
|
497 |
+
for r in results:
|
498 |
+
merge_result(r.get())
|
499 |
+
else:
|
500 |
+
merge_result(
|
501 |
+
Dictionary._add_file_to_dictionary_single_worker(
|
502 |
+
local_file, tokenize, dict.eos_word, offsets[0], offsets[1]
|
503 |
+
)
|
504 |
+
)
|
guoke_tokenizer.py
ADDED
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import re
|
2 |
+
|
3 |
+
import unicodedata
|
4 |
+
|
5 |
+
TRANS_TABLE = dict([(ord(x), ord(y)) for x, y in zip(u"‘’´“”—–-", u"'''\"\"---")])
|
6 |
+
|
7 |
+
|
8 |
+
def _is_punctuation(char):
|
9 |
+
"""Checks whether `chars` is a punctuation character."""
|
10 |
+
cp = ord(char)
|
11 |
+
# We treat all non-letter/number ASCII as punctuation.
|
12 |
+
# Characters such as "^", "$", and "`" are not in the Unicode
|
13 |
+
# Punctuation class but we treat them as punctuation anyways, for
|
14 |
+
# consistency.
|
15 |
+
if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126):
|
16 |
+
return True
|
17 |
+
cat = unicodedata.category(char)
|
18 |
+
if cat.startswith("P"):
|
19 |
+
return True
|
20 |
+
return False
|
21 |
+
|
22 |
+
|
23 |
+
def _handle_single_quote(tokens):
|
24 |
+
line = ' '.join(tokens)
|
25 |
+
line = re.sub(r"' ([smdSMDtT])\b", r"'\1", line)
|
26 |
+
line = re.sub(r"' ll\b", "'ll", line)
|
27 |
+
line = re.sub(r"' re\b", "'re", line)
|
28 |
+
line = re.sub(r"' ve\b", "'ve", line)
|
29 |
+
line = re.sub(r"' LL\b", "'LL ", line)
|
30 |
+
line = re.sub(r"' RE\b", "'RE ", line)
|
31 |
+
line = re.sub(r"' VE\b", "'VE ", line)
|
32 |
+
return line.split()
|
33 |
+
|
34 |
+
|
35 |
+
def _split_on_cont_punc(tokens):
|
36 |
+
new_tokens = []
|
37 |
+
for token in tokens:
|
38 |
+
if len(token) > 1:
|
39 |
+
last_j = 0
|
40 |
+
pre_is_punc = _is_punctuation(token[0])
|
41 |
+
for j, ch in enumerate(token):
|
42 |
+
is_punc = _is_punctuation(ch)
|
43 |
+
if is_punc != pre_is_punc:
|
44 |
+
new_tokens.append(token[last_j: j])
|
45 |
+
last_j = j
|
46 |
+
pre_is_punc = is_punc
|
47 |
+
if last_j < len(token):
|
48 |
+
new_tokens.append(token[last_j:])
|
49 |
+
else:
|
50 |
+
new_tokens.append(token)
|
51 |
+
return new_tokens
|
52 |
+
|
53 |
+
|
54 |
+
def _split_pre_and_post_punc(tokens):
|
55 |
+
def pre_punc(token):
|
56 |
+
last_j = 0
|
57 |
+
for j in range(1, len(token)):
|
58 |
+
if not _is_punctuation(token[j]):
|
59 |
+
last_j = j
|
60 |
+
break
|
61 |
+
return token[:last_j], token[last_j:]
|
62 |
+
|
63 |
+
def post_punc(token):
|
64 |
+
last_j = len(token)
|
65 |
+
for j in range(len(token) - 2, -1, -1):
|
66 |
+
if not _is_punctuation(token[j]):
|
67 |
+
last_j = j + 1
|
68 |
+
break
|
69 |
+
return token[:last_j], token[last_j:]
|
70 |
+
|
71 |
+
new_tokens = []
|
72 |
+
for token in tokens:
|
73 |
+
if len(token) > 1 and _is_punctuation(token[0]):
|
74 |
+
a, b = pre_punc(token)
|
75 |
+
if a:
|
76 |
+
new_tokens.append(a)
|
77 |
+
if b:
|
78 |
+
if _is_punctuation(b[-1]):
|
79 |
+
c, d = post_punc(b)
|
80 |
+
if c:
|
81 |
+
new_tokens.append(c)
|
82 |
+
if d:
|
83 |
+
new_tokens.append(d)
|
84 |
+
else:
|
85 |
+
new_tokens.append(b)
|
86 |
+
elif len(token) > 1 and _is_punctuation(token[-1]):
|
87 |
+
a, b = post_punc(token)
|
88 |
+
if a:
|
89 |
+
new_tokens.append(a)
|
90 |
+
if b:
|
91 |
+
new_tokens.append(b)
|
92 |
+
else:
|
93 |
+
new_tokens.append(token)
|
94 |
+
return new_tokens
|
95 |
+
|
96 |
+
|
97 |
+
class GuokeTokenizer(object):
|
98 |
+
def __init__(self, cfg):
|
99 |
+
self.cfg = cfg
|
100 |
+
|
101 |
+
def encode(self, x: str) -> str:
|
102 |
+
x = x.strip()
|
103 |
+
x = x.replace("``", '"').replace("''", '"')
|
104 |
+
x = x.translate(TRANS_TABLE)
|
105 |
+
tokens = x.split()
|
106 |
+
tokens = _split_pre_and_post_punc(tokens)
|
107 |
+
tokens = _handle_single_quote(tokens)
|
108 |
+
x = " ".join(tokens)
|
109 |
+
if self.cfg.lower:
|
110 |
+
x = x.lower()
|
111 |
+
return x
|
112 |
+
|
113 |
+
def decode(self, x: str) -> str:
|
114 |
+
raise NotImplementedError()
|
sentencepiece_bpe.py
ADDED
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
2 |
+
#
|
3 |
+
# This source code is ported from fairseq:
|
4 |
+
# https://github.com/facebookresearch/fairseq
|
5 |
+
#
|
6 |
+
# This source code is licensed under the MIT license found in the
|
7 |
+
# LICENSE file in the root directory of the fairseq repo
|
8 |
+
|
9 |
+
from dataclasses import dataclass, field
|
10 |
+
from typing import Optional
|
11 |
+
|
12 |
+
|
13 |
+
@dataclass
|
14 |
+
class SentencepieceConfig:
|
15 |
+
sentencepiece_model: str = field(
|
16 |
+
default="???", metadata={"help": "path to sentencepiece model"}
|
17 |
+
)
|
18 |
+
sentencepiece_enable_sampling: bool = field(
|
19 |
+
default=False, metadata={"help": "enable sampling"}
|
20 |
+
)
|
21 |
+
sentencepiece_alpha: Optional[float] = field(
|
22 |
+
default=None, metadata={
|
23 |
+
"help": "soothing parameter for unigram sampling, "
|
24 |
+
"and merge probability for BPE-dropout"
|
25 |
+
}
|
26 |
+
)
|
27 |
+
|
28 |
+
|
29 |
+
class SentencepieceBPE(object):
|
30 |
+
def __init__(self, cfg):
|
31 |
+
self.enable_sampling = cfg.sentencepiece_enable_sampling
|
32 |
+
self.alpha = cfg.sentencepiece_alpha
|
33 |
+
sentencepiece_model = cfg.sentencepiece_model
|
34 |
+
try:
|
35 |
+
import sentencepiece as spm
|
36 |
+
|
37 |
+
self.sp = spm.SentencePieceProcessor()
|
38 |
+
self.sp.Load(sentencepiece_model)
|
39 |
+
except ImportError:
|
40 |
+
raise ImportError(
|
41 |
+
"Please install sentencepiece with: pip install sentencepiece"
|
42 |
+
)
|
43 |
+
|
44 |
+
def encode(self, x: str) -> str:
|
45 |
+
return " ".join(
|
46 |
+
self.sp.Encode(
|
47 |
+
x, out_type=str, enable_sampling=self.enable_sampling, alpha=self.alpha
|
48 |
+
)
|
49 |
+
)
|
50 |
+
|
51 |
+
def decode(self, x: str) -> str:
|
52 |
+
return x.replace(" ", "").replace("\u2581", " ").strip()
|
53 |
+
|
54 |
+
def is_beginning_of_word(self, x: str) -> bool:
|
55 |
+
if x in ["<unk>", "<s>", "</s>", "<pad>"]:
|
56 |
+
# special elements are always considered beginnings
|
57 |
+
# HACK: this logic is already present in fairseq/tasks/masked_lm.py
|
58 |
+
# but these special tokens are also contained in the sentencepiece
|
59 |
+
# vocabulary which causes duplicate special tokens. This hack makes
|
60 |
+
# sure that they are all taken into account.
|
61 |
+
return True
|
62 |
+
return x.startswith("\u2581")
|
sp.model
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3b2f031ad52ac175466dc380dbd6878873b7d45f20921dbe54c5e9a009decdb6
|
3 |
+
size 1358163
|
special_tokens_map.json
ADDED
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token": "<s>",
|
3 |
+
"cls_token": "<s>",
|
4 |
+
"eos_token": "</s>",
|
5 |
+
"pad_token": "<pad>",
|
6 |
+
"sep_token": "</s>",
|
7 |
+
"unk_token": "<unk>"
|
8 |
+
}
|
tokenization_fairseq_t5.py
ADDED
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
from shutil import copyfile
|
3 |
+
from typing import List, Optional
|
4 |
+
|
5 |
+
from omegaconf import DictConfig
|
6 |
+
from transformers.tokenization_utils import PreTrainedTokenizer
|
7 |
+
from transformers.utils import logging
|
8 |
+
|
9 |
+
from .fairseq_dictionary import Dictionary
|
10 |
+
from .guoke_tokenizer import GuokeTokenizer
|
11 |
+
from .sentencepiece_bpe import SentencepieceBPE
|
12 |
+
|
13 |
+
logger = logging.get_logger(__name__)
|
14 |
+
|
15 |
+
VOCAB_FILES_NAMES = {
|
16 |
+
"sp_path": "sp.model",
|
17 |
+
"dict_path": "dict.txt"
|
18 |
+
}
|
19 |
+
|
20 |
+
|
21 |
+
class FairseqT5Tokenizer(PreTrainedTokenizer):
|
22 |
+
vocab_files_names = VOCAB_FILES_NAMES
|
23 |
+
model_input_names = ["input_ids", "attention_mask"]
|
24 |
+
|
25 |
+
def __init__(
|
26 |
+
self,
|
27 |
+
sp_path,
|
28 |
+
dict_path,
|
29 |
+
lower,
|
30 |
+
n_sentinel_tokens=0,
|
31 |
+
bos_token="<s>",
|
32 |
+
eos_token="</s>",
|
33 |
+
unk_token="<unk>",
|
34 |
+
pad_token="<pad>",
|
35 |
+
**kwargs
|
36 |
+
) -> None:
|
37 |
+
|
38 |
+
self.sp_path = sp_path
|
39 |
+
self.dict_path = dict_path
|
40 |
+
self.lower = lower
|
41 |
+
|
42 |
+
self.fs_tokenizer = GuokeTokenizer(
|
43 |
+
DictConfig(
|
44 |
+
dict(
|
45 |
+
lower=lower
|
46 |
+
)
|
47 |
+
)
|
48 |
+
)
|
49 |
+
self.fs_bpe = SentencepieceBPE(
|
50 |
+
DictConfig(
|
51 |
+
dict(
|
52 |
+
sentencepiece_model=sp_path,
|
53 |
+
)
|
54 |
+
)
|
55 |
+
)
|
56 |
+
self.fs_dict = Dictionary.load(dict_path)
|
57 |
+
for i in range(n_sentinel_tokens):
|
58 |
+
self.fs_dict.add_symbol(f'<sen{i:03d}>')
|
59 |
+
|
60 |
+
if "sep_token" in kwargs:
|
61 |
+
assert kwargs["sep_token"] == eos_token
|
62 |
+
kwargs.pop("sep_token")
|
63 |
+
if "cls_token" in kwargs:
|
64 |
+
assert kwargs["cls_token"] == bos_token
|
65 |
+
kwargs.pop("cls_token")
|
66 |
+
|
67 |
+
super().__init__(
|
68 |
+
bos_token=bos_token,
|
69 |
+
eos_token=eos_token,
|
70 |
+
unk_token=unk_token,
|
71 |
+
pad_token=pad_token,
|
72 |
+
sep_token=eos_token,
|
73 |
+
cls_token=bos_token,
|
74 |
+
lower=self.lower,
|
75 |
+
n_sentinel_tokens=n_sentinel_tokens,
|
76 |
+
**kwargs,
|
77 |
+
)
|
78 |
+
|
79 |
+
@property
|
80 |
+
def vocab_size(self):
|
81 |
+
return len(self.fs_dict)
|
82 |
+
|
83 |
+
def get_vocab(self):
|
84 |
+
return self.fs_dict.indices
|
85 |
+
|
86 |
+
def get_special_tokens_mask(
|
87 |
+
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
|
88 |
+
) -> List[int]:
|
89 |
+
"""
|
90 |
+
Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
|
91 |
+
special tokens using the tokenizer `prepare_for_model` method.
|
92 |
+
|
93 |
+
Args:
|
94 |
+
token_ids_0 (`List[int]`):
|
95 |
+
List of IDs.
|
96 |
+
token_ids_1 (`List[int]`, *optional*):
|
97 |
+
Optional second list of IDs for sequence pairs.
|
98 |
+
already_has_special_tokens (`bool`, *optional*, defaults to `False`):
|
99 |
+
Whether or not the token list is already formatted with special tokens for the model.
|
100 |
+
|
101 |
+
Returns:
|
102 |
+
`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
|
103 |
+
"""
|
104 |
+
if already_has_special_tokens:
|
105 |
+
return super().get_special_tokens_mask(
|
106 |
+
token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
|
107 |
+
)
|
108 |
+
|
109 |
+
if token_ids_1 is None:
|
110 |
+
return [1] + ([0] * len(token_ids_0)) + [1]
|
111 |
+
return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]
|
112 |
+
|
113 |
+
def create_token_type_ids_from_sequences(
|
114 |
+
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
|
115 |
+
) -> List[int]:
|
116 |
+
sep = [self.sep_token_id]
|
117 |
+
cls = [self.cls_token_id]
|
118 |
+
|
119 |
+
if token_ids_1 is None:
|
120 |
+
return len(cls + token_ids_0 + sep) * [0]
|
121 |
+
return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]
|
122 |
+
|
123 |
+
def build_inputs_with_special_tokens(
|
124 |
+
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
|
125 |
+
) -> List[int]:
|
126 |
+
if token_ids_1 is None:
|
127 |
+
return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
|
128 |
+
cls = [self.cls_token_id]
|
129 |
+
sep = [self.sep_token_id]
|
130 |
+
return cls + token_ids_0 + sep + sep + token_ids_1 + sep
|
131 |
+
|
132 |
+
def _tokenize(self, text: str) -> List[str]:
|
133 |
+
return self.fs_bpe.encode(self.fs_tokenizer.encode(text)).split(" ")
|
134 |
+
|
135 |
+
def _convert_token_to_id(self, token):
|
136 |
+
return self.fs_dict.index(token)
|
137 |
+
|
138 |
+
def _convert_id_to_token(self, index):
|
139 |
+
return self.fs_dict[index]
|
140 |
+
|
141 |
+
def convert_tokens_to_string(self, tokens):
|
142 |
+
return self.fs_bpe.decode(" ".join(tokens))
|
143 |
+
|
144 |
+
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
|
145 |
+
if not os.path.isdir(save_directory):
|
146 |
+
logger.error(f"Vocabulary path ({save_directory}) should be a directory")
|
147 |
+
return
|
148 |
+
out_sp_path = os.path.join(
|
149 |
+
save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["sp_path"]
|
150 |
+
)
|
151 |
+
out_dict_path = os.path.join(
|
152 |
+
save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["dict_path"]
|
153 |
+
)
|
154 |
+
|
155 |
+
if os.path.abspath(self.sp_path) != os.path.abspath(out_sp_path):
|
156 |
+
copyfile(self.sp_path, out_sp_path)
|
157 |
+
logger.info(f"Copy from {self.sp_path} to {out_sp_path}")
|
158 |
+
if os.path.abspath(self.dict_path) != os.path.abspath(out_dict_path):
|
159 |
+
copyfile(self.dict_path, out_dict_path)
|
160 |
+
logger.info(f"Copy from {self.dict_path} to {out_dict_path}")
|
161 |
+
|
162 |
+
return out_sp_path, out_dict_path
|
tokenizer_config.json
ADDED
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"auto_map": {
|
3 |
+
"AutoTokenizer": [
|
4 |
+
"tokenization_fairseq_t5.FairseqT5Tokenizer",
|
5 |
+
null
|
6 |
+
]
|
7 |
+
},
|
8 |
+
"bos_token": "<s>",
|
9 |
+
"clean_up_tokenization_spaces": true,
|
10 |
+
"cls_token": "<s>",
|
11 |
+
"eos_token": "</s>",
|
12 |
+
"lower": true,
|
13 |
+
"model_max_length": 1000000000000000019884624838656,
|
14 |
+
"n_sentinel_tokens": 512,
|
15 |
+
"pad_token": "<pad>",
|
16 |
+
"sep_token": "</s>",
|
17 |
+
"tokenizer_class": "FairseqT5Tokenizer",
|
18 |
+
"unk_token": "<unk>",
|
19 |
+
"use_fast": false
|
20 |
+
}
|