gonglinyuan commited on
Commit
f44a10f
1 Parent(s): 4753d6b

Upload tokenizer

Browse files
dict.txt ADDED
The diff for this file is too large to render. See raw diff
 
fairseq_dictionary.py ADDED
@@ -0,0 +1,504 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is ported from fairseq:
4
+ # https://github.com/facebookresearch/fairseq
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of the fairseq repo
8
+
9
+
10
+ import os
11
+ import re
12
+ from collections import Counter
13
+ from multiprocessing import Pool
14
+ from typing import Iterable, List
15
+
16
+ import torch
17
+
18
+
19
+ def item(tensor):
20
+ # tpu-comment: making this a no-op for xla devices.
21
+ if torch.is_tensor(tensor) and tensor.device.type == "xla":
22
+ return tensor.detach()
23
+ if hasattr(tensor, "item"):
24
+ return tensor.item()
25
+ if hasattr(tensor, "__getitem__"):
26
+ return tensor[0]
27
+ return tensor
28
+
29
+
30
+ def post_process(sentence: str, symbol: str):
31
+ if symbol == "sentencepiece":
32
+ sentence = sentence.replace(" ", "").replace("\u2581", " ").strip()
33
+ elif symbol == "wordpiece":
34
+ sentence = sentence.replace(" ", "").replace("_", " ").strip()
35
+ elif symbol == "letter":
36
+ sentence = sentence.replace(" ", "").replace("|", " ").strip()
37
+ elif symbol == "silence":
38
+ import re
39
+
40
+ sentence = sentence.replace("<SIL>", "")
41
+ sentence = re.sub(" +", " ", sentence).strip()
42
+ elif symbol == "_EOW":
43
+ sentence = sentence.replace(" ", "").replace("_EOW", " ").strip()
44
+ elif symbol in {"subword_nmt", "@@ ", "@@"}:
45
+ if symbol == "subword_nmt":
46
+ symbol = "@@ "
47
+ sentence = (sentence + " ").replace(symbol, "").rstrip()
48
+ elif symbol == "none":
49
+ pass
50
+ elif symbol is not None:
51
+ raise NotImplementedError(f"Unknown post_process option: {symbol}")
52
+ return sentence
53
+
54
+
55
+ SPACE_NORMALIZER = re.compile(r"\s+")
56
+
57
+
58
+ def tokenize_line(line):
59
+ line = SPACE_NORMALIZER.sub(" ", line)
60
+ line = line.strip()
61
+ return line.split()
62
+
63
+
64
+ def _safe_readline(fd) -> str:
65
+ pos = fd.tell()
66
+ while True:
67
+ try:
68
+ return fd.readline()
69
+ except UnicodeDecodeError:
70
+ pos -= 1
71
+ fd.seek(pos) # search where this character begins
72
+
73
+
74
+ def find_offsets(filename: str, num_chunks: int) -> List[int]:
75
+ """
76
+ given a file and a number of chuncks, find the offsets in the file
77
+ to be able to chunk around full lines.
78
+ """
79
+ with open(filename, "r", encoding="utf-8") as f:
80
+ size = os.fstat(f.fileno()).st_size
81
+ chunk_size = size // num_chunks
82
+ offsets = [0 for _ in range(num_chunks + 1)]
83
+ for i in range(1, num_chunks):
84
+ f.seek(chunk_size * i)
85
+ _safe_readline(f)
86
+ offsets[i] = f.tell()
87
+ offsets[-1] = size
88
+ return offsets
89
+
90
+
91
+ class ChunkLineIterator:
92
+ """
93
+ Iterator to properly iterate over lines of a file chunck.
94
+ """
95
+
96
+ def __init__(self, fd, start_offset: int, end_offset: int):
97
+ self._fd = fd
98
+ self._start_offset = start_offset
99
+ self._end_offset = end_offset
100
+
101
+ def __iter__(self) -> Iterable[str]:
102
+ self._fd.seek(self._start_offset)
103
+ # next(f) breaks f.tell(), hence readline() must be used
104
+ line = _safe_readline(self._fd)
105
+ while line:
106
+ pos = self._fd.tell()
107
+ # f.tell() does not always give the byte position in the file
108
+ # sometimes it skips to a very large number
109
+ # it is unlikely that through a normal read we go from
110
+ # end bytes to end + 2**32 bytes (4 GB) and this makes it unlikely
111
+ # that the procedure breaks by the undeterministic behavior of
112
+ # f.tell()
113
+ if (
114
+ self._end_offset > 0
115
+ and pos > self._end_offset
116
+ and pos < self._end_offset + 2 ** 32
117
+ ):
118
+ break
119
+ yield line
120
+ line = self._fd.readline()
121
+
122
+
123
+ class Chunker:
124
+ """
125
+ contextmanager to read a chunck of a file line by line.
126
+ """
127
+
128
+ def __init__(self, path: str, start_offset: int, end_offset: int):
129
+ self.path = path
130
+ self.start_offset = start_offset
131
+ self.end_offset = end_offset
132
+
133
+ def __enter__(self) -> ChunkLineIterator:
134
+ self.fd = open(self.path, "r", encoding="utf-8")
135
+ return ChunkLineIterator(self.fd, self.start_offset, self.end_offset)
136
+
137
+ def __exit__(self, exc_type, exc_val, exc_tb) -> None:
138
+ self.fd.close()
139
+
140
+
141
+ class Dictionary:
142
+ """A mapping from symbols to consecutive integers"""
143
+
144
+ def __init__(
145
+ self,
146
+ *, # begin keyword-only arguments
147
+ bos="<s>",
148
+ pad="<pad>",
149
+ eos="</s>",
150
+ unk="<unk>",
151
+ extra_special_symbols=None,
152
+ ):
153
+ self.bos_word, self.unk_word, self.pad_word, self.eos_word = bos, unk, pad, eos
154
+ self.symbols = []
155
+ self.count = []
156
+ self.indices = {}
157
+ self.bos_index = self.add_symbol(bos)
158
+ self.pad_index = self.add_symbol(pad)
159
+ self.eos_index = self.add_symbol(eos)
160
+ self.unk_index = self.add_symbol(unk)
161
+ if extra_special_symbols:
162
+ for s in extra_special_symbols:
163
+ self.add_symbol(s)
164
+ self.nspecial = len(self.symbols)
165
+
166
+ def __eq__(self, other):
167
+ return self.indices == other.indices
168
+
169
+ def __getitem__(self, idx):
170
+ if idx < len(self.symbols):
171
+ return self.symbols[idx]
172
+ return self.unk_word
173
+
174
+ def get_count(self, idx):
175
+ return self.count[idx]
176
+
177
+ def __len__(self):
178
+ """Returns the number of symbols in the dictionary"""
179
+ return len(self.symbols)
180
+
181
+ def __contains__(self, sym):
182
+ return sym in self.indices
183
+
184
+ def index(self, sym):
185
+ """Returns the index of the specified symbol"""
186
+ assert isinstance(sym, str)
187
+ if sym in self.indices:
188
+ return self.indices[sym]
189
+ return self.unk_index
190
+
191
+ def string(
192
+ self,
193
+ tensor,
194
+ bpe_symbol=None,
195
+ escape_unk=False,
196
+ extra_symbols_to_ignore=None,
197
+ unk_string=None,
198
+ include_eos=False,
199
+ separator=" ",
200
+ ):
201
+ """Helper for converting a tensor of token indices to a string.
202
+
203
+ Can optionally remove BPE symbols or escape <unk> words.
204
+ """
205
+ if torch.is_tensor(tensor) and tensor.dim() == 2:
206
+ return "\n".join(
207
+ self.string(
208
+ t,
209
+ bpe_symbol,
210
+ escape_unk,
211
+ extra_symbols_to_ignore,
212
+ include_eos=include_eos,
213
+ )
214
+ for t in tensor
215
+ )
216
+
217
+ extra_symbols_to_ignore = set(extra_symbols_to_ignore or [])
218
+ if not include_eos:
219
+ extra_symbols_to_ignore.add(self.eos())
220
+
221
+ def token_string(i):
222
+ if i == self.unk():
223
+ if unk_string is not None:
224
+ return unk_string
225
+ else:
226
+ return self.unk_string(escape_unk)
227
+ else:
228
+ return self[i]
229
+
230
+ if hasattr(self, "bos_index"):
231
+ extra_symbols_to_ignore.add(self.bos())
232
+
233
+ sent = separator.join(
234
+ token_string(i)
235
+ for i in tensor
236
+ if item(i) not in extra_symbols_to_ignore
237
+ )
238
+
239
+ return post_process(sent, bpe_symbol)
240
+
241
+ def unk_string(self, escape=False):
242
+ """Return unknown string, optionally escaped as: <<unk>>"""
243
+ if escape:
244
+ return "<{}>".format(self.unk_word)
245
+ else:
246
+ return self.unk_word
247
+
248
+ def add_symbol(self, word, n=1, overwrite=False):
249
+ """Adds a word to the dictionary"""
250
+ if word in self.indices and not overwrite:
251
+ idx = self.indices[word]
252
+ self.count[idx] = self.count[idx] + n
253
+ return idx
254
+ else:
255
+ idx = len(self.symbols)
256
+ self.indices[word] = idx
257
+ self.symbols.append(word)
258
+ self.count.append(n)
259
+ return idx
260
+
261
+ def update(self, new_dict):
262
+ """Updates counts from new dictionary."""
263
+ for word in new_dict.symbols:
264
+ idx2 = new_dict.indices[word]
265
+ if word in self.indices:
266
+ idx = self.indices[word]
267
+ self.count[idx] = self.count[idx] + new_dict.count[idx2]
268
+ else:
269
+ idx = len(self.symbols)
270
+ self.indices[word] = idx
271
+ self.symbols.append(word)
272
+ self.count.append(new_dict.count[idx2])
273
+
274
+ def finalize(self, threshold=-1, nwords=-1, padding_factor=8):
275
+ """Sort symbols by frequency in descending order, ignoring special ones.
276
+
277
+ Args:
278
+ - threshold defines the minimum word count
279
+ - nwords defines the total number of words in the final dictionary,
280
+ including special symbols
281
+ - padding_factor can be used to pad the dictionary size to be a
282
+ multiple of 8, which is important on some hardware (e.g., Nvidia
283
+ Tensor Cores).
284
+ """
285
+ if nwords <= 0:
286
+ nwords = len(self)
287
+
288
+ new_indices = dict(zip(self.symbols[: self.nspecial], range(self.nspecial)))
289
+ new_symbols = self.symbols[: self.nspecial]
290
+ new_count = self.count[: self.nspecial]
291
+
292
+ c = Counter(
293
+ dict(
294
+ sorted(zip(self.symbols[self.nspecial:], self.count[self.nspecial:]))
295
+ )
296
+ )
297
+ for symbol, count in c.most_common(nwords - self.nspecial):
298
+ if count >= threshold:
299
+ new_indices[symbol] = len(new_symbols)
300
+ new_symbols.append(symbol)
301
+ new_count.append(count)
302
+ else:
303
+ break
304
+
305
+ assert len(new_symbols) == len(new_indices)
306
+
307
+ self.count = list(new_count)
308
+ self.symbols = list(new_symbols)
309
+ self.indices = new_indices
310
+
311
+ self.pad_to_multiple_(padding_factor)
312
+
313
+ def pad_to_multiple_(self, padding_factor):
314
+ """Pad Dictionary size to be a multiple of *padding_factor*."""
315
+ if padding_factor > 1:
316
+ i = 0
317
+ while len(self) % padding_factor != 0:
318
+ symbol = "madeupword{:04d}".format(i)
319
+ self.add_symbol(symbol, n=0)
320
+ i += 1
321
+
322
+ def bos(self):
323
+ """Helper to get index of beginning-of-sentence symbol"""
324
+ return self.bos_index
325
+
326
+ def pad(self):
327
+ """Helper to get index of pad symbol"""
328
+ return self.pad_index
329
+
330
+ def eos(self):
331
+ """Helper to get index of end-of-sentence symbol"""
332
+ return self.eos_index
333
+
334
+ def unk(self):
335
+ """Helper to get index of unk symbol"""
336
+ return self.unk_index
337
+
338
+ @classmethod
339
+ def load(cls, f):
340
+ """Loads the dictionary from a text file with the format:
341
+
342
+ ```
343
+ <symbol0> <count0>
344
+ <symbol1> <count1>
345
+ ...
346
+ ```
347
+ """
348
+ d = cls()
349
+ d.add_from_file(f)
350
+ return d
351
+
352
+ def add_from_file(self, f):
353
+ """
354
+ Loads a pre-existing dictionary from a text file and adds its symbols
355
+ to this instance.
356
+ """
357
+ if isinstance(f, str):
358
+ try:
359
+ with open(f, "r", encoding="utf-8") as fd:
360
+ self.add_from_file(fd)
361
+ except FileNotFoundError as fnfe:
362
+ raise fnfe
363
+ except UnicodeError:
364
+ raise Exception(
365
+ "Incorrect encoding detected in {}, please "
366
+ "rebuild the dataset".format(f)
367
+ )
368
+ return
369
+
370
+ lines = f.readlines()
371
+ indices_start_line = self._load_meta(lines)
372
+
373
+ for line in lines[indices_start_line:]:
374
+ try:
375
+ line, field = line.rstrip().rsplit(" ", 1)
376
+ if field == "#fairseq:overwrite":
377
+ overwrite = True
378
+ line, field = line.rsplit(" ", 1)
379
+ else:
380
+ overwrite = False
381
+ count = int(field)
382
+ word = line
383
+ if word in self and not overwrite:
384
+ raise RuntimeError(
385
+ "Duplicate word found when loading Dictionary: '{}'. "
386
+ "Duplicate words can overwrite earlier ones by adding the "
387
+ "#fairseq:overwrite flag at the end of the corresponding row "
388
+ "in the dictionary file. If using the Camembert model, please "
389
+ "download an updated copy of the model file.".format(word)
390
+ )
391
+ self.add_symbol(word, n=count, overwrite=overwrite)
392
+ except ValueError:
393
+ raise ValueError(
394
+ f"Incorrect dictionary format, expected '<token> <cnt> [flags]': \"{line}\""
395
+ )
396
+
397
+ def _save(self, f, kv_iterator):
398
+ if isinstance(f, str):
399
+ os.makedirs(os.path.dirname(f), exist_ok=True)
400
+ with open(f, "w", encoding="utf-8") as fd:
401
+ return self.save(fd)
402
+ for k, v in kv_iterator:
403
+ print("{} {}".format(k, v), file=f)
404
+
405
+ def _get_meta(self):
406
+ return [], []
407
+
408
+ def _load_meta(self, lines):
409
+ return 0
410
+
411
+ def save(self, f):
412
+ """Stores dictionary into a text file"""
413
+ ex_keys, ex_vals = self._get_meta()
414
+ self._save(
415
+ f,
416
+ zip(
417
+ ex_keys + self.symbols[self.nspecial:],
418
+ ex_vals + self.count[self.nspecial:],
419
+ ),
420
+ )
421
+
422
+ def dummy_sentence(self, length):
423
+ t = torch.Tensor(length).uniform_(self.nspecial + 1, len(self)).long()
424
+ t[-1] = self.eos()
425
+ return t
426
+
427
+ def encode_line(
428
+ self,
429
+ line,
430
+ line_tokenizer=tokenize_line,
431
+ add_if_not_exist=True,
432
+ consumer=None,
433
+ append_eos=True,
434
+ reverse_order=False,
435
+ ) -> torch.IntTensor:
436
+ words = line_tokenizer(line)
437
+ if reverse_order:
438
+ words = list(reversed(words))
439
+ nwords = len(words)
440
+ ids = torch.IntTensor(nwords + 1 if append_eos else nwords)
441
+
442
+ for i, word in enumerate(words):
443
+ if add_if_not_exist:
444
+ idx = self.add_symbol(word)
445
+ else:
446
+ idx = self.index(word)
447
+ if consumer is not None:
448
+ consumer(word, idx)
449
+ ids[i] = idx
450
+ if append_eos:
451
+ ids[nwords] = self.eos_index
452
+ return ids
453
+
454
+ @staticmethod
455
+ def _add_file_to_dictionary_single_worker(
456
+ filename,
457
+ tokenize,
458
+ eos_word,
459
+ start_offset,
460
+ end_offset,
461
+ ):
462
+ counter = Counter()
463
+ with Chunker(filename, start_offset, end_offset) as line_iterator:
464
+ for line in line_iterator:
465
+ for word in tokenize(line):
466
+ counter.update([word])
467
+ counter.update([eos_word])
468
+ return counter
469
+
470
+ @staticmethod
471
+ def add_file_to_dictionary(filename, dict, tokenize, num_workers):
472
+ def merge_result(counter):
473
+ for w, c in sorted(counter.items()):
474
+ dict.add_symbol(w, c)
475
+
476
+ local_file = filename
477
+ offsets = find_offsets(local_file, num_workers)
478
+ if num_workers > 1:
479
+ chunks = zip(offsets, offsets[1:])
480
+ pool = Pool(processes=num_workers)
481
+ results = []
482
+ for (start_offset, end_offset) in chunks:
483
+ results.append(
484
+ pool.apply_async(
485
+ Dictionary._add_file_to_dictionary_single_worker,
486
+ (
487
+ local_file,
488
+ tokenize,
489
+ dict.eos_word,
490
+ start_offset,
491
+ end_offset,
492
+ ),
493
+ )
494
+ )
495
+ pool.close()
496
+ pool.join()
497
+ for r in results:
498
+ merge_result(r.get())
499
+ else:
500
+ merge_result(
501
+ Dictionary._add_file_to_dictionary_single_worker(
502
+ local_file, tokenize, dict.eos_word, offsets[0], offsets[1]
503
+ )
504
+ )
guoke_tokenizer.py ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+
3
+ import unicodedata
4
+
5
+ TRANS_TABLE = dict([(ord(x), ord(y)) for x, y in zip(u"‘’´“”—–-", u"'''\"\"---")])
6
+
7
+
8
+ def _is_punctuation(char):
9
+ """Checks whether `chars` is a punctuation character."""
10
+ cp = ord(char)
11
+ # We treat all non-letter/number ASCII as punctuation.
12
+ # Characters such as "^", "$", and "`" are not in the Unicode
13
+ # Punctuation class but we treat them as punctuation anyways, for
14
+ # consistency.
15
+ if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126):
16
+ return True
17
+ cat = unicodedata.category(char)
18
+ if cat.startswith("P"):
19
+ return True
20
+ return False
21
+
22
+
23
+ def _handle_single_quote(tokens):
24
+ line = ' '.join(tokens)
25
+ line = re.sub(r"' ([smdSMDtT])\b", r"'\1", line)
26
+ line = re.sub(r"' ll\b", "'ll", line)
27
+ line = re.sub(r"' re\b", "'re", line)
28
+ line = re.sub(r"' ve\b", "'ve", line)
29
+ line = re.sub(r"' LL\b", "'LL ", line)
30
+ line = re.sub(r"' RE\b", "'RE ", line)
31
+ line = re.sub(r"' VE\b", "'VE ", line)
32
+ return line.split()
33
+
34
+
35
+ def _split_on_cont_punc(tokens):
36
+ new_tokens = []
37
+ for token in tokens:
38
+ if len(token) > 1:
39
+ last_j = 0
40
+ pre_is_punc = _is_punctuation(token[0])
41
+ for j, ch in enumerate(token):
42
+ is_punc = _is_punctuation(ch)
43
+ if is_punc != pre_is_punc:
44
+ new_tokens.append(token[last_j: j])
45
+ last_j = j
46
+ pre_is_punc = is_punc
47
+ if last_j < len(token):
48
+ new_tokens.append(token[last_j:])
49
+ else:
50
+ new_tokens.append(token)
51
+ return new_tokens
52
+
53
+
54
+ def _split_pre_and_post_punc(tokens):
55
+ def pre_punc(token):
56
+ last_j = 0
57
+ for j in range(1, len(token)):
58
+ if not _is_punctuation(token[j]):
59
+ last_j = j
60
+ break
61
+ return token[:last_j], token[last_j:]
62
+
63
+ def post_punc(token):
64
+ last_j = len(token)
65
+ for j in range(len(token) - 2, -1, -1):
66
+ if not _is_punctuation(token[j]):
67
+ last_j = j + 1
68
+ break
69
+ return token[:last_j], token[last_j:]
70
+
71
+ new_tokens = []
72
+ for token in tokens:
73
+ if len(token) > 1 and _is_punctuation(token[0]):
74
+ a, b = pre_punc(token)
75
+ if a:
76
+ new_tokens.append(a)
77
+ if b:
78
+ if _is_punctuation(b[-1]):
79
+ c, d = post_punc(b)
80
+ if c:
81
+ new_tokens.append(c)
82
+ if d:
83
+ new_tokens.append(d)
84
+ else:
85
+ new_tokens.append(b)
86
+ elif len(token) > 1 and _is_punctuation(token[-1]):
87
+ a, b = post_punc(token)
88
+ if a:
89
+ new_tokens.append(a)
90
+ if b:
91
+ new_tokens.append(b)
92
+ else:
93
+ new_tokens.append(token)
94
+ return new_tokens
95
+
96
+
97
+ class GuokeTokenizer(object):
98
+ def __init__(self, cfg):
99
+ self.cfg = cfg
100
+
101
+ def encode(self, x: str) -> str:
102
+ x = x.strip()
103
+ x = x.replace("``", '"').replace("''", '"')
104
+ x = x.translate(TRANS_TABLE)
105
+ tokens = x.split()
106
+ tokens = _split_pre_and_post_punc(tokens)
107
+ tokens = _handle_single_quote(tokens)
108
+ x = " ".join(tokens)
109
+ if self.cfg.lower:
110
+ x = x.lower()
111
+ return x
112
+
113
+ def decode(self, x: str) -> str:
114
+ raise NotImplementedError()
sentencepiece_bpe.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is ported from fairseq:
4
+ # https://github.com/facebookresearch/fairseq
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of the fairseq repo
8
+
9
+ from dataclasses import dataclass, field
10
+ from typing import Optional
11
+
12
+
13
+ @dataclass
14
+ class SentencepieceConfig:
15
+ sentencepiece_model: str = field(
16
+ default="???", metadata={"help": "path to sentencepiece model"}
17
+ )
18
+ sentencepiece_enable_sampling: bool = field(
19
+ default=False, metadata={"help": "enable sampling"}
20
+ )
21
+ sentencepiece_alpha: Optional[float] = field(
22
+ default=None, metadata={
23
+ "help": "soothing parameter for unigram sampling, "
24
+ "and merge probability for BPE-dropout"
25
+ }
26
+ )
27
+
28
+
29
+ class SentencepieceBPE(object):
30
+ def __init__(self, cfg):
31
+ self.enable_sampling = cfg.sentencepiece_enable_sampling
32
+ self.alpha = cfg.sentencepiece_alpha
33
+ sentencepiece_model = cfg.sentencepiece_model
34
+ try:
35
+ import sentencepiece as spm
36
+
37
+ self.sp = spm.SentencePieceProcessor()
38
+ self.sp.Load(sentencepiece_model)
39
+ except ImportError:
40
+ raise ImportError(
41
+ "Please install sentencepiece with: pip install sentencepiece"
42
+ )
43
+
44
+ def encode(self, x: str) -> str:
45
+ return " ".join(
46
+ self.sp.Encode(
47
+ x, out_type=str, enable_sampling=self.enable_sampling, alpha=self.alpha
48
+ )
49
+ )
50
+
51
+ def decode(self, x: str) -> str:
52
+ return x.replace(" ", "").replace("\u2581", " ").strip()
53
+
54
+ def is_beginning_of_word(self, x: str) -> bool:
55
+ if x in ["<unk>", "<s>", "</s>", "<pad>"]:
56
+ # special elements are always considered beginnings
57
+ # HACK: this logic is already present in fairseq/tasks/masked_lm.py
58
+ # but these special tokens are also contained in the sentencepiece
59
+ # vocabulary which causes duplicate special tokens. This hack makes
60
+ # sure that they are all taken into account.
61
+ return True
62
+ return x.startswith("\u2581")
sp.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b2f031ad52ac175466dc380dbd6878873b7d45f20921dbe54c5e9a009decdb6
3
+ size 1358163
special_tokens_map.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "cls_token": "<s>",
4
+ "eos_token": "</s>",
5
+ "pad_token": "<pad>",
6
+ "sep_token": "</s>",
7
+ "unk_token": "<unk>"
8
+ }
tokenization_fairseq_t5.py ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from shutil import copyfile
3
+ from typing import List, Optional
4
+
5
+ from omegaconf import DictConfig
6
+ from transformers.tokenization_utils import PreTrainedTokenizer
7
+ from transformers.utils import logging
8
+
9
+ from .fairseq_dictionary import Dictionary
10
+ from .guoke_tokenizer import GuokeTokenizer
11
+ from .sentencepiece_bpe import SentencepieceBPE
12
+
13
+ logger = logging.get_logger(__name__)
14
+
15
+ VOCAB_FILES_NAMES = {
16
+ "sp_path": "sp.model",
17
+ "dict_path": "dict.txt"
18
+ }
19
+
20
+
21
+ class FairseqT5Tokenizer(PreTrainedTokenizer):
22
+ vocab_files_names = VOCAB_FILES_NAMES
23
+ model_input_names = ["input_ids", "attention_mask"]
24
+
25
+ def __init__(
26
+ self,
27
+ sp_path,
28
+ dict_path,
29
+ lower,
30
+ n_sentinel_tokens=0,
31
+ bos_token="<s>",
32
+ eos_token="</s>",
33
+ unk_token="<unk>",
34
+ pad_token="<pad>",
35
+ **kwargs
36
+ ) -> None:
37
+
38
+ self.sp_path = sp_path
39
+ self.dict_path = dict_path
40
+ self.lower = lower
41
+
42
+ self.fs_tokenizer = GuokeTokenizer(
43
+ DictConfig(
44
+ dict(
45
+ lower=lower
46
+ )
47
+ )
48
+ )
49
+ self.fs_bpe = SentencepieceBPE(
50
+ DictConfig(
51
+ dict(
52
+ sentencepiece_model=sp_path,
53
+ )
54
+ )
55
+ )
56
+ self.fs_dict = Dictionary.load(dict_path)
57
+ for i in range(n_sentinel_tokens):
58
+ self.fs_dict.add_symbol(f'<sen{i:03d}>')
59
+
60
+ if "sep_token" in kwargs:
61
+ assert kwargs["sep_token"] == eos_token
62
+ kwargs.pop("sep_token")
63
+ if "cls_token" in kwargs:
64
+ assert kwargs["cls_token"] == bos_token
65
+ kwargs.pop("cls_token")
66
+
67
+ super().__init__(
68
+ bos_token=bos_token,
69
+ eos_token=eos_token,
70
+ unk_token=unk_token,
71
+ pad_token=pad_token,
72
+ sep_token=eos_token,
73
+ cls_token=bos_token,
74
+ lower=self.lower,
75
+ n_sentinel_tokens=n_sentinel_tokens,
76
+ **kwargs,
77
+ )
78
+
79
+ @property
80
+ def vocab_size(self):
81
+ return len(self.fs_dict)
82
+
83
+ def get_vocab(self):
84
+ return self.fs_dict.indices
85
+
86
+ def get_special_tokens_mask(
87
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
88
+ ) -> List[int]:
89
+ """
90
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
91
+ special tokens using the tokenizer `prepare_for_model` method.
92
+
93
+ Args:
94
+ token_ids_0 (`List[int]`):
95
+ List of IDs.
96
+ token_ids_1 (`List[int]`, *optional*):
97
+ Optional second list of IDs for sequence pairs.
98
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
99
+ Whether or not the token list is already formatted with special tokens for the model.
100
+
101
+ Returns:
102
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
103
+ """
104
+ if already_has_special_tokens:
105
+ return super().get_special_tokens_mask(
106
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
107
+ )
108
+
109
+ if token_ids_1 is None:
110
+ return [1] + ([0] * len(token_ids_0)) + [1]
111
+ return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]
112
+
113
+ def create_token_type_ids_from_sequences(
114
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
115
+ ) -> List[int]:
116
+ sep = [self.sep_token_id]
117
+ cls = [self.cls_token_id]
118
+
119
+ if token_ids_1 is None:
120
+ return len(cls + token_ids_0 + sep) * [0]
121
+ return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]
122
+
123
+ def build_inputs_with_special_tokens(
124
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
125
+ ) -> List[int]:
126
+ if token_ids_1 is None:
127
+ return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
128
+ cls = [self.cls_token_id]
129
+ sep = [self.sep_token_id]
130
+ return cls + token_ids_0 + sep + sep + token_ids_1 + sep
131
+
132
+ def _tokenize(self, text: str) -> List[str]:
133
+ return self.fs_bpe.encode(self.fs_tokenizer.encode(text)).split(" ")
134
+
135
+ def _convert_token_to_id(self, token):
136
+ return self.fs_dict.index(token)
137
+
138
+ def _convert_id_to_token(self, index):
139
+ return self.fs_dict[index]
140
+
141
+ def convert_tokens_to_string(self, tokens):
142
+ return self.fs_bpe.decode(" ".join(tokens))
143
+
144
+ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
145
+ if not os.path.isdir(save_directory):
146
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
147
+ return
148
+ out_sp_path = os.path.join(
149
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["sp_path"]
150
+ )
151
+ out_dict_path = os.path.join(
152
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["dict_path"]
153
+ )
154
+
155
+ if os.path.abspath(self.sp_path) != os.path.abspath(out_sp_path):
156
+ copyfile(self.sp_path, out_sp_path)
157
+ logger.info(f"Copy from {self.sp_path} to {out_sp_path}")
158
+ if os.path.abspath(self.dict_path) != os.path.abspath(out_dict_path):
159
+ copyfile(self.dict_path, out_dict_path)
160
+ logger.info(f"Copy from {self.dict_path} to {out_dict_path}")
161
+
162
+ return out_sp_path, out_dict_path
tokenizer_config.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_map": {
3
+ "AutoTokenizer": [
4
+ "tokenization_fairseq_t5.FairseqT5Tokenizer",
5
+ null
6
+ ]
7
+ },
8
+ "bos_token": "<s>",
9
+ "clean_up_tokenization_spaces": true,
10
+ "cls_token": "<s>",
11
+ "eos_token": "</s>",
12
+ "lower": true,
13
+ "model_max_length": 1000000000000000019884624838656,
14
+ "n_sentinel_tokens": 512,
15
+ "pad_token": "<pad>",
16
+ "sep_token": "</s>",
17
+ "tokenizer_class": "FairseqT5Tokenizer",
18
+ "unk_token": "<unk>",
19
+ "use_fast": false
20
+ }