aloobun commited on
Commit
86ab9ed
1 Parent(s): c36ca75

Upload tokenizer

Browse files
arcade100k.tiktoken ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "eos_token": "<|im_end|>",
3
+ "pad_token": "<|endoftext|>"
4
+ }
tokenization_arcade100k.py ADDED
@@ -0,0 +1,292 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright (c) 2023 Alibaba Cloud & Stability AI.
3
+ #
4
+ # Tongyi Qianwen LICENSE AGREEMENT:
5
+ # https://github.com/QwenLM/Qwen/blob/5aa84bdfd3237b37f01bc88cd49b3279b9a71d0b/Tongyi%20Qianwen%20LICENSE%20AGREEMENT
6
+ """Tokenization classes for Arcade100k."""
7
+
8
+ import base64
9
+ import os
10
+ import unicodedata
11
+ from typing import Collection, Dict, List, Set, Tuple, Union
12
+
13
+ import tiktoken
14
+ from transformers.utils import logging
15
+ from transformers import PreTrainedTokenizer, AddedToken
16
+
17
+ logger = logging.get_logger(__name__)
18
+
19
+ VOCAB_FILES_NAMES = {"vocab_file": "arcade100k.tiktoken"}
20
+ NAME = "arcade100k"
21
+
22
+
23
+ def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
24
+ with open(tiktoken_bpe_file, "rb") as f:
25
+ contents = f.read()
26
+ return {
27
+ base64.b64decode(token): int(rank)
28
+ for token, rank in (line.split() for line in contents.splitlines() if line)
29
+ }
30
+
31
+
32
+ ENDOFTEXT = "<|endoftext|>"
33
+ FIM = [
34
+ "<|fim_prefix|>",
35
+ "<|fim_middle|>",
36
+ "<|fim_suffix|>",
37
+ "<|fim_pad|>",
38
+ ]
39
+ # `StarCoder` Tokens
40
+ CODE = [
41
+ "<gh_stars>",
42
+ "<filename>",
43
+ "<issue_start>",
44
+ "<issue_comment>",
45
+ "<issue_closed>",
46
+ "<jupyter_start>",
47
+ "<jupyter_text>",
48
+ "<jupyter_code>",
49
+ "<jupyter_output>",
50
+ "<empty_output>",
51
+ "<commit_before>",
52
+ "<commit_msg>",
53
+ "<commit_after>",
54
+ "<reponame>",
55
+ ]
56
+ CHAT = [
57
+ "<|im_start|>", # Chat: Input message start
58
+ "<|im_end|>", # Chat: Input message end
59
+ ]
60
+ PAUSE = "<|pause|>" # Think before you speak (https://arxiv.org/abs/2310.02226)
61
+ REGISTERS = [
62
+ f"<|reg{i}|>" for i in range(0, 8)
63
+ ] # Register 0 sink token (https://arxiv.org/abs/2309.17453)
64
+ ENDOFPROMPT = "<|endofprompt|>"
65
+ SPECIAL_TOKENS_NAMES = (
66
+ [ENDOFTEXT]
67
+ + FIM
68
+ + CODE
69
+ + [ENDOFPROMPT]
70
+ + CHAT
71
+ + [PAUSE]
72
+ + REGISTERS
73
+ + ["<|extra0|>"]
74
+ )
75
+ START_ID = 100257
76
+ SPECIAL_TOKENS = {t: START_ID + i for i, t in enumerate(SPECIAL_TOKENS_NAMES)}
77
+
78
+
79
+ def _arcade100k(vocab_file: str):
80
+ mergeable_ranks = _load_tiktoken_bpe(vocab_file)
81
+
82
+ return {
83
+ "name": NAME,
84
+ "pat_str": r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""",
85
+ "mergeable_ranks": mergeable_ranks,
86
+ "special_tokens": SPECIAL_TOKENS,
87
+ }
88
+
89
+
90
+ class Arcade100kTokenizer(PreTrainedTokenizer):
91
+ """
92
+ Construct a Arcade100k tokenizer backed by `tiktoken`.
93
+
94
+ Args:
95
+ vocab_file (`str`):
96
+ Path to the vocabulary file.
97
+ errors (`str`, *optional*, defaults to `"replace"`):
98
+ How to handle errors in decoding UTF-8 byte sequences.
99
+ WARNING: the default behaviour of this function is lossy, since decoded bytes are not
100
+ guaranteed to be valid UTF-8. You can control this behaviour using the `errors` parameter,
101
+ for instance, setting `errors=strict`.
102
+ """
103
+
104
+ vocab_files_names = VOCAB_FILES_NAMES
105
+ model_input_names = ["input_ids", "attention_mask"]
106
+
107
+ def __init__(
108
+ self,
109
+ vocab_file: str,
110
+ errors: str = "replace",
111
+ **kwargs,
112
+ ):
113
+ super().__init__(errors=errors, **kwargs)
114
+ self.errors = errors
115
+
116
+ self._tiktoken_config = _arcade100k(vocab_file)
117
+ self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
118
+
119
+ # TODO: Remove this assertion
120
+ assert (
121
+ len(self.tokenizer._mergeable_ranks)
122
+ + len(self.tokenizer._special_tokens)
123
+ + 1
124
+ == self.tokenizer.n_vocab
125
+ ), f"{len(self.tokenizer._mergeable_ranks) + len(self.tokenizer._special_tokens)} != {self.tokenizer.n_vocab} in encoding"
126
+
127
+ self.decoder = {i: n for n, i in self.tokenizer._mergeable_ranks.items()}
128
+ self.decoder.update({i: n for n, i in self.tokenizer._special_tokens.items()})
129
+ # Provide default `eos_token` and `pad_token`
130
+ if self.eos_token is None:
131
+ self.eos_token = self.decoder[self.tokenizer.eot_token]
132
+ if self.pad_token is None:
133
+ self.pad_token = self.decoder[self.tokenizer.pad_token]
134
+
135
+ # Expose for convenience
136
+ self.mergeable_ranks = self.tokenizer._mergeable_ranks
137
+ self.special_tokens = self.tokenizer._special_tokens
138
+
139
+ def __len__(self):
140
+ return self.tokenizer.n_vocab
141
+
142
+ def __getstate__(self):
143
+ # Required for `pickle` support
144
+ state = self.__dict__.copy()
145
+ del state["tokenizer"]
146
+ return state
147
+
148
+ def __setstate__(self, state):
149
+ self.__dict__.update(state)
150
+ self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
151
+
152
+ @property
153
+ def vocab_size(self):
154
+ return self.tokenizer.n_vocab
155
+
156
+ def get_vocab(self) -> Dict[bytes, int]:
157
+ return self.tokenizer._mergeable_ranks
158
+
159
+ def convert_tokens_to_ids(
160
+ self, tokens: Union[bytes, str, List[Union[bytes, str]]]
161
+ ) -> List[int]:
162
+ ids = []
163
+ if isinstance(tokens, (str, bytes)):
164
+ if tokens in self.tokenizer._special_tokens:
165
+ return self.tokenizer._special_tokens[tokens]
166
+ else:
167
+ return self.tokenizer._mergeable_ranks.get(tokens)
168
+ for token in tokens:
169
+ if token in self.tokenizer._special_tokens:
170
+ ids.append(self.tokenizer._special_tokens[token])
171
+ else:
172
+ ids.append(self.tokenizer._mergeable_ranks.get(token))
173
+ return ids
174
+
175
+ def _add_tokens(
176
+ self,
177
+ new_tokens: Union[List[str], List[AddedToken]],
178
+ special_tokens: bool = False,
179
+ ) -> int:
180
+ if not special_tokens and new_tokens:
181
+ raise ValueError("Adding regular tokens is not supported")
182
+ for token in new_tokens:
183
+ surface_form = token.content if isinstance(token, AddedToken) else token
184
+ if surface_form not in SPECIAL_TOKENS:
185
+ raise ValueError("Adding unknown special tokens is not supported")
186
+ return 0
187
+
188
+ def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
189
+ """
190
+ Save only the vocabulary of the tokenizer (vocabulary).
191
+
192
+ Returns:
193
+ `Tuple(str)`: Paths to the files saved.
194
+ """
195
+ file_path = os.path.join(save_directory, "arcade100k.tiktoken")
196
+ with open(file_path, "w", encoding="utf8") as w:
197
+ for k, v in self.tokenizer._mergeable_ranks.items():
198
+ line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
199
+ w.write(line)
200
+ return (file_path,)
201
+
202
+ def tokenize(
203
+ self,
204
+ text: str,
205
+ allowed_special: Union[Set, str] = "all",
206
+ disallowed_special: Union[Collection, str] = (),
207
+ **kwargs,
208
+ ) -> List[Union[bytes, str]]:
209
+ """
210
+ Converts a string in a sequence of tokens.
211
+
212
+ Args:
213
+ text (`str`):
214
+ The sequence to be encoded.
215
+ allowed_special (`Literal["all"]` or `set`):
216
+ The surface forms of the tokens to be encoded as special tokens in regular texts.
217
+ Default to "all".
218
+ disallowed_special (`Literal["all"]` or `Collection`):
219
+ The surface forms of the tokens that should not be in regular texts and trigger errors.
220
+ Default to an empty tuple.
221
+
222
+ kwargs (additional keyword arguments, *optional*):
223
+ Will be passed to the underlying model specific encode method.
224
+
225
+ Returns:
226
+ `List[bytes|str]`: The list of tokens.
227
+ """
228
+ tokens = []
229
+ text = unicodedata.normalize("NFC", text)
230
+
231
+ # this implementation takes a detour: text -> token id -> token surface forms
232
+ for t in self.tokenizer.encode(
233
+ text, allowed_special=allowed_special, disallowed_special=disallowed_special
234
+ ):
235
+ tokens.append(self.decoder[t])
236
+ return tokens
237
+
238
+ def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
239
+ """
240
+ Converts a sequence of tokens in a single string.
241
+ """
242
+ text = ""
243
+ temp = b""
244
+ for t in tokens:
245
+ if isinstance(t, str):
246
+ if temp:
247
+ text += temp.decode("utf-8", errors=self.errors)
248
+ temp = b""
249
+ text += t
250
+ elif isinstance(t, bytes):
251
+ temp += t
252
+ else:
253
+ raise TypeError("token should only be of type types or str")
254
+ if temp:
255
+ text += temp.decode("utf-8", errors=self.errors)
256
+ return text
257
+
258
+ def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
259
+ """Converts an id to a token, special tokens included"""
260
+ if index in self.decoder:
261
+ return self.decoder[index]
262
+ raise ValueError("unknown ids")
263
+
264
+ def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
265
+ """Converts a token to an id using the vocab, special tokens included"""
266
+ if token in self.tokenizer._special_tokens:
267
+ return self.tokenizer._special_tokens[token]
268
+ if token in self.tokenizer._mergeable_ranks:
269
+ return self.tokenizer._mergeable_ranks[token]
270
+ raise ValueError("unknown token")
271
+
272
+ def _tokenize(self, text: str, **kwargs):
273
+ """
274
+ Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
275
+ vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
276
+
277
+ Do NOT take care of added tokens.
278
+ """
279
+ raise NotImplementedError
280
+
281
+ def _decode(
282
+ self,
283
+ token_ids: Union[int, List[int]],
284
+ skip_special_tokens: bool = False,
285
+ errors: str = None,
286
+ **kwargs,
287
+ ) -> str:
288
+ if isinstance(token_ids, int):
289
+ token_ids = [token_ids]
290
+ if skip_special_tokens:
291
+ token_ids = [i for i in token_ids if i < self.tokenizer.eot_token]
292
+ return self.tokenizer.decode(token_ids)
tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {},
3
+ "auto_map": {
4
+ "AutoTokenizer": [
5
+ "tokenization_arcade100k.Arcade100kTokenizer",
6
+ null
7
+ ]
8
+ },
9
+ "clean_up_tokenization_spaces": true,
10
+ "eos_token": "<|im_end|>",
11
+ "errors": "replace",
12
+ "model_max_length": 1000000000000000019884624838656,
13
+ "pad_token": "<|endoftext|>",
14
+ "tokenizer_class": "Arcade100kTokenizer"
15
+ }