dmayhem93 commited on
Commit
909ab34
1 Parent(s): aaf191f

Upload tokenizer

Browse files
arcade100k.tiktoken ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>"
5
+ }
tokenization_arcade100k.py ADDED
@@ -0,0 +1,273 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright (c) 2023 Alibaba Cloud & Stability AI.
3
+ #
4
+ # Tongyi Qianwen LICENSE AGREEMENT:
5
+ # https://github.com/QwenLM/Qwen/blob/5aa84bdfd3237b37f01bc88cd49b3279b9a71d0b/Tongyi%20Qianwen%20LICENSE%20AGREEMENT
6
+ """Tokenization classes for Arcade100k."""
7
+
8
+ import base64
9
+ import os
10
+ import unicodedata
11
+ from typing import Collection, Dict, List, Set, Tuple, Union
12
+
13
+ import tiktoken
14
+ from transformers.utils import logging
15
+ from transformers import PreTrainedTokenizer, AddedToken
16
+
17
+ logger = logging.get_logger(__name__)
18
+
19
+ VOCAB_FILES_NAMES = {"vocab_file": "arcade100k.tiktoken"}
20
+ NAME = "arcade100k"
21
+
22
+
23
+ def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
24
+ with open(tiktoken_bpe_file, "rb") as f:
25
+ contents = f.read()
26
+ return {
27
+ base64.b64decode(token): int(rank)
28
+ for token, rank in (line.split() for line in contents.splitlines() if line)
29
+ }
30
+
31
+
32
+ ENDOFTEXT = "<|endoftext|>"
33
+ FIM = [
34
+ "<|fim_prefix|>",
35
+ "<|fim_middle|>",
36
+ "<|fim_suffix|>",
37
+ "<|fim_pad|>",
38
+ ]
39
+ # `StarCoder` Tokens
40
+ CODE = [
41
+ "<gh_stars>",
42
+ "<filename>",
43
+ "<issue_start>",
44
+ "<issue_comment>",
45
+ "<issue_closed>",
46
+ "<jupyter_start>",
47
+ "<jupyter_text>",
48
+ "<jupyter_code>",
49
+ "<jupyter_output>",
50
+ "<empty_output>",
51
+ "<commit_before>",
52
+ "<commit_msg>",
53
+ "<commit_after>",
54
+ "<reponame>",
55
+ ]
56
+ CHAT = [
57
+ "<|im_start|>", # Chat: Input message start
58
+ "<|im_end|>", # Chat: Input message end
59
+ ]
60
+ PAUSE = "<|pause|>" # Think before you speak (https://arxiv.org/abs/2310.02226)
61
+ REGISTERS = [
62
+ f"<|reg{i}|>" for i in range(0, 8)
63
+ ] # Register 0 sink token (https://arxiv.org/abs/2309.17453)
64
+ ENDOFPROMPT = "<|endofprompt|>"
65
+ SPECIAL_TOKENS_NAMES = (
66
+ [ENDOFTEXT]
67
+ + FIM
68
+ + CODE
69
+ + [ENDOFPROMPT]
70
+ + CHAT
71
+ + [PAUSE]
72
+ + REGISTERS
73
+ + ["<|extra0|>"]
74
+ )
75
+ START_ID = 100257
76
+ SPECIAL_TOKENS = {t: START_ID + i for i, t in enumerate(SPECIAL_TOKENS_NAMES)}
77
+
78
+
79
+ def _arcade100k(vocab_file: str):
80
+ mergeable_ranks = _load_tiktoken_bpe(vocab_file)
81
+
82
+ return {
83
+ "name": NAME,
84
+ "pat_str": r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""",
85
+ "mergeable_ranks": mergeable_ranks,
86
+ "special_tokens": SPECIAL_TOKENS,
87
+ }
88
+
89
+
90
+ class Arcade100kTokenizer(PreTrainedTokenizer):
91
+ """
92
+ Construct a Arcade100k tokenizer backed by `tiktoken`.
93
+
94
+ Args:
95
+ vocab_file (`str`):
96
+ Path to the vocabulary file.
97
+ errors (`str`, *optional*, defaults to `"replace"`):
98
+ How to handle errors in decoding UTF-8 byte sequences.
99
+ WARNING: the default behaviour of this function is lossy, since decoded bytes are not
100
+ guaranteed to be valid UTF-8. You can control this behaviour using the `errors` parameter,
101
+ for instance, setting `errors=strict`.
102
+ """
103
+
104
+ vocab_files_names = VOCAB_FILES_NAMES
105
+ model_input_names = ["input_ids", "attention_mask"]
106
+
107
+ def __init__(
108
+ self,
109
+ vocab_file: str,
110
+ errors: str = "replace",
111
+ **kwargs,
112
+ ):
113
+ super().__init__(errors=errors, **kwargs)
114
+ self._tiktoken_config = _arcade100k(vocab_file)
115
+ self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
116
+ self.errors = errors
117
+ # TODO: Remove this assertion
118
+ assert (
119
+ len(self.tokenizer._mergeable_ranks)
120
+ + len(self.tokenizer._special_tokens)
121
+ + 1
122
+ == self.tokenizer.n_vocab
123
+ ), f"{len(self.tokenizer._mergeable_ranks) + len(self.tokenizer._special_tokens)} != {self.tokenizer.n_vocab} in encoding"
124
+
125
+ self.decoder = {i: n for n, i in self.tokenizer._mergeable_ranks.items()}
126
+ self.decoder.update({i: n for n, i in self.tokenizer._special_tokens.items()})
127
+ self.eos_token = self.decoder[self.tokenizer.eot_token]
128
+ self.pad_token = self.decoder[self.tokenizer.eot_token]
129
+
130
+ def __len__(self):
131
+ return self.tokenizer.n_vocab
132
+
133
+ @property
134
+ def vocab_size(self):
135
+ return self.tokenizer.n_vocab
136
+
137
+ def get_vocab(self) -> Dict[bytes, int]:
138
+ return self.tokenizer._mergeable_ranks
139
+
140
+ def convert_tokens_to_ids(
141
+ self, tokens: Union[bytes, str, List[Union[bytes, str]]]
142
+ ) -> List[int]:
143
+ ids = []
144
+ if isinstance(tokens, (str, bytes)):
145
+ if tokens in self.tokenizer._special_tokens:
146
+ return self.tokenizer._special_tokens[tokens]
147
+ else:
148
+ return self.tokenizer._mergeable_ranks.get(tokens)
149
+ for token in tokens:
150
+ if token in self.tokenizer._special_tokens:
151
+ ids.append(self.tokenizer._special_tokens[token])
152
+ else:
153
+ ids.append(self.tokenizer._mergeable_ranks.get(token))
154
+ return ids
155
+
156
+ def _add_tokens(
157
+ self,
158
+ new_tokens: Union[List[str], List[AddedToken]],
159
+ special_tokens: bool = False,
160
+ ) -> int:
161
+ if not special_tokens and new_tokens:
162
+ raise ValueError("Adding regular tokens is not supported")
163
+ for token in new_tokens:
164
+ surface_form = token.content if isinstance(token, AddedToken) else token
165
+ if surface_form not in SPECIAL_TOKENS:
166
+ raise ValueError("Adding unknown special tokens is not supported")
167
+ return 0
168
+
169
+ def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
170
+ """
171
+ Save only the vocabulary of the tokenizer (vocabulary).
172
+
173
+ Returns:
174
+ `Tuple(str)`: Paths to the files saved.
175
+ """
176
+ file_path = os.path.join(save_directory, "arcade100k.tiktoken")
177
+ with open(file_path, "w", encoding="utf8") as w:
178
+ for k, v in self.tokenizer._mergeable_ranks.items():
179
+ line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
180
+ w.write(line)
181
+ return (file_path,)
182
+
183
+ def tokenize(
184
+ self,
185
+ text: str,
186
+ allowed_special: Union[Set, str] = "all",
187
+ disallowed_special: Union[Collection, str] = (),
188
+ **kwargs,
189
+ ) -> List[Union[bytes, str]]:
190
+ """
191
+ Converts a string in a sequence of tokens.
192
+
193
+ Args:
194
+ text (`str`):
195
+ The sequence to be encoded.
196
+ allowed_special (`Literal["all"]` or `set`):
197
+ The surface forms of the tokens to be encoded as special tokens in regular texts.
198
+ Default to "all".
199
+ disallowed_special (`Literal["all"]` or `Collection`):
200
+ The surface forms of the tokens that should not be in regular texts and trigger errors.
201
+ Default to an empty tuple.
202
+
203
+ kwargs (additional keyword arguments, *optional*):
204
+ Will be passed to the underlying model specific encode method.
205
+
206
+ Returns:
207
+ `List[bytes|str]`: The list of tokens.
208
+ """
209
+ tokens = []
210
+ text = unicodedata.normalize("NFC", text)
211
+
212
+ # this implementation takes a detour: text -> token id -> token surface forms
213
+ for t in self.tokenizer.encode(
214
+ text, allowed_special=allowed_special, disallowed_special=disallowed_special
215
+ ):
216
+ tokens.append(self.decoder[t])
217
+ return tokens
218
+
219
+ def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
220
+ """
221
+ Converts a sequence of tokens in a single string.
222
+ """
223
+ text = ""
224
+ temp = b""
225
+ for t in tokens:
226
+ if isinstance(t, str):
227
+ if temp:
228
+ text += temp.decode("utf-8", errors=self.errors)
229
+ temp = b""
230
+ text += t
231
+ elif isinstance(t, bytes):
232
+ temp += t
233
+ else:
234
+ raise TypeError("token should only be of type types or str")
235
+ if temp:
236
+ text += temp.decode("utf-8", errors=self.errors)
237
+ return text
238
+
239
+ def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
240
+ """Converts an id to a token, special tokens included"""
241
+ if index in self.decoder:
242
+ return self.decoder[index]
243
+ raise ValueError("unknown ids")
244
+
245
+ def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
246
+ """Converts a token to an id using the vocab, special tokens included"""
247
+ if token in self.tokenizer._special_tokens:
248
+ return self.tokenizer._special_tokens[token]
249
+ if token in self.tokenizer._mergeable_ranks:
250
+ return self.tokenizer._mergeable_ranks[token]
251
+ raise ValueError("unknown token")
252
+
253
+ def _tokenize(self, text: str, **kwargs):
254
+ """
255
+ Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
256
+ vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
257
+
258
+ Do NOT take care of added tokens.
259
+ """
260
+ raise NotImplementedError
261
+
262
+ def _decode(
263
+ self,
264
+ token_ids: Union[int, List[int]],
265
+ skip_special_tokens: bool = False,
266
+ errors: str = None,
267
+ **kwargs,
268
+ ) -> str:
269
+ if isinstance(token_ids, int):
270
+ token_ids = [token_ids]
271
+ if skip_special_tokens:
272
+ token_ids = [i for i in token_ids if i < self.tokenizer.eot_token]
273
+ return self.tokenizer.decode(token_ids)
tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {},
3
+ "auto_map": {
4
+ "AutoTokenizer": [
5
+ "tokenization_arcade100k.Arcade100kTokenizer",
6
+ null
7
+ ]
8
+ },
9
+ "bos_token": "<|endoftext|>",
10
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
11
+ "clean_up_tokenization_spaces": true,
12
+ "eos_token": "<|endoftext|>",
13
+ "errors": "replace",
14
+ "model_max_length": 2048,
15
+ "pad_token": "<|endoftext|>",
16
+ "tokenizer_class": "Arcade100kTokenizer"
17
+ }