SearchUnify-ML commited on
Commit
52c3ddc
1 Parent(s): ed3caa7

Upload tokenizer

Browse files
special_tokens_map.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eos_token": {
3
+ "content": "<|endoftext|>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "pad_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ }
16
+ }
tokenization_xgen.py ADDED
@@ -0,0 +1,234 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2023, salesforce.com, inc.
2
+ # All rights reserved.
3
+ # SPDX-License-Identifier: Apache-2.0
4
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/Apache-2.0
5
+ """Tokenization classes for xgen."""
6
+
7
+ from typing import List, Optional
8
+
9
+ from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
10
+ from transformers.utils import logging
11
+
12
+ try:
13
+ import tiktoken
14
+ except ModuleNotFoundError as e:
15
+ raise ModuleNotFoundError("XGen requires the installation of tiktoken. Please install it via `pip install tiktoken`.") from e
16
+
17
+
18
+ logger = logging.get_logger(__name__)
19
+
20
+ MAX_MODEL_INPUT_SIZES = {
21
+ "Salesforce/xgen-7b-4k-base": 4096,
22
+ "Salesforce/xgen-7b-8k-base": 8192,
23
+ "Salesforce/xgen-7b-4k-inst": 4096,
24
+ "Salesforce/xgen-7b-8k-inst": 8192
25
+ }
26
+
27
+
28
+ def tiktoken_tokenizer(base="gpt2", pad_token=None, add_special=True):
29
+ if not add_special:
30
+ return tiktoken.get_encoding(base)
31
+
32
+ def include_whitespace(n_min=2, n_max=20):
33
+ whitespaces = [" " * n for n in reversed(range(n_min, n_max))]
34
+ return whitespaces
35
+
36
+ def include_tabs(n_min=2, n_max=20):
37
+ tabs = ["\t" * n for n in reversed(range(n_min, n_max))]
38
+ return tabs
39
+
40
+ def include_fim_tokens():
41
+ fim_tokens = [
42
+ "<fim_prefix>",
43
+ "<fim_middle>",
44
+ "<fim_suffix>",
45
+ "<fim_pad>",
46
+ "<filename>",
47
+ "<gh_stars>",
48
+ "<issue_start>",
49
+ "<issue_comment>",
50
+ "<issue_closed>",
51
+ "<jupyter_start>",
52
+ "<jupyter_text>",
53
+ "<jupyter_code>",
54
+ "<jupyter_output>",
55
+ "<empty_output>",
56
+ "<commit_before>",
57
+ "<commit_msg>",
58
+ "<commit_after>",
59
+ "<reponame>"
60
+ ]
61
+ return fim_tokens
62
+
63
+ add_whitespaces = include_whitespace(n_min=2, n_max=32)
64
+ add_tabs = include_tabs(n_min=2, n_max=10)
65
+ fim_tokens = include_fim_tokens()
66
+
67
+ tokenizer = tiktoken.get_encoding(base)
68
+
69
+ idx = tokenizer.n_vocab
70
+
71
+ bpe_ranks = tokenizer._mergeable_ranks
72
+
73
+ for wsp in add_whitespaces:
74
+ bpe_ranks[bytes(wsp, 'ascii')] = idx
75
+ idx += 1
76
+ for t in add_tabs:
77
+ bpe_ranks[bytes(t, 'ascii')] = idx
78
+ idx += 1
79
+
80
+ special_tokens = dict()
81
+
82
+ for sp in fim_tokens:
83
+ special_tokens[sp] = idx
84
+ idx += 1
85
+
86
+ if pad_token and pad_token not in tokenizer._special_tokens and pad_token not in special_tokens:
87
+ special_tokens[pad_token] = idx
88
+ idx += 1
89
+ # In production, load the arguments directly instead of accessing private attributes
90
+ # See openai_public.py for examples of arguments for specific encodings
91
+ enc = tiktoken.Encoding(
92
+ # If you're changing the set of special tokens, make sure to use a different name
93
+ # It should be clear from the name what behaviour to expect.
94
+ name=base.replace("base", "im"),
95
+ pat_str=tokenizer._pat_str,
96
+ mergeable_ranks=bpe_ranks,
97
+ special_tokens={
98
+ **tokenizer._special_tokens,
99
+ **special_tokens
100
+ }
101
+ )
102
+ return enc
103
+
104
+
105
+ class XgenTokenizer(PreTrainedTokenizer):
106
+ """
107
+ Construct a Xgen tokenizer. Based on byte-level Byte-Pair-Encoding.
108
+ Args:
109
+ vocab_file (`str`):
110
+ Path to the vocabulary file.
111
+ """
112
+ max_model_input_sizes = MAX_MODEL_INPUT_SIZES
113
+ model_input_names = ["input_ids", "attention_mask"]
114
+
115
+ def __init__(
116
+ self,
117
+ pad_token=None,
118
+ eos_token="<|endoftext|>",
119
+ add_eos_token=False,
120
+ add_special_tokens=True,
121
+ **kwargs,
122
+ ):
123
+ pad_token_added = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
124
+ eos_token_added = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
125
+ super().__init__(
126
+ pad_token=pad_token_added,
127
+ eos_token=eos_token_added,
128
+ add_eos_token=add_eos_token,
129
+ add_special_tokens=add_special_tokens,
130
+ **kwargs,
131
+ )
132
+ self.add_eos_token = add_eos_token
133
+ self.encoder = tiktoken_tokenizer(base="gpt2", pad_token=pad_token, add_special=add_special_tokens)
134
+
135
+ @property
136
+ def vocab_size(self):
137
+ """Returns vocab size"""
138
+ return self.encoder.n_vocab
139
+
140
+ def get_vocab(self):
141
+ """Returns vocab as a dict"""
142
+ vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
143
+ return vocab
144
+
145
+ def _tokenize(self, text, **kwargs):
146
+ """Returns a tokenized string."""
147
+ return self.encoder.encode(text, allowed_special="all")
148
+
149
+ def _convert_token_to_id(self, token):
150
+ """Converts a token (str) in an id using the vocab."""
151
+ if isinstance(token, str):
152
+ return self.encoder.encode_single_token(token)
153
+ else:
154
+ return token
155
+
156
+ def _convert_id_to_token(self, index):
157
+ """Converts an index (integer) in a token (str) using the vocab."""
158
+ return self.encoder.decode_single_token_bytes(index).decode("utf-8")
159
+
160
+ def _decode(self, token_ids: List[int], skip_special_tokens: bool = False, **kwargs):
161
+ if skip_special_tokens:
162
+ token_ids = [t for t in token_ids if t not in self.all_special_ids]
163
+ return self.encoder.decode(token_ids)
164
+
165
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
166
+ """Build model inputs from a sequence by appending eos_token_id."""
167
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
168
+
169
+ output = token_ids_0 + eos_token_id
170
+
171
+ if token_ids_1 is not None:
172
+ output = output + token_ids_1 + eos_token_id
173
+
174
+ return output
175
+
176
+ def get_special_tokens_mask(
177
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None,
178
+ already_has_special_tokens: bool = False
179
+ ) -> List[int]:
180
+ """
181
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
182
+ special tokens using the tokenizer `prepare_for_model` method.
183
+ Args:
184
+ token_ids_0 (`List[int]`):
185
+ List of IDs.
186
+ token_ids_1 (`List[int]`, *optional*):
187
+ Optional second list of IDs for sequence pairs.
188
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
189
+ Whether the token list is already formatted with special tokens for the model.
190
+ Returns:
191
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
192
+ """
193
+ if already_has_special_tokens:
194
+ return super().get_special_tokens_mask(
195
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
196
+ )
197
+
198
+ eos_token_id = [1] if self.add_eos_token else []
199
+
200
+ if token_ids_1 is None:
201
+ return ([0] * len(token_ids_0)) + eos_token_id
202
+ return ([0] * len(token_ids_0)) + eos_token_id + ([0] * len(token_ids_1)) + eos_token_id
203
+
204
+ def create_token_type_ids_from_sequences(
205
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
206
+ ) -> List[int]:
207
+ """
208
+ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
209
+ sequence pair mask has the following format:
210
+ ```
211
+ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
212
+ | first sequence | second sequence |
213
+ ```
214
+ if token_ids_1 is None, only returns the first portion of the mask (0s).
215
+ Args:
216
+ token_ids_0 (`List[int]`):
217
+ List of ids.
218
+ token_ids_1 (`List[int]`, *optional*):
219
+ Optional second list of IDs for sequence pairs.
220
+ Returns:
221
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
222
+ """
223
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
224
+
225
+ output = [0] * len(token_ids_0 + eos_token_id)
226
+
227
+ if token_ids_1 is not None:
228
+ output += [1] * len(token_ids_1 + eos_token_id)
229
+
230
+ return output
231
+
232
+ # has no vocab file
233
+ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
234
+ return ()
tokenizer_config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_eos_token": false,
3
+ "add_special_tokens": true,
4
+ "auto_map": {
5
+ "AutoTokenizer": [
6
+ "tokenization_xgen.XgenTokenizer",
7
+ null
8
+ ]
9
+ },
10
+ "clean_up_tokenization_spaces": true,
11
+ "eos_token": {
12
+ "__type": "AddedToken",
13
+ "content": "<|endoftext|>",
14
+ "lstrip": false,
15
+ "normalized": true,
16
+ "rstrip": false,
17
+ "single_word": false
18
+ },
19
+ "model_max_length": 8192,
20
+ "pad_token": null,
21
+ "tokenizer_class": "XgenTokenizer"
22
+ }