thomadev0 commited on
Commit
bd0969f
1 Parent(s): 0bbcfba

Upload folder using huggingface_hub

Browse files
README.md ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - en
4
+ license: other
5
+ tags:
6
+ - causal-lm
7
+ - mlx
8
+ datasets:
9
+ - HuggingFaceH4/ultrachat_200k
10
+ - allenai/ultrafeedback_binarized_cleaned
11
+ - meta-math/MetaMathQA
12
+ - WizardLM/WizardLM_evol_instruct_V2_196k
13
+ - openchat/openchat_sharegpt4_dataset
14
+ - LDJnr/Capybara
15
+ - Intel/orca_dpo_pairs
16
+ - hkust-nlp/deita-10k-v0
17
+ extra_gated_fields:
18
+ Name: text
19
+ Email: text
20
+ Country: text
21
+ Organization or Affiliation: text
22
+ I ALLOW Stability AI to email me about new model releases: checkbox
23
+ ---
24
+
25
+ # mlx-community/stablelm-2-zephyr-1_6b
26
+ This model was converted to MLX format from [`stabilityai/stablelm-2-zephyr-1_6b`]().
27
+ Refer to the [original model card](https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b) for more details on the model.
28
+ ## Use with mlx
29
+
30
+ ```bash
31
+ pip install mlx-lm
32
+ ```
33
+
34
+ ```python
35
+ from mlx_lm import load, generate
36
+
37
+ model, tokenizer = load("mlx-community/stablelm-2-zephyr-1_6b")
38
+ response = generate(model, tokenizer, prompt="hello", verbose=True)
39
+ ```
arcade100k.tiktoken ADDED
The diff for this file is too large to render. See raw diff
 
config.json ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 100352,
3
+ "max_position_embeddings": 4096,
4
+ "intermediate_size": 5632,
5
+ "hidden_size": 2048,
6
+ "num_hidden_layers": 24,
7
+ "num_attention_heads": 32,
8
+ "num_key_value_heads": 32,
9
+ "hidden_act": "silu",
10
+ "rope_pct": 0.25,
11
+ "rope_theta": 10000,
12
+ "initializer_range": 0.02,
13
+ "norm_eps": 1e-05,
14
+ "use_cache": true,
15
+ "use_qkv_bias": true,
16
+ "tie_word_embeddings": false,
17
+ "attention_dropout": 0.0,
18
+ "return_dict": true,
19
+ "output_hidden_states": false,
20
+ "output_attentions": false,
21
+ "torchscript": false,
22
+ "torch_dtype": "float16",
23
+ "use_bfloat16": false,
24
+ "tf_legacy_loss": false,
25
+ "pruned_heads": {},
26
+ "chunk_size_feed_forward": 0,
27
+ "is_encoder_decoder": false,
28
+ "is_decoder": false,
29
+ "cross_attention_hidden_size": null,
30
+ "add_cross_attention": false,
31
+ "tie_encoder_decoder": false,
32
+ "max_length": 20,
33
+ "min_length": 0,
34
+ "do_sample": false,
35
+ "early_stopping": false,
36
+ "num_beams": 1,
37
+ "num_beam_groups": 1,
38
+ "diversity_penalty": 0.0,
39
+ "temperature": 1.0,
40
+ "top_k": 50,
41
+ "top_p": 1.0,
42
+ "typical_p": 1.0,
43
+ "repetition_penalty": 1.0,
44
+ "length_penalty": 1.0,
45
+ "no_repeat_ngram_size": 0,
46
+ "encoder_no_repeat_ngram_size": 0,
47
+ "bad_words_ids": null,
48
+ "num_return_sequences": 1,
49
+ "output_scores": false,
50
+ "return_dict_in_generate": false,
51
+ "forced_bos_token_id": null,
52
+ "forced_eos_token_id": null,
53
+ "remove_invalid_values": false,
54
+ "exponential_decay_length_penalty": null,
55
+ "suppress_tokens": null,
56
+ "begin_suppress_tokens": null,
57
+ "architectures": [
58
+ "StableLMEpochForCausalLM"
59
+ ],
60
+ "finetuning_task": null,
61
+ "id2label": {
62
+ "0": "LABEL_0",
63
+ "1": "LABEL_1"
64
+ },
65
+ "label2id": {
66
+ "LABEL_0": 0,
67
+ "LABEL_1": 1
68
+ },
69
+ "tokenizer_class": null,
70
+ "prefix": null,
71
+ "bos_token_id": 100257,
72
+ "pad_token_id": null,
73
+ "eos_token_id": 100257,
74
+ "sep_token_id": null,
75
+ "decoder_start_token_id": null,
76
+ "task_specific_params": null,
77
+ "problem_type": null,
78
+ "_name_or_path": "/Users/socialkeyboard/.cache/huggingface/hub/models--stabilityai--stablelm-2-zephyr-1_6b/snapshots/e795a4efc68858019a199894d100a517c24ff21b",
79
+ "transformers_version": "4.37.1",
80
+ "auto_map": {
81
+ "AutoConfig": "configuration_stablelm_epoch.StableLMEpochConfig",
82
+ "AutoModelForCausalLM": "modeling_stablelm_epoch.StableLMEpochForCausalLM"
83
+ },
84
+ "model_type": "stablelm_epoch",
85
+ "num_heads": 32,
86
+ "rotary_scaling_factor": 1.0
87
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>"
5
+ }
tokenization_arcade100k.py ADDED
@@ -0,0 +1,292 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright (c) 2023 Alibaba Cloud & Stability AI.
3
+ #
4
+ # Tongyi Qianwen LICENSE AGREEMENT:
5
+ # https://github.com/QwenLM/Qwen/blob/5aa84bdfd3237b37f01bc88cd49b3279b9a71d0b/Tongyi%20Qianwen%20LICENSE%20AGREEMENT
6
+ """Tokenization classes for Arcade100k."""
7
+
8
+ import base64
9
+ import os
10
+ import unicodedata
11
+ from typing import Collection, Dict, List, Set, Tuple, Union
12
+
13
+ import tiktoken
14
+ from transformers.utils import logging
15
+ from transformers import PreTrainedTokenizer, AddedToken
16
+
17
+ logger = logging.get_logger(__name__)
18
+
19
+ VOCAB_FILES_NAMES = {"vocab_file": "arcade100k.tiktoken"}
20
+ NAME = "arcade100k"
21
+
22
+
23
+ def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
24
+ with open(tiktoken_bpe_file, "rb") as f:
25
+ contents = f.read()
26
+ return {
27
+ base64.b64decode(token): int(rank)
28
+ for token, rank in (line.split() for line in contents.splitlines() if line)
29
+ }
30
+
31
+
32
+ ENDOFTEXT = "<|endoftext|>"
33
+ FIM = [
34
+ "<|fim_prefix|>",
35
+ "<|fim_middle|>",
36
+ "<|fim_suffix|>",
37
+ "<|fim_pad|>",
38
+ ]
39
+ # `StarCoder` Tokens
40
+ CODE = [
41
+ "<gh_stars>",
42
+ "<filename>",
43
+ "<issue_start>",
44
+ "<issue_comment>",
45
+ "<issue_closed>",
46
+ "<jupyter_start>",
47
+ "<jupyter_text>",
48
+ "<jupyter_code>",
49
+ "<jupyter_output>",
50
+ "<empty_output>",
51
+ "<commit_before>",
52
+ "<commit_msg>",
53
+ "<commit_after>",
54
+ "<reponame>",
55
+ ]
56
+ CHAT = [
57
+ "<|im_start|>", # Chat: Input message start
58
+ "<|im_end|>", # Chat: Input message end
59
+ ]
60
+ PAUSE = "<|pause|>" # Think before you speak (https://arxiv.org/abs/2310.02226)
61
+ REGISTERS = [
62
+ f"<|reg{i}|>" for i in range(0, 8)
63
+ ] # Register 0 sink token (https://arxiv.org/abs/2309.17453)
64
+ ENDOFPROMPT = "<|endofprompt|>"
65
+ SPECIAL_TOKENS_NAMES = (
66
+ [ENDOFTEXT]
67
+ + FIM
68
+ + CODE
69
+ + [ENDOFPROMPT]
70
+ + CHAT
71
+ + [PAUSE]
72
+ + REGISTERS
73
+ + ["<|extra0|>"]
74
+ )
75
+ START_ID = 100257
76
+ SPECIAL_TOKENS = {t: START_ID + i for i, t in enumerate(SPECIAL_TOKENS_NAMES)}
77
+
78
+
79
+ def _arcade100k(vocab_file: str):
80
+ mergeable_ranks = _load_tiktoken_bpe(vocab_file)
81
+
82
+ return {
83
+ "name": NAME,
84
+ "pat_str": r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""",
85
+ "mergeable_ranks": mergeable_ranks,
86
+ "special_tokens": SPECIAL_TOKENS,
87
+ }
88
+
89
+
90
+ class Arcade100kTokenizer(PreTrainedTokenizer):
91
+ """
92
+ Construct a Arcade100k tokenizer backed by `tiktoken`.
93
+
94
+ Args:
95
+ vocab_file (`str`):
96
+ Path to the vocabulary file.
97
+ errors (`str`, *optional*, defaults to `"replace"`):
98
+ How to handle errors in decoding UTF-8 byte sequences.
99
+ WARNING: the default behaviour of this function is lossy, since decoded bytes are not
100
+ guaranteed to be valid UTF-8. You can control this behaviour using the `errors` parameter,
101
+ for instance, setting `errors=strict`.
102
+ """
103
+
104
+ vocab_files_names = VOCAB_FILES_NAMES
105
+ model_input_names = ["input_ids", "attention_mask"]
106
+
107
+ def __init__(
108
+ self,
109
+ vocab_file: str,
110
+ errors: str = "replace",
111
+ **kwargs,
112
+ ):
113
+ super().__init__(errors=errors, **kwargs)
114
+ self.errors = errors
115
+
116
+ self._tiktoken_config = _arcade100k(vocab_file)
117
+ self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
118
+
119
+ # TODO: Remove this assertion
120
+ assert (
121
+ len(self.tokenizer._mergeable_ranks)
122
+ + len(self.tokenizer._special_tokens)
123
+ + 1
124
+ == self.tokenizer.n_vocab
125
+ ), f"{len(self.tokenizer._mergeable_ranks) + len(self.tokenizer._special_tokens)} != {self.tokenizer.n_vocab} in encoding"
126
+
127
+ self.decoder = {i: n for n, i in self.tokenizer._mergeable_ranks.items()}
128
+ self.decoder.update({i: n for n, i in self.tokenizer._special_tokens.items()})
129
+ # Provide default `eos_token` and `pad_token`
130
+ if self.eos_token is None:
131
+ self.eos_token = self.decoder[self.tokenizer.eot_token]
132
+ if self.pad_token is None:
133
+ self.pad_token = self.decoder[self.tokenizer.pad_token]
134
+
135
+ # Expose for convenience
136
+ self.mergeable_ranks = self.tokenizer._mergeable_ranks
137
+ self.special_tokens = self.tokenizer._special_tokens
138
+
139
+ def __len__(self):
140
+ return self.tokenizer.n_vocab
141
+
142
+ def __getstate__(self):
143
+ # Required for `pickle` support
144
+ state = self.__dict__.copy()
145
+ del state["tokenizer"]
146
+ return state
147
+
148
+ def __setstate__(self, state):
149
+ self.__dict__.update(state)
150
+ self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
151
+
152
+ @property
153
+ def vocab_size(self):
154
+ return self.tokenizer.n_vocab
155
+
156
+ def get_vocab(self) -> Dict[bytes, int]:
157
+ return self.tokenizer._mergeable_ranks
158
+
159
+ def convert_tokens_to_ids(
160
+ self, tokens: Union[bytes, str, List[Union[bytes, str]]]
161
+ ) -> List[int]:
162
+ ids = []
163
+ if isinstance(tokens, (str, bytes)):
164
+ if tokens in self.tokenizer._special_tokens:
165
+ return self.tokenizer._special_tokens[tokens]
166
+ else:
167
+ return self.tokenizer._mergeable_ranks.get(tokens)
168
+ for token in tokens:
169
+ if token in self.tokenizer._special_tokens:
170
+ ids.append(self.tokenizer._special_tokens[token])
171
+ else:
172
+ ids.append(self.tokenizer._mergeable_ranks.get(token))
173
+ return ids
174
+
175
+ def _add_tokens(
176
+ self,
177
+ new_tokens: Union[List[str], List[AddedToken]],
178
+ special_tokens: bool = False,
179
+ ) -> int:
180
+ if not special_tokens and new_tokens:
181
+ raise ValueError("Adding regular tokens is not supported")
182
+ for token in new_tokens:
183
+ surface_form = token.content if isinstance(token, AddedToken) else token
184
+ if surface_form not in SPECIAL_TOKENS:
185
+ raise ValueError("Adding unknown special tokens is not supported")
186
+ return 0
187
+
188
+ def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
189
+ """
190
+ Save only the vocabulary of the tokenizer (vocabulary).
191
+
192
+ Returns:
193
+ `Tuple(str)`: Paths to the files saved.
194
+ """
195
+ file_path = os.path.join(save_directory, "arcade100k.tiktoken")
196
+ with open(file_path, "w", encoding="utf8") as w:
197
+ for k, v in self.tokenizer._mergeable_ranks.items():
198
+ line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
199
+ w.write(line)
200
+ return (file_path,)
201
+
202
+ def tokenize(
203
+ self,
204
+ text: str,
205
+ allowed_special: Union[Set, str] = "all",
206
+ disallowed_special: Union[Collection, str] = (),
207
+ **kwargs,
208
+ ) -> List[Union[bytes, str]]:
209
+ """
210
+ Converts a string in a sequence of tokens.
211
+
212
+ Args:
213
+ text (`str`):
214
+ The sequence to be encoded.
215
+ allowed_special (`Literal["all"]` or `set`):
216
+ The surface forms of the tokens to be encoded as special tokens in regular texts.
217
+ Default to "all".
218
+ disallowed_special (`Literal["all"]` or `Collection`):
219
+ The surface forms of the tokens that should not be in regular texts and trigger errors.
220
+ Default to an empty tuple.
221
+
222
+ kwargs (additional keyword arguments, *optional*):
223
+ Will be passed to the underlying model specific encode method.
224
+
225
+ Returns:
226
+ `List[bytes|str]`: The list of tokens.
227
+ """
228
+ tokens = []
229
+ text = unicodedata.normalize("NFC", text)
230
+
231
+ # this implementation takes a detour: text -> token id -> token surface forms
232
+ for t in self.tokenizer.encode(
233
+ text, allowed_special=allowed_special, disallowed_special=disallowed_special
234
+ ):
235
+ tokens.append(self.decoder[t])
236
+ return tokens
237
+
238
+ def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
239
+ """
240
+ Converts a sequence of tokens in a single string.
241
+ """
242
+ text = ""
243
+ temp = b""
244
+ for t in tokens:
245
+ if isinstance(t, str):
246
+ if temp:
247
+ text += temp.decode("utf-8", errors=self.errors)
248
+ temp = b""
249
+ text += t
250
+ elif isinstance(t, bytes):
251
+ temp += t
252
+ else:
253
+ raise TypeError("token should only be of type types or str")
254
+ if temp:
255
+ text += temp.decode("utf-8", errors=self.errors)
256
+ return text
257
+
258
+ def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
259
+ """Converts an id to a token, special tokens included"""
260
+ if index in self.decoder:
261
+ return self.decoder[index]
262
+ raise ValueError("unknown ids")
263
+
264
+ def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
265
+ """Converts a token to an id using the vocab, special tokens included"""
266
+ if token in self.tokenizer._special_tokens:
267
+ return self.tokenizer._special_tokens[token]
268
+ if token in self.tokenizer._mergeable_ranks:
269
+ return self.tokenizer._mergeable_ranks[token]
270
+ raise ValueError("unknown token")
271
+
272
+ def _tokenize(self, text: str, **kwargs):
273
+ """
274
+ Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
275
+ vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
276
+
277
+ Do NOT take care of added tokens.
278
+ """
279
+ raise NotImplementedError
280
+
281
+ def _decode(
282
+ self,
283
+ token_ids: Union[int, List[int]],
284
+ skip_special_tokens: bool = False,
285
+ errors: str = None,
286
+ **kwargs,
287
+ ) -> str:
288
+ if isinstance(token_ids, int):
289
+ token_ids = [token_ids]
290
+ if skip_special_tokens:
291
+ token_ids = [i for i in token_ids if i < self.tokenizer.eot_token]
292
+ return self.tokenizer.decode(token_ids)
tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {},
3
+ "auto_map": {
4
+ "AutoTokenizer": [
5
+ "tokenization_arcade100k.Arcade100kTokenizer",
6
+ null
7
+ ]
8
+ },
9
+ "bos_token": "<|endoftext|>",
10
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
11
+ "clean_up_tokenization_spaces": true,
12
+ "eos_token": "<|endoftext|>",
13
+ "errors": "replace",
14
+ "model_max_length": 2048,
15
+ "pad_token": "<|endoftext|>",
16
+ "tokenizer_class": "Arcade100kTokenizer"
17
+ }
weights.00.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de4a5fa26a83e9bada9f524000c663859a4cb8ccbabc4e05d7f89bb796cbba4e
3
+ size 3289069095