Hugging Face's logo

mlx-community
/
dbrx-instruct-4bit

MLX
Safetensors
dbrx
custom_code
Model card Files Community
8

fix the incorrect config.json

#5
by mzbac - opened
base: refs/heads/main
←
from: refs/pr/5
Discussion Files changed
+39
-222
Files changed (1) hide show
  1. config.json +39 -222
config.json CHANGED
@@ -1,224 +1,41 @@
1
  {
2
- "add_cross_attention": false,
3
- "architectures": [
4
- "DbrxForCausalLM"
5
- ],
6
- "attn_config": {
7
- "return_dict": true,
8
- "output_hidden_states": false,
9
- "output_attentions": false,
10
- "torchscript": false,
11
- "torch_dtype": null,
12
- "use_bfloat16": false,
13
- "tf_legacy_loss": false,
14
- "pruned_heads": {},
15
- "tie_word_embeddings": true,
16
- "chunk_size_feed_forward": 0,
17
- "is_encoder_decoder": false,
18
- "is_decoder": false,
19
- "cross_attention_hidden_size": null,
20
- "add_cross_attention": false,
21
- "tie_encoder_decoder": false,
22
- "max_length": 20,
23
- "min_length": 0,
24
- "do_sample": false,
25
- "early_stopping": false,
26
- "num_beams": 1,
27
- "num_beam_groups": 1,
28
- "diversity_penalty": 0.0,
29
- "temperature": 1.0,
30
- "top_k": 50,
31
- "top_p": 1.0,
32
- "typical_p": 1.0,
33
- "repetition_penalty": 1.0,
34
- "length_penalty": 1.0,
35
- "no_repeat_ngram_size": 0,
36
- "encoder_no_repeat_ngram_size": 0,
37
- "bad_words_ids": null,
38
- "num_return_sequences": 1,
39
- "output_scores": false,
40
- "return_dict_in_generate": false,
41
- "forced_bos_token_id": null,
42
- "forced_eos_token_id": null,
43
- "remove_invalid_values": false,
44
- "exponential_decay_length_penalty": null,
45
- "suppress_tokens": null,
46
- "begin_suppress_tokens": null,
47
- "architectures": null,
48
- "finetuning_task": null,
49
- "id2label": {
50
- "0": "LABEL_0",
51
- "1": "LABEL_1"
52
- },
53
- "label2id": {
54
- "LABEL_0": 0,
55
- "LABEL_1": 1
56
- },
57
- "tokenizer_class": null,
58
- "prefix": null,
59
- "bos_token_id": null,
60
- "pad_token_id": null,
61
- "eos_token_id": null,
62
- "sep_token_id": null,
63
- "decoder_start_token_id": null,
64
- "task_specific_params": null,
65
- "problem_type": null,
66
- "_name_or_path": "",
67
- "model_type": "",
68
- "attn_pdrop": 0,
69
- "clip_qkv": 8,
70
- "kv_n_heads": 8,
71
- "rope_theta": 500000
72
- },
73
- "auto_map": {
74
- "AutoConfig": "configuration_dbrx.DbrxConfig",
75
- "AutoModelForCausalLM": "modeling_dbrx.DbrxForCausalLM"
76
- },
77
- "bad_words_ids": null,
78
- "begin_suppress_tokens": null,
79
- "bos_token_id": null,
80
- "chunk_size_feed_forward": 0,
81
- "cross_attention_hidden_size": null,
82
- "d_model": 6144,
83
- "decoder_start_token_id": null,
84
- "diversity_penalty": 0.0,
85
- "do_sample": false,
86
- "early_stopping": false,
87
- "emb_pdrop": 0.0,
88
- "encoder_no_repeat_ngram_size": 0,
89
- "eos_token_id": null,
90
- "exponential_decay_length_penalty": null,
91
- "ffn_config": {
92
- "return_dict": true,
93
- "output_hidden_states": false,
94
- "output_attentions": false,
95
- "torchscript": false,
96
- "torch_dtype": null,
97
- "use_bfloat16": false,
98
- "tf_legacy_loss": false,
99
- "pruned_heads": {},
100
- "tie_word_embeddings": true,
101
- "chunk_size_feed_forward": 0,
102
- "is_encoder_decoder": false,
103
- "is_decoder": false,
104
- "cross_attention_hidden_size": null,
105
- "add_cross_attention": false,
106
- "tie_encoder_decoder": false,
107
- "max_length": 20,
108
- "min_length": 0,
109
- "do_sample": false,
110
- "early_stopping": false,
111
- "num_beams": 1,
112
- "num_beam_groups": 1,
113
- "diversity_penalty": 0.0,
114
- "temperature": 1.0,
115
- "top_k": 50,
116
- "top_p": 1.0,
117
- "typical_p": 1.0,
118
- "repetition_penalty": 1.0,
119
- "length_penalty": 1.0,
120
- "no_repeat_ngram_size": 0,
121
- "encoder_no_repeat_ngram_size": 0,
122
- "bad_words_ids": null,
123
- "num_return_sequences": 1,
124
- "output_scores": false,
125
- "return_dict_in_generate": false,
126
- "forced_bos_token_id": null,
127
- "forced_eos_token_id": null,
128
- "remove_invalid_values": false,
129
- "exponential_decay_length_penalty": null,
130
- "suppress_tokens": null,
131
- "begin_suppress_tokens": null,
132
- "architectures": null,
133
- "finetuning_task": null,
134
- "id2label": {
135
- "0": "LABEL_0",
136
- "1": "LABEL_1"
137
- },
138
- "label2id": {
139
- "LABEL_0": 0,
140
- "LABEL_1": 1
141
- },
142
- "tokenizer_class": null,
143
- "prefix": null,
144
- "bos_token_id": null,
145
- "pad_token_id": null,
146
- "eos_token_id": null,
147
- "sep_token_id": null,
148
- "decoder_start_token_id": null,
149
- "task_specific_params": null,
150
- "problem_type": null,
151
- "_name_or_path": "",
152
- "ffn_act_fn": {
153
- "name": "silu"
154
- },
155
- "ffn_hidden_size": 10752,
156
- "moe_num_experts": 16,
157
- "moe_top_k": 4,
158
- "moe_jitter_eps": 0,
159
- "moe_loss_weight": 0.05,
160
- "moe_normalize_expert_weights": 1,
161
- "uniform_expert_assignment": false,
162
- "model_type": ""
163
- },
164
- "finetuning_task": null,
165
- "forced_bos_token_id": null,
166
- "forced_eos_token_id": null,
167
- "id2label": {
168
- "0": "LABEL_0",
169
- "1": "LABEL_1"
170
- },
171
- "initializer_range": 0.02,
172
- "is_decoder": false,
173
- "is_encoder_decoder": false,
174
- "label2id": {
175
- "LABEL_0": 0,
176
- "LABEL_1": 1
177
- },
178
- "length_penalty": 1.0,
179
- "max_length": 20,
180
- "max_seq_len": 32768,
181
- "min_length": 0,
182
- "model_type": "dbrx",
183
- "n_heads": 48,
184
- "n_layers": 40,
185
- "no_repeat_ngram_size": 0,
186
- "num_beam_groups": 1,
187
- "num_beams": 1,
188
- "num_return_sequences": 1,
189
- "output_attentions": false,
190
- "output_hidden_states": false,
191
- "output_router_logits": false,
192
- "output_scores": false,
193
- "pad_token_id": null,
194
- "prefix": null,
195
- "problem_type": null,
196
- "pruned_heads": {},
197
- "quantization": {
198
- "group_size": 64,
199
- "bits": 4
200
- },
201
- "remove_invalid_values": false,
202
- "repetition_penalty": 1.0,
203
- "resid_pdrop": 0.0,
204
- "return_dict": true,
205
- "return_dict_in_generate": false,
206
- "router_aux_loss_coef": 0.05,
207
- "sep_token_id": null,
208
- "suppress_tokens": null,
209
- "task_specific_params": null,
210
- "temperature": 1.0,
211
- "tf_legacy_loss": false,
212
- "tie_encoder_decoder": false,
213
- "tie_word_embeddings": false,
214
- "tokenizer_class": null,
215
- "top_k": 50,
216
- "top_p": 1.0,
217
- "torch_dtype": "bfloat16",
218
- "torchscript": false,
219
- "transformers_version": "4.39.2",
220
- "typical_p": 1.0,
221
- "use_bfloat16": false,
222
- "use_cache": true,
223
- "vocab_size": 100352
224
  }
 
1
  {
2
+ "architectures": [
3
+ "DbrxForCausalLM"
4
+ ],
5
+ "attn_config": {
6
+ "clip_qkv": 8,
7
+ "kv_n_heads": 8,
8
+ "model_type": "",
9
+ "rope_theta": 500000
10
+ },
11
+ "auto_map": {
12
+ "AutoConfig": "configuration_dbrx.DbrxConfig",
13
+ "AutoModelForCausalLM": "modeling_dbrx.DbrxForCausalLM"
14
+ },
15
+ "d_model": 6144,
16
+ "emb_pdrop": 0.0,
17
+ "ffn_config": {
18
+ "ffn_hidden_size": 10752,
19
+ "model_type": "",
20
+ "moe_jitter_eps": 0,
21
+ "moe_loss_weight": 0.05,
22
+ "moe_num_experts": 16,
23
+ "moe_top_k": 4,
24
+ "ffn_act_fn": {
25
+ "name": "silu"
26
+ }
27
+ },
28
+ "initializer_range": 0.02,
29
+ "max_seq_len": 32768,
30
+ "model_type": "dbrx",
31
+ "n_heads": 48,
32
+ "n_layers": 40,
33
+ "output_router_logits": false,
34
+ "resid_pdrop": 0.0,
35
+ "router_aux_loss_coef": 0.05,
36
+ "tie_word_embeddings": false,
37
+ "torch_dtype": "bfloat16",
38
+ "transformers_version": "4.38.2",
39
+ "use_cache": true,
40
+ "vocab_size": 100352
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41
  }