ChibuUkachi commited on
Commit
7a95a38
·
verified ·
1 Parent(s): 63d70c9

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. added_tokens.json +56 -0
  3. chat_template.jinja +159 -0
  4. config.json +470 -0
  5. configuration_minimax_m2.py +200 -0
  6. generation_config.json +8 -0
  7. merges.txt +0 -0
  8. model-00001-of-00047.safetensors +3 -0
  9. model-00002-of-00047.safetensors +3 -0
  10. model-00003-of-00047.safetensors +3 -0
  11. model-00004-of-00047.safetensors +3 -0
  12. model-00005-of-00047.safetensors +3 -0
  13. model-00006-of-00047.safetensors +3 -0
  14. model-00007-of-00047.safetensors +3 -0
  15. model-00008-of-00047.safetensors +3 -0
  16. model-00009-of-00047.safetensors +3 -0
  17. model-00010-of-00047.safetensors +3 -0
  18. model-00011-of-00047.safetensors +3 -0
  19. model-00012-of-00047.safetensors +3 -0
  20. model-00013-of-00047.safetensors +3 -0
  21. model-00014-of-00047.safetensors +3 -0
  22. model-00015-of-00047.safetensors +3 -0
  23. model-00016-of-00047.safetensors +3 -0
  24. model-00017-of-00047.safetensors +3 -0
  25. model-00018-of-00047.safetensors +3 -0
  26. model-00019-of-00047.safetensors +3 -0
  27. model-00020-of-00047.safetensors +3 -0
  28. model-00021-of-00047.safetensors +3 -0
  29. model-00022-of-00047.safetensors +3 -0
  30. model-00023-of-00047.safetensors +3 -0
  31. model-00024-of-00047.safetensors +3 -0
  32. model-00025-of-00047.safetensors +3 -0
  33. model-00026-of-00047.safetensors +3 -0
  34. model-00027-of-00047.safetensors +3 -0
  35. model-00028-of-00047.safetensors +3 -0
  36. model-00029-of-00047.safetensors +3 -0
  37. model-00030-of-00047.safetensors +3 -0
  38. model-00031-of-00047.safetensors +3 -0
  39. model-00032-of-00047.safetensors +3 -0
  40. model-00033-of-00047.safetensors +3 -0
  41. model-00034-of-00047.safetensors +3 -0
  42. model-00035-of-00047.safetensors +3 -0
  43. model-00036-of-00047.safetensors +3 -0
  44. model-00037-of-00047.safetensors +3 -0
  45. model-00038-of-00047.safetensors +3 -0
  46. model-00039-of-00047.safetensors +3 -0
  47. model-00040-of-00047.safetensors +3 -0
  48. model-00041-of-00047.safetensors +3 -0
  49. model-00042-of-00047.safetensors +3 -0
  50. model-00043-of-00047.safetensors +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
added_tokens.json ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</minimax:tool_call>": 200053,
3
+ "</think>": 200051,
4
+ "<add_file>": 200036,
5
+ "<code_context>": 200043,
6
+ "<code_interpreter>": 200023,
7
+ "<commit_after>": 200018,
8
+ "<commit_before>": 200016,
9
+ "<commit_message>": 200040,
10
+ "<commit_msg>": 200017,
11
+ "<delete_file>": 200037,
12
+ "<edit_file>": 200039,
13
+ "<empty_output>": 200015,
14
+ "<empty_source_file>": 200041,
15
+ "<file_content>": 200044,
16
+ "<file_sep>": 200049,
17
+ "<filename>": 200006,
18
+ "<filepath>": 200048,
19
+ "<fim_middle>": 200002,
20
+ "<fim_pad>": 200004,
21
+ "<fim_prefix>": 200001,
22
+ "<fim_suffix>": 200003,
23
+ "<function_call>": 200022,
24
+ "<gh_stars>": 200007,
25
+ "<issue_closed>": 200010,
26
+ "<issue_comment>": 200009,
27
+ "<issue_start>": 200008,
28
+ "<jupyter_code>": 200013,
29
+ "<jupyter_error>": 200035,
30
+ "<jupyter_output>": 200014,
31
+ "<jupyter_start>": 200011,
32
+ "<jupyter_text>": 200012,
33
+ "<minimax:tool_call>": 200052,
34
+ "<pr_start>": 200046,
35
+ "<rename_file>": 200038,
36
+ "<repo_struct>": 200042,
37
+ "<reponame>": 200005,
38
+ "<review_comment>": 200047,
39
+ "<source_files>": 200045,
40
+ "<think>": 200050,
41
+ "[e~[": 200020,
42
+ "]!d~[": 200021,
43
+ "]!p~[": 200000,
44
+ "]<]end of image[>[": 200030,
45
+ "]<]end of speech[>[": 200028,
46
+ "]<]end of video[>[": 200032,
47
+ "]<]image[>[": 200025,
48
+ "]<]speech[>[": 200024,
49
+ "]<]start of image[>[": 200029,
50
+ "]<]start of speech[>[": 200027,
51
+ "]<]start of video[>[": 200031,
52
+ "]<]video[>[": 200026,
53
+ "]<]vision pad[>[": 200033,
54
+ "]~!b[": 200034,
55
+ "]~b]": 200019
56
+ }
chat_template.jinja ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {# ----------‑‑‑ special token variables ‑‑‑---------- #}
2
+ {%- set toolcall_begin_token = '<minimax:tool_call>' -%}
3
+ {%- set toolcall_end_token = '</minimax:tool_call>' -%}
4
+ {#- Tool Rendering Functions ============================================== -#}
5
+ {%- macro render_tool_namespace(namespace_name, tool_list) -%}
6
+ {%- for tool in tool_list -%}
7
+ <tool>{{ tool.function | tojson(ensure_ascii=False) }}</tool>
8
+ {% endfor -%}
9
+ {%- endmacro -%}
10
+ {%- macro visible_text(content) -%}
11
+ {%- if content is string -%}
12
+ {{ content }}
13
+ {%- elif content is iterable and content is not mapping -%}
14
+ {%- for item in content -%}
15
+ {%- if item is mapping and item.type == 'text' -%}
16
+ {{- item.text }}
17
+ {%- elif item is string -%}
18
+ {{- item }}
19
+ {%- endif -%}
20
+ {%- endfor -%}
21
+ {%- else -%}
22
+ {{- content }}
23
+ {%- endif -%}
24
+ {%- endmacro -%}
25
+ {#- System Message Construction ============================================ -#}
26
+ {%- macro build_system_message(system_message) -%}
27
+ {%- if system_message and system_message.content -%}
28
+ {{- visible_text(system_message.content) }}
29
+ {%- else -%}
30
+ {%- if model_identity is not defined -%}
31
+ {%- set model_identity = "You are a helpful assistant. Your name is MiniMax-M2.5 and is built by MiniMax." -%}
32
+ {%- endif -%}
33
+ {{- model_identity }}
34
+ {%- endif -%}
35
+
36
+ {#- Handle current_date -#}
37
+ {%- if system_message and system_message.current_date -%}
38
+ {{- '\n' ~ 'Current date: ' + system_message.current_date }}
39
+ {%- endif -%}
40
+ {#- Handle current_location -#}
41
+ {%- if system_message and system_message.current_location -%}
42
+ {{- '\n' ~ 'Current location: ' + system_message.current_location }}
43
+ {%- endif -%}
44
+ {%- endmacro -%}
45
+ {#- Main Template Logic ================================================= -#}
46
+ {#- Extract system message (only first message if it's system) -#}
47
+ {%- set system_message = none -%}
48
+ {%- set conversation_messages = messages -%}
49
+ {%- if messages and messages[0].role == "system" -%}
50
+ {%- set system_message = messages[0] -%}
51
+ {%- set conversation_messages = messages[1:] -%}
52
+ {%- endif -%}
53
+ {#- Get the last user message turn, for interleved thinking -#}
54
+ {%- set ns = namespace(last_user_index=-1) %}
55
+ {% for m in conversation_messages %}
56
+ {%- if m.role == 'user' %}
57
+ {% set ns.last_user_index = loop.index0 -%}
58
+ {%- endif %}
59
+ {%- endfor %}
60
+ {#- Render system message -#}
61
+ {{- ']~!b[' ~ ']~b]system' ~ '\n' }}
62
+ {{- build_system_message(system_message) }}
63
+ {#- Render tools if available -#}
64
+ {%- if tools -%}
65
+ {{- '\n\n' ~ '# Tools' ~ '\n' ~ 'You may call one or more tools to assist with the user query.\nHere are the tools available in JSONSchema format:' ~ '\n' }}
66
+ {{- '\n' ~ '<tools>' ~ '\n' }}
67
+ {{- render_tool_namespace("functions", tools) }}
68
+ {{- '</tools>' ~ '\n\n' }}
69
+ {{- 'When making tool calls, use XML format to invoke tools and pass parameters:' ~ '\n' }}
70
+ {{- '\n' ~ toolcall_begin_token }}
71
+ <invoke name="tool-name-1">
72
+ <parameter name="param-key-1">param-value-1</parameter>
73
+ <parameter name="param-key-2">param-value-2</parameter>
74
+ ...
75
+ </invoke>
76
+ {{- '\n' ~ toolcall_end_token }}
77
+ {%- endif -%}
78
+ {{- '[e~[\n' }}
79
+
80
+ {#- Render messages -#}
81
+ {%- set last_tool_call = namespace(name=none) -%}
82
+ {%- for message in conversation_messages -%}
83
+ {%- if message.role == 'assistant' -%}
84
+ {#- Only render reasoning_content if no user message follows -#}
85
+ {{- ']~b]ai' ~ '\n' }}
86
+
87
+ {%- set reasoning_content = '' %}
88
+ {%- set content = visible_text(message.content) %}
89
+ {%- if message.reasoning_content is string %}
90
+ {%- set reasoning_content = message.reasoning_content %}
91
+ {%- else %}
92
+ {%- if '</think>' in content %}
93
+ {%- set reasoning_content = content.split('</think>')[0].strip('\n').split('<think>')[-1].strip('\n') %}
94
+ {%- set content = content.split('</think>')[-1].strip('\n') %}
95
+ {%- endif %}
96
+ {%- endif %}
97
+ {%- if reasoning_content and loop.index0 > ns.last_user_index -%}
98
+ {{- '<think>' ~ '\n' ~ reasoning_content ~ '\n' ~ '</think>' ~ '\n\n' }}
99
+ {%- endif -%}
100
+ {%- if content -%}
101
+ {{- content }}
102
+ {%- endif -%}
103
+ {%- if message.tool_calls -%}
104
+ {{- '\n' ~ toolcall_begin_token ~ '\n' }}
105
+
106
+ {%- for tool_call in message.tool_calls -%}
107
+ {%- if tool_call.function %}
108
+ {%- set tool_call = tool_call.function %}
109
+ {%- endif %}
110
+ {{- '<invoke name="' + tool_call.name + '">' }}
111
+ {% set _args = tool_call.arguments %}
112
+ {%- for k, v in _args.items() %}
113
+ {{- '<parameter name="' + k + '">' }}
114
+ {{- v | tojson(ensure_ascii=False) if v is not string else v }}
115
+ {{- '</parameter>' }}
116
+ {% endfor %}
117
+ {{- '</invoke>' ~ '\n' }}
118
+ {%- endfor -%}
119
+
120
+ {{- toolcall_end_token}}
121
+ {%- set last_tool_call.name = message.tool_calls[-1].name -%}
122
+ {%- else -%}
123
+ {%- set last_tool_call.name = none -%}
124
+ {%- endif -%}
125
+ {{- '[e~[' ~ '\n' }}
126
+
127
+ {%- elif message.role == 'tool' -%}
128
+ {%- if last_tool_call.name is none -%}
129
+ {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }}
130
+ {%- endif -%}
131
+ {%- if loop.first or (conversation_messages[loop.index0 - 1].role != 'tool') -%}
132
+ {{- ']~b]tool' }}
133
+ {%- endif -%}
134
+ {%- if message.content is string -%}
135
+ {{- '\n<response>' }}
136
+ {{- message.content }}
137
+ {{- '</response>' }}
138
+ {%- else -%}
139
+ {%- for tr in message.content -%}
140
+ {{- '\n<response>' }}
141
+ {{- tr.output if tr.output is defined else (tr.text if tr.type == 'text' and tr.text is defined else tr) }}
142
+ {{- '\n</response>' }}
143
+ {%- endfor -%}
144
+ {%- endif -%}
145
+ {%- if loop.last or (conversation_messages[loop.index0 + 1].role != 'tool') -%}
146
+ {{- '[e~[\n' -}}
147
+ {%- endif -%}
148
+
149
+ {%- elif message.role == 'user' -%}
150
+ {{- ']~b]user' ~ '\n' }}
151
+ {{- visible_text(message.content) }}
152
+ {{- '[e~[' ~ '\n' }}
153
+ {%- endif -%}
154
+ {%- endfor -%}
155
+
156
+ {#- Generation prompt -#}
157
+ {%- if add_generation_prompt -%}
158
+ {{- ']~b]ai' ~ '\n' ~ '<think>' ~ '\n' }}
159
+ {%- endif -%}
config.json ADDED
@@ -0,0 +1,470 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MiniMaxM2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "attn_type_list": [
7
+ 1,
8
+ 1,
9
+ 1,
10
+ 1,
11
+ 1,
12
+ 1,
13
+ 1,
14
+ 1,
15
+ 1,
16
+ 1,
17
+ 1,
18
+ 1,
19
+ 1,
20
+ 1,
21
+ 1,
22
+ 1,
23
+ 1,
24
+ 1,
25
+ 1,
26
+ 1,
27
+ 1,
28
+ 1,
29
+ 1,
30
+ 1,
31
+ 1,
32
+ 1,
33
+ 1,
34
+ 1,
35
+ 1,
36
+ 1,
37
+ 1,
38
+ 1,
39
+ 1,
40
+ 1,
41
+ 1,
42
+ 1,
43
+ 1,
44
+ 1,
45
+ 1,
46
+ 1,
47
+ 1,
48
+ 1,
49
+ 1,
50
+ 1,
51
+ 1,
52
+ 1,
53
+ 1,
54
+ 1,
55
+ 1,
56
+ 1,
57
+ 1,
58
+ 1,
59
+ 1,
60
+ 1,
61
+ 1,
62
+ 1,
63
+ 1,
64
+ 1,
65
+ 1,
66
+ 1,
67
+ 1,
68
+ 1
69
+ ],
70
+ "auto_map": {
71
+ "AutoConfig": "configuration_minimax_m2.MiniMaxM2Config",
72
+ "AutoModelForCausalLM": "modeling_minimax_m2.MiniMaxM2ForCausalLM"
73
+ },
74
+ "bos_token_id": 1,
75
+ "dtype": "bfloat16",
76
+ "eos_token_id": 2,
77
+ "head_dim": 128,
78
+ "hidden_act": "silu",
79
+ "hidden_size": 3072,
80
+ "initializer_range": 0.02,
81
+ "intermediate_size": 1536,
82
+ "max_position_embeddings": 196608,
83
+ "model_type": "minimax_m2",
84
+ "mtp_transformer_layers": 1,
85
+ "num_attention_heads": 48,
86
+ "num_experts_per_tok": 8,
87
+ "num_hidden_layers": 62,
88
+ "num_key_value_heads": 8,
89
+ "num_local_experts": 256,
90
+ "num_mtp_modules": 3,
91
+ "output_router_logits": false,
92
+ "partial_rotary_factor": 0.5,
93
+ "qk_norm_type": "per_layer",
94
+ "quantization_config": {
95
+ "config_groups": {
96
+ "group_0": {
97
+ "format": "int-quantized",
98
+ "input_activations": {
99
+ "actorder": null,
100
+ "block_structure": null,
101
+ "dynamic": true,
102
+ "group_size": null,
103
+ "num_bits": 8,
104
+ "observer": null,
105
+ "observer_kwargs": {},
106
+ "scale_dtype": null,
107
+ "strategy": "token",
108
+ "symmetric": true,
109
+ "type": "int",
110
+ "zp_dtype": null
111
+ },
112
+ "output_activations": null,
113
+ "targets": [
114
+ "re:.*block_sparse_moe\\.experts\\.\\d+\\.w[1-3]$",
115
+ "re:.*mlp\\.experts\\.\\d+\\.(gate|up|gate_up|down)_proj$"
116
+ ],
117
+ "weights": {
118
+ "actorder": null,
119
+ "block_structure": null,
120
+ "dynamic": false,
121
+ "group_size": null,
122
+ "num_bits": 8,
123
+ "observer": "mse",
124
+ "observer_kwargs": {},
125
+ "scale_dtype": null,
126
+ "strategy": "channel",
127
+ "symmetric": true,
128
+ "type": "int",
129
+ "zp_dtype": null
130
+ }
131
+ }
132
+ },
133
+ "format": "int-quantized",
134
+ "global_compression_ratio": null,
135
+ "ignore": [
136
+ "model.layers.0.self_attn.q_proj",
137
+ "model.layers.0.self_attn.k_proj",
138
+ "model.layers.0.self_attn.v_proj",
139
+ "model.layers.0.self_attn.o_proj",
140
+ "model.layers.0.block_sparse_moe.gate",
141
+ "model.layers.1.self_attn.q_proj",
142
+ "model.layers.1.self_attn.k_proj",
143
+ "model.layers.1.self_attn.v_proj",
144
+ "model.layers.1.self_attn.o_proj",
145
+ "model.layers.1.block_sparse_moe.gate",
146
+ "model.layers.2.self_attn.q_proj",
147
+ "model.layers.2.self_attn.k_proj",
148
+ "model.layers.2.self_attn.v_proj",
149
+ "model.layers.2.self_attn.o_proj",
150
+ "model.layers.2.block_sparse_moe.gate",
151
+ "model.layers.3.self_attn.q_proj",
152
+ "model.layers.3.self_attn.k_proj",
153
+ "model.layers.3.self_attn.v_proj",
154
+ "model.layers.3.self_attn.o_proj",
155
+ "model.layers.3.block_sparse_moe.gate",
156
+ "model.layers.4.self_attn.q_proj",
157
+ "model.layers.4.self_attn.k_proj",
158
+ "model.layers.4.self_attn.v_proj",
159
+ "model.layers.4.self_attn.o_proj",
160
+ "model.layers.4.block_sparse_moe.gate",
161
+ "model.layers.5.self_attn.q_proj",
162
+ "model.layers.5.self_attn.k_proj",
163
+ "model.layers.5.self_attn.v_proj",
164
+ "model.layers.5.self_attn.o_proj",
165
+ "model.layers.5.block_sparse_moe.gate",
166
+ "model.layers.6.self_attn.q_proj",
167
+ "model.layers.6.self_attn.k_proj",
168
+ "model.layers.6.self_attn.v_proj",
169
+ "model.layers.6.self_attn.o_proj",
170
+ "model.layers.6.block_sparse_moe.gate",
171
+ "model.layers.7.self_attn.q_proj",
172
+ "model.layers.7.self_attn.k_proj",
173
+ "model.layers.7.self_attn.v_proj",
174
+ "model.layers.7.self_attn.o_proj",
175
+ "model.layers.7.block_sparse_moe.gate",
176
+ "model.layers.8.self_attn.q_proj",
177
+ "model.layers.8.self_attn.k_proj",
178
+ "model.layers.8.self_attn.v_proj",
179
+ "model.layers.8.self_attn.o_proj",
180
+ "model.layers.8.block_sparse_moe.gate",
181
+ "model.layers.9.self_attn.q_proj",
182
+ "model.layers.9.self_attn.k_proj",
183
+ "model.layers.9.self_attn.v_proj",
184
+ "model.layers.9.self_attn.o_proj",
185
+ "model.layers.9.block_sparse_moe.gate",
186
+ "model.layers.10.self_attn.q_proj",
187
+ "model.layers.10.self_attn.k_proj",
188
+ "model.layers.10.self_attn.v_proj",
189
+ "model.layers.10.self_attn.o_proj",
190
+ "model.layers.10.block_sparse_moe.gate",
191
+ "model.layers.11.self_attn.q_proj",
192
+ "model.layers.11.self_attn.k_proj",
193
+ "model.layers.11.self_attn.v_proj",
194
+ "model.layers.11.self_attn.o_proj",
195
+ "model.layers.11.block_sparse_moe.gate",
196
+ "model.layers.12.self_attn.q_proj",
197
+ "model.layers.12.self_attn.k_proj",
198
+ "model.layers.12.self_attn.v_proj",
199
+ "model.layers.12.self_attn.o_proj",
200
+ "model.layers.12.block_sparse_moe.gate",
201
+ "model.layers.13.self_attn.q_proj",
202
+ "model.layers.13.self_attn.k_proj",
203
+ "model.layers.13.self_attn.v_proj",
204
+ "model.layers.13.self_attn.o_proj",
205
+ "model.layers.13.block_sparse_moe.gate",
206
+ "model.layers.14.self_attn.q_proj",
207
+ "model.layers.14.self_attn.k_proj",
208
+ "model.layers.14.self_attn.v_proj",
209
+ "model.layers.14.self_attn.o_proj",
210
+ "model.layers.14.block_sparse_moe.gate",
211
+ "model.layers.15.self_attn.q_proj",
212
+ "model.layers.15.self_attn.k_proj",
213
+ "model.layers.15.self_attn.v_proj",
214
+ "model.layers.15.self_attn.o_proj",
215
+ "model.layers.15.block_sparse_moe.gate",
216
+ "model.layers.16.self_attn.q_proj",
217
+ "model.layers.16.self_attn.k_proj",
218
+ "model.layers.16.self_attn.v_proj",
219
+ "model.layers.16.self_attn.o_proj",
220
+ "model.layers.16.block_sparse_moe.gate",
221
+ "model.layers.17.self_attn.q_proj",
222
+ "model.layers.17.self_attn.k_proj",
223
+ "model.layers.17.self_attn.v_proj",
224
+ "model.layers.17.self_attn.o_proj",
225
+ "model.layers.17.block_sparse_moe.gate",
226
+ "model.layers.18.self_attn.q_proj",
227
+ "model.layers.18.self_attn.k_proj",
228
+ "model.layers.18.self_attn.v_proj",
229
+ "model.layers.18.self_attn.o_proj",
230
+ "model.layers.18.block_sparse_moe.gate",
231
+ "model.layers.19.self_attn.q_proj",
232
+ "model.layers.19.self_attn.k_proj",
233
+ "model.layers.19.self_attn.v_proj",
234
+ "model.layers.19.self_attn.o_proj",
235
+ "model.layers.19.block_sparse_moe.gate",
236
+ "model.layers.20.self_attn.q_proj",
237
+ "model.layers.20.self_attn.k_proj",
238
+ "model.layers.20.self_attn.v_proj",
239
+ "model.layers.20.self_attn.o_proj",
240
+ "model.layers.20.block_sparse_moe.gate",
241
+ "model.layers.21.self_attn.q_proj",
242
+ "model.layers.21.self_attn.k_proj",
243
+ "model.layers.21.self_attn.v_proj",
244
+ "model.layers.21.self_attn.o_proj",
245
+ "model.layers.21.block_sparse_moe.gate",
246
+ "model.layers.22.self_attn.q_proj",
247
+ "model.layers.22.self_attn.k_proj",
248
+ "model.layers.22.self_attn.v_proj",
249
+ "model.layers.22.self_attn.o_proj",
250
+ "model.layers.22.block_sparse_moe.gate",
251
+ "model.layers.23.self_attn.q_proj",
252
+ "model.layers.23.self_attn.k_proj",
253
+ "model.layers.23.self_attn.v_proj",
254
+ "model.layers.23.self_attn.o_proj",
255
+ "model.layers.23.block_sparse_moe.gate",
256
+ "model.layers.24.self_attn.q_proj",
257
+ "model.layers.24.self_attn.k_proj",
258
+ "model.layers.24.self_attn.v_proj",
259
+ "model.layers.24.self_attn.o_proj",
260
+ "model.layers.24.block_sparse_moe.gate",
261
+ "model.layers.25.self_attn.q_proj",
262
+ "model.layers.25.self_attn.k_proj",
263
+ "model.layers.25.self_attn.v_proj",
264
+ "model.layers.25.self_attn.o_proj",
265
+ "model.layers.25.block_sparse_moe.gate",
266
+ "model.layers.26.self_attn.q_proj",
267
+ "model.layers.26.self_attn.k_proj",
268
+ "model.layers.26.self_attn.v_proj",
269
+ "model.layers.26.self_attn.o_proj",
270
+ "model.layers.26.block_sparse_moe.gate",
271
+ "model.layers.27.self_attn.q_proj",
272
+ "model.layers.27.self_attn.k_proj",
273
+ "model.layers.27.self_attn.v_proj",
274
+ "model.layers.27.self_attn.o_proj",
275
+ "model.layers.27.block_sparse_moe.gate",
276
+ "model.layers.28.self_attn.q_proj",
277
+ "model.layers.28.self_attn.k_proj",
278
+ "model.layers.28.self_attn.v_proj",
279
+ "model.layers.28.self_attn.o_proj",
280
+ "model.layers.28.block_sparse_moe.gate",
281
+ "model.layers.29.self_attn.q_proj",
282
+ "model.layers.29.self_attn.k_proj",
283
+ "model.layers.29.self_attn.v_proj",
284
+ "model.layers.29.self_attn.o_proj",
285
+ "model.layers.29.block_sparse_moe.gate",
286
+ "model.layers.30.self_attn.q_proj",
287
+ "model.layers.30.self_attn.k_proj",
288
+ "model.layers.30.self_attn.v_proj",
289
+ "model.layers.30.self_attn.o_proj",
290
+ "model.layers.30.block_sparse_moe.gate",
291
+ "model.layers.31.self_attn.q_proj",
292
+ "model.layers.31.self_attn.k_proj",
293
+ "model.layers.31.self_attn.v_proj",
294
+ "model.layers.31.self_attn.o_proj",
295
+ "model.layers.31.block_sparse_moe.gate",
296
+ "model.layers.32.self_attn.q_proj",
297
+ "model.layers.32.self_attn.k_proj",
298
+ "model.layers.32.self_attn.v_proj",
299
+ "model.layers.32.self_attn.o_proj",
300
+ "model.layers.32.block_sparse_moe.gate",
301
+ "model.layers.33.self_attn.q_proj",
302
+ "model.layers.33.self_attn.k_proj",
303
+ "model.layers.33.self_attn.v_proj",
304
+ "model.layers.33.self_attn.o_proj",
305
+ "model.layers.33.block_sparse_moe.gate",
306
+ "model.layers.34.self_attn.q_proj",
307
+ "model.layers.34.self_attn.k_proj",
308
+ "model.layers.34.self_attn.v_proj",
309
+ "model.layers.34.self_attn.o_proj",
310
+ "model.layers.34.block_sparse_moe.gate",
311
+ "model.layers.35.self_attn.q_proj",
312
+ "model.layers.35.self_attn.k_proj",
313
+ "model.layers.35.self_attn.v_proj",
314
+ "model.layers.35.self_attn.o_proj",
315
+ "model.layers.35.block_sparse_moe.gate",
316
+ "model.layers.36.self_attn.q_proj",
317
+ "model.layers.36.self_attn.k_proj",
318
+ "model.layers.36.self_attn.v_proj",
319
+ "model.layers.36.self_attn.o_proj",
320
+ "model.layers.36.block_sparse_moe.gate",
321
+ "model.layers.37.self_attn.q_proj",
322
+ "model.layers.37.self_attn.k_proj",
323
+ "model.layers.37.self_attn.v_proj",
324
+ "model.layers.37.self_attn.o_proj",
325
+ "model.layers.37.block_sparse_moe.gate",
326
+ "model.layers.38.self_attn.q_proj",
327
+ "model.layers.38.self_attn.k_proj",
328
+ "model.layers.38.self_attn.v_proj",
329
+ "model.layers.38.self_attn.o_proj",
330
+ "model.layers.38.block_sparse_moe.gate",
331
+ "model.layers.39.self_attn.q_proj",
332
+ "model.layers.39.self_attn.k_proj",
333
+ "model.layers.39.self_attn.v_proj",
334
+ "model.layers.39.self_attn.o_proj",
335
+ "model.layers.39.block_sparse_moe.gate",
336
+ "model.layers.40.self_attn.q_proj",
337
+ "model.layers.40.self_attn.k_proj",
338
+ "model.layers.40.self_attn.v_proj",
339
+ "model.layers.40.self_attn.o_proj",
340
+ "model.layers.40.block_sparse_moe.gate",
341
+ "model.layers.41.self_attn.q_proj",
342
+ "model.layers.41.self_attn.k_proj",
343
+ "model.layers.41.self_attn.v_proj",
344
+ "model.layers.41.self_attn.o_proj",
345
+ "model.layers.41.block_sparse_moe.gate",
346
+ "model.layers.42.self_attn.q_proj",
347
+ "model.layers.42.self_attn.k_proj",
348
+ "model.layers.42.self_attn.v_proj",
349
+ "model.layers.42.self_attn.o_proj",
350
+ "model.layers.42.block_sparse_moe.gate",
351
+ "model.layers.43.self_attn.q_proj",
352
+ "model.layers.43.self_attn.k_proj",
353
+ "model.layers.43.self_attn.v_proj",
354
+ "model.layers.43.self_attn.o_proj",
355
+ "model.layers.43.block_sparse_moe.gate",
356
+ "model.layers.44.self_attn.q_proj",
357
+ "model.layers.44.self_attn.k_proj",
358
+ "model.layers.44.self_attn.v_proj",
359
+ "model.layers.44.self_attn.o_proj",
360
+ "model.layers.44.block_sparse_moe.gate",
361
+ "model.layers.45.self_attn.q_proj",
362
+ "model.layers.45.self_attn.k_proj",
363
+ "model.layers.45.self_attn.v_proj",
364
+ "model.layers.45.self_attn.o_proj",
365
+ "model.layers.45.block_sparse_moe.gate",
366
+ "model.layers.46.self_attn.q_proj",
367
+ "model.layers.46.self_attn.k_proj",
368
+ "model.layers.46.self_attn.v_proj",
369
+ "model.layers.46.self_attn.o_proj",
370
+ "model.layers.46.block_sparse_moe.gate",
371
+ "model.layers.47.self_attn.q_proj",
372
+ "model.layers.47.self_attn.k_proj",
373
+ "model.layers.47.self_attn.v_proj",
374
+ "model.layers.47.self_attn.o_proj",
375
+ "model.layers.47.block_sparse_moe.gate",
376
+ "model.layers.48.self_attn.q_proj",
377
+ "model.layers.48.self_attn.k_proj",
378
+ "model.layers.48.self_attn.v_proj",
379
+ "model.layers.48.self_attn.o_proj",
380
+ "model.layers.48.block_sparse_moe.gate",
381
+ "model.layers.49.self_attn.q_proj",
382
+ "model.layers.49.self_attn.k_proj",
383
+ "model.layers.49.self_attn.v_proj",
384
+ "model.layers.49.self_attn.o_proj",
385
+ "model.layers.49.block_sparse_moe.gate",
386
+ "model.layers.50.self_attn.q_proj",
387
+ "model.layers.50.self_attn.k_proj",
388
+ "model.layers.50.self_attn.v_proj",
389
+ "model.layers.50.self_attn.o_proj",
390
+ "model.layers.50.block_sparse_moe.gate",
391
+ "model.layers.51.self_attn.q_proj",
392
+ "model.layers.51.self_attn.k_proj",
393
+ "model.layers.51.self_attn.v_proj",
394
+ "model.layers.51.self_attn.o_proj",
395
+ "model.layers.51.block_sparse_moe.gate",
396
+ "model.layers.52.self_attn.q_proj",
397
+ "model.layers.52.self_attn.k_proj",
398
+ "model.layers.52.self_attn.v_proj",
399
+ "model.layers.52.self_attn.o_proj",
400
+ "model.layers.52.block_sparse_moe.gate",
401
+ "model.layers.53.self_attn.q_proj",
402
+ "model.layers.53.self_attn.k_proj",
403
+ "model.layers.53.self_attn.v_proj",
404
+ "model.layers.53.self_attn.o_proj",
405
+ "model.layers.53.block_sparse_moe.gate",
406
+ "model.layers.54.self_attn.q_proj",
407
+ "model.layers.54.self_attn.k_proj",
408
+ "model.layers.54.self_attn.v_proj",
409
+ "model.layers.54.self_attn.o_proj",
410
+ "model.layers.54.block_sparse_moe.gate",
411
+ "model.layers.55.self_attn.q_proj",
412
+ "model.layers.55.self_attn.k_proj",
413
+ "model.layers.55.self_attn.v_proj",
414
+ "model.layers.55.self_attn.o_proj",
415
+ "model.layers.55.block_sparse_moe.gate",
416
+ "model.layers.56.self_attn.q_proj",
417
+ "model.layers.56.self_attn.k_proj",
418
+ "model.layers.56.self_attn.v_proj",
419
+ "model.layers.56.self_attn.o_proj",
420
+ "model.layers.56.block_sparse_moe.gate",
421
+ "model.layers.57.self_attn.q_proj",
422
+ "model.layers.57.self_attn.k_proj",
423
+ "model.layers.57.self_attn.v_proj",
424
+ "model.layers.57.self_attn.o_proj",
425
+ "model.layers.57.block_sparse_moe.gate",
426
+ "model.layers.58.self_attn.q_proj",
427
+ "model.layers.58.self_attn.k_proj",
428
+ "model.layers.58.self_attn.v_proj",
429
+ "model.layers.58.self_attn.o_proj",
430
+ "model.layers.58.block_sparse_moe.gate",
431
+ "model.layers.59.self_attn.q_proj",
432
+ "model.layers.59.self_attn.k_proj",
433
+ "model.layers.59.self_attn.v_proj",
434
+ "model.layers.59.self_attn.o_proj",
435
+ "model.layers.59.block_sparse_moe.gate",
436
+ "model.layers.60.self_attn.q_proj",
437
+ "model.layers.60.self_attn.k_proj",
438
+ "model.layers.60.self_attn.v_proj",
439
+ "model.layers.60.self_attn.o_proj",
440
+ "model.layers.60.block_sparse_moe.gate",
441
+ "model.layers.61.self_attn.q_proj",
442
+ "model.layers.61.self_attn.k_proj",
443
+ "model.layers.61.self_attn.v_proj",
444
+ "model.layers.61.self_attn.o_proj",
445
+ "model.layers.61.block_sparse_moe.gate",
446
+ "lm_head"
447
+ ],
448
+ "kv_cache_scheme": null,
449
+ "quant_method": "compressed-tensors",
450
+ "quantization_status": "compressed",
451
+ "sparsity_config": {},
452
+ "transform_config": {},
453
+ "version": "0.14.0.1"
454
+ },
455
+ "rms_norm_eps": 1e-06,
456
+ "rope_theta": 5000000,
457
+ "rotary_dim": 64,
458
+ "router_aux_loss_coef": 0.001,
459
+ "router_jitter_noise": 0.0,
460
+ "scoring_func": "sigmoid",
461
+ "shared_intermediate_size": 0,
462
+ "sliding_window": null,
463
+ "tie_word_embeddings": false,
464
+ "transformers_version": "4.57.6",
465
+ "use_cache": true,
466
+ "use_mtp": true,
467
+ "use_qk_norm": true,
468
+ "use_routing_bias": true,
469
+ "vocab_size": 200064
470
+ }
configuration_minimax_m2.py ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
2
+ # This file was automatically generated from src/transformers/models/minimax_m2/modular_minimax_m2.py.
3
+ # Do NOT edit this file manually as any edits will be overwritten by the generation of
4
+ # the file from the modular. If any change should be done, please apply the change to the
5
+ # modular_minimax_m2.py file directly. One of our CI enforces this.
6
+ # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
7
+ # coding=utf-8
8
+ # Copyright 2025 the HuggingFace Team. All rights reserved.
9
+ #
10
+ # Licensed under the Apache License, Version 2.0 (the "License");
11
+ # you may not use this file except in compliance with the License.
12
+ # You may obtain a copy of the License at
13
+ #
14
+ # http://www.apache.org/licenses/LICENSE-2.0
15
+ #
16
+ # Unless required by applicable law or agreed to in writing, software
17
+ # distributed under the License is distributed on an "AS IS" BASIS,
18
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19
+ # See the License for the specific language governing permissions and
20
+ # limitations under the License.
21
+
22
+
23
+ from transformers.configuration_utils import PretrainedConfig
24
+
25
+
26
+ class MiniMaxM2Config(PretrainedConfig):
27
+ r"""
28
+ This is the configuration class to store the configuration of a [`MiniMaxM2Model`]. It is used to instantiate an
29
+ MiniMaxM2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
30
+ with the defaults will yield a similar configuration to that of the MiniMaxM2-7B-v0.1 or MiniMaxM2-7B-Instruct-v0.1.
31
+
32
+ [minimax_m2ai/MiniMaxM2-8x7B](https://huggingface.co/minimax_m2ai/MiniMaxM2-8x7B)
33
+ [minimax_m2ai/MiniMaxM2-7B-Instruct-v0.1](https://huggingface.co/minimax_m2ai/MiniMaxM2-7B-Instruct-v0.1)
34
+
35
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
36
+ documentation from [`PretrainedConfig`] for more information.
37
+
38
+
39
+ Args:
40
+ vocab_size (`int`, *optional*, defaults to 32000):
41
+ Vocabulary size of the MiniMaxM2 model. Defines the number of different tokens that can be represented by the
42
+ `inputs_ids` passed when calling [`MiniMaxM2Model`]
43
+ hidden_size (`int`, *optional*, defaults to 4096):
44
+ Dimension of the hidden representations.
45
+ intermediate_size (`int`, *optional*, defaults to 14336):
46
+ Dimension of the MLP representations.
47
+ num_hidden_layers (`int`, *optional*, defaults to 32):
48
+ Number of hidden layers in the Transformer encoder.
49
+ num_attention_heads (`int`, *optional*, defaults to 32):
50
+ Number of attention heads for each attention layer in the Transformer encoder.
51
+ num_key_value_heads (`int`, *optional*, defaults to 8):
52
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
53
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
54
+ `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
55
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
56
+ by meanpooling all the original heads within that group. For more details, check out [this
57
+ paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `8`.
58
+ head_dim (`int`, *optional*, defaults to `hidden_size // num_attention_heads`):
59
+ The attention head dimension.
60
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
61
+ The non-linear activation function (function or string) in the decoder.
62
+ max_position_embeddings (`int`, *optional*, defaults to `4096*32`):
63
+ The maximum sequence length that this model might ever be used with. MiniMaxM2's sliding window attention
64
+ allows sequence of up to 4096*32 tokens.
65
+ initializer_range (`float`, *optional*, defaults to 0.02):
66
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
67
+ rms_norm_eps (`float`, *optional*, defaults to 1e-05):
68
+ The epsilon used by the rms normalization layers.
69
+ use_cache (`bool`, *optional*, defaults to `True`):
70
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
71
+ relevant if `config.is_decoder=True`.
72
+ pad_token_id (`int`, *optional*):
73
+ The id of the padding token.
74
+ bos_token_id (`int`, *optional*, defaults to 1):
75
+ The id of the "beginning-of-sequence" token.
76
+ eos_token_id (`int`, *optional*, defaults to 2):
77
+ The id of the "end-of-sequence" token.
78
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
79
+ Whether the model's input and output word embeddings should be tied.
80
+ rope_theta (`float`, *optional*, defaults to 1000000.0):
81
+ The base period of the RoPE embeddings.
82
+ sliding_window (`int`, *optional*):
83
+ Sliding window attention window size. If not specified, will default to `4096`.
84
+ attention_dropout (`float`, *optional*, defaults to 0.0):
85
+ The dropout ratio for the attention probabilities.
86
+ num_experts_per_tok (`int`, *optional*, defaults to 2):
87
+ The number of experts to route per-token, can be also interpreted as the `top-k` routing
88
+ parameter
89
+ num_local_experts (`int`, *optional*, defaults to 8):
90
+ Number of experts per Sparse MLP layer.
91
+ output_router_logits (`bool`, *optional*, defaults to `False`):
92
+ Whether or not the router logits should be returned by the model. Enabling this will also
93
+ allow the model to output the auxiliary loss. See [here]() for more details
94
+ router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
95
+ The aux loss factor for the total loss.
96
+ router_jitter_noise (`float`, *optional*, defaults to 0.0):
97
+ Amount of noise to add to the router.
98
+
99
+ ```python
100
+ >>> from transformers import MiniMaxM2Model, MiniMaxM2Config
101
+
102
+ >>> # Initializing a MiniMaxM2 7B style configuration
103
+ >>> configuration = MiniMaxM2Config()
104
+
105
+ >>> # Initializing a model from the MiniMaxM2 7B style configuration
106
+ >>> model = MiniMaxM2Model(configuration)
107
+
108
+ >>> # Accessing the model configuration
109
+ >>> configuration = model.config
110
+ ```"""
111
+
112
+ model_type = "minimax_m2"
113
+ keys_to_ignore_at_inference = ["past_key_values"]
114
+ base_model_tp_plan = {
115
+ "layers.*.self_attn.q_proj": "colwise",
116
+ "layers.*.self_attn.k_proj": "colwise",
117
+ "layers.*.self_attn.v_proj": "colwise",
118
+ "layers.*.self_attn.o_proj": "rowwise",
119
+ "layers.*.block_sparse_moe.gate": "colwise_rep", # we need to replicate here to correctly route experts
120
+ "layers.*.block_sparse_moe.experts.*.w1": "colwise",
121
+ "layers.*.block_sparse_moe.experts.*.w2": "rowwise",
122
+ "layers.*.block_sparse_moe.experts.*.w3": "colwise",
123
+ }
124
+ base_model_pp_plan = {
125
+ "embed_tokens": (["input_ids"], ["inputs_embeds"]),
126
+ "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
127
+ "norm": (["hidden_states"], ["hidden_states"]),
128
+ }
129
+
130
+ def __init__(
131
+ self,
132
+ vocab_size=32000,
133
+ hidden_size=4096,
134
+ intermediate_size=14336,
135
+ num_hidden_layers=32,
136
+ num_attention_heads=32,
137
+ num_key_value_heads=8,
138
+ head_dim=None,
139
+ hidden_act="silu",
140
+ max_position_embeddings=4096 * 32,
141
+ initializer_range=0.02,
142
+ rms_norm_eps=1e-5,
143
+ use_cache=True,
144
+ pad_token_id=None,
145
+ bos_token_id=1,
146
+ eos_token_id=2,
147
+ tie_word_embeddings=False,
148
+ rope_theta=1e6,
149
+ sliding_window=None,
150
+ attention_dropout=0.0,
151
+ num_experts_per_tok=2,
152
+ num_local_experts=8,
153
+ output_router_logits=False,
154
+ router_aux_loss_coef=0.001,
155
+ router_jitter_noise=0.0,
156
+ **kwargs,
157
+ ):
158
+ self.vocab_size = vocab_size
159
+ self.max_position_embeddings = max_position_embeddings
160
+ self.hidden_size = hidden_size
161
+ self.intermediate_size = intermediate_size
162
+ self.num_hidden_layers = num_hidden_layers
163
+ self.num_attention_heads = num_attention_heads
164
+ self.sliding_window = sliding_window
165
+
166
+ # for backward compatibility
167
+ if num_key_value_heads is None:
168
+ num_key_value_heads = num_attention_heads
169
+
170
+ self.num_key_value_heads = num_key_value_heads
171
+ self.hidden_act = hidden_act
172
+ self.initializer_range = initializer_range
173
+ self.rms_norm_eps = rms_norm_eps
174
+ self.use_cache = use_cache
175
+ self.rope_theta = rope_theta
176
+ self.attention_dropout = attention_dropout
177
+ self.head_dim = head_dim
178
+
179
+ self.num_experts_per_tok = num_experts_per_tok
180
+ self.num_local_experts = num_local_experts
181
+ self.output_router_logits = output_router_logits
182
+ self.router_aux_loss_coef = router_aux_loss_coef
183
+ self.router_jitter_noise = router_jitter_noise
184
+
185
+ self.use_qk_norm = kwargs.pop("use_qk_norm", False)
186
+ self.rotary_dim = kwargs.pop("rotary_dim", self.head_dim)
187
+ self.partial_rotary_factor = kwargs.pop("partial_rotary_factor", 1)
188
+ if self.head_dim is not None:
189
+ self.partial_rotary_factor = self.rotary_dim / self.head_dim
190
+
191
+ super().__init__(
192
+ pad_token_id=pad_token_id,
193
+ bos_token_id=bos_token_id,
194
+ eos_token_id=eos_token_id,
195
+ tie_word_embeddings=tie_word_embeddings,
196
+ **kwargs,
197
+ )
198
+
199
+
200
+ __all__ = ["MiniMaxM2Config"]
generation_config.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 200019,
3
+ "do_sample": true,
4
+ "eos_token_id": 200020,
5
+ "top_k": 40,
6
+ "top_p": 0.95,
7
+ "transformers_version": "4.57.6"
8
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model-00001-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b2e98ca062ba96b17a29df675eb7606b8be83062be65e90342abff7e9e478bb
3
+ size 4996434152
model-00002-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb98e82dc790cf34b4f37f8ec5f52361d8f33afe3199c361d416cb0fbdd18fd6
3
+ size 4998376312
model-00003-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:88802b39d063806b15453b6c8db2a852414b13d0511dc77e643c5f3b911d0ace
3
+ size 4996820792
model-00004-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aebbc9169a24fb8e489b8eebb931fcc24cdaed3656373fed7d0c1eb05cb9c94d
3
+ size 4996767424
model-00005-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3038c6075269dea753a9adc3e020d8048c634120572b6867d42ea33e0876a0a6
3
+ size 4996820296
model-00006-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46bd03a58d873827e4973db3cb8bbd9d1fbf9561c7299013c2f9465252c746f9
3
+ size 4996823912
model-00007-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e331f534ac98e8a75592ee34139bb16d54c15aa58ff6e7c00b0f4934b89be3eb
3
+ size 4996767400
model-00008-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b3b95dcd84c8a59b6ebe1ab1341e547f46a76b24d9536d29933afdee42642c3
3
+ size 4996822432
model-00009-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b47a648f6919146392007d2d72e0b10ba66ee0aaf3632abc0daf9bc741e205c
3
+ size 4996822952
model-00010-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f30ce9bcd69d35193260b0c494e6029d3e1c76f895b4f0d4165cbff8cc24c401
3
+ size 4996769456
model-00011-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c3902a382006921ae8d5a865e40312402a9704aedf71964c98b6878f55893d4
3
+ size 4996825560
model-00012-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ed4c2f20cf92a7f964ddb5f5e30efcdf80852195a209ebc20efdc2062eca401
3
+ size 4996822952
model-00013-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:753922d9062422880a5cee40305701eb9a05cf98c7e0c5dbde36a39e849b313c
3
+ size 4996769456
model-00014-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:966b1c1e8dc4084095cf6430ae3f2cdcc1b5e24bd4612b0225d32359840c8661
3
+ size 4996822544
model-00015-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:784c35f11c98da8f60e3a705b80695268999ccc4916449b400e4a047fd3ac564
3
+ size 4996826024
model-00016-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42af50c6ff4dfffec6f93fbba94cb6a79fd5184d9a93476c283a5ce243229e32
3
+ size 4996769456
model-00017-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:18b6799c3dd6ef1b11ec722e68f3e1f4c96fb827a9511938e855f81253301539
3
+ size 4996822592
model-00018-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce00886d538bbb87b4b99a57b17cee987aa4286614462cb5c61c73a495fa77d4
3
+ size 4996822952
model-00019-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d17a68047b4816c8c67f3499dcbb87bea03d55312741ffc9e2446764f28a871
3
+ size 4996769456
model-00020-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:befffc56c7968ad387096c72e6571778758f9ec28676d0b3c203979920952316
3
+ size 4996825720
model-00021-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4706350921a14fcf31b7cfc478ff548c6d04e21dcbd9ebc9af4fa914eaaa7b0c
3
+ size 4996822952
model-00022-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5b4effcabc32503436fe2452eada117872fb3068509878697aa1b789c2e9303
3
+ size 4996769456
model-00023-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4b15484714da4a904195ef5979bb5ce0b452d01e68c44d7660b195786e9645d
3
+ size 4996822696
model-00024-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:15a3d7965ba9644a5c479b99b73aed116f81e6747b69a6e7839895b35f4aab22
3
+ size 4996826024
model-00025-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90028ead41c7cdbb32315c516cd0ffc3768ef6074fd3bcb6d83bf48c0045708e
3
+ size 4996769456
model-00026-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77a803ea81913eebbf28cc2de5f2e270695cfbbfbc1175ee6808eb8644f2394c
3
+ size 4996822752
model-00027-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ecc8b62889b5322100aa1ed432fab3c882e32968345e2d8c71666449db9eef3
3
+ size 4996822952
model-00028-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d95bf2fcbc39a729b3092e2df6dbeb0e58cfa039b007568c457dc24fa2d1763f
3
+ size 4996769456
model-00029-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a6c5bb5d857b7db8b5e4ca96e823e538598978077b55a452f8e55030fb88fb1
3
+ size 4996825872
model-00030-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:02e295f2b84f8c83989aa7ce190a5c22d90b0375d23b5a645082824181de3180
3
+ size 4996835464
model-00031-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ccd60e1a95d9e32cdebbb21ec94bd728fb29405ff5dc592988c79681aa55465
3
+ size 4996756936
model-00032-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b59d99483141dd8e880e077ae2ad53bb6ebcdcfbf9a74592f17cc5a591a6117
3
+ size 4996822856
model-00033-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f2e380ad28ee921f3eee52564e93bd4884ec4c1ed548fbabd83444701b353d3
3
+ size 4996769504
model-00034-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1c6d9c0d738c66418fa6a74408698e7a7a623c120eb4fcaf0838c6a24bd9a6e
3
+ size 4996825432
model-00035-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:442c52ba10ff1a9ba527b3bea76cf9357d0b759d57430da978f9b4d75adf54ac
3
+ size 4996822904
model-00036-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da732514ca52f6f21835c3276e97e562f0987d328590c93e85863432b2dae46c
3
+ size 4996769456
model-00037-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:779426d4bdf7ab754b6ed64cf56231efa37cefe55c7e6d1c6a82271de5b88ef3
3
+ size 4996822416
model-00038-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7ab18a2ad97fc2015c45e8841cb4acfd16283d87a1682af1958369cb6e6671b
3
+ size 4996826024
model-00039-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8a0a80a9d6bd320f0c1fc9cbd43e90c632f9246813e55bb6d8a876fc6b48fdf8
3
+ size 4996769456
model-00040-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dca8d2cd380c5dc5c11a22a37f0fb0d12c8e9fcea6b674193b84fc687f92fb0b
3
+ size 4996822472
model-00041-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e36e2459ffd461e0afa2354805201408c3436f155265d44cf5aff21d50abb085
3
+ size 4996822952
model-00042-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af9ce2d5451f834801bdd23e5ec463a3ba82f055c889801aa07d272f3bbdb2e0
3
+ size 4996769456
model-00043-of-00047.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9fd1cb7f4f3e9ff3a6948a0ee65f463a557597ca1adecd009e012be4bfa9344d
3
+ size 4996825592