ZwwWayne
commited on
Commit
•
03da3f2
1
Parent(s):
405ebfe
use bin instead of safetensors with max shard of 2GB
Browse files- .gitattributes +10 -2
- config.json +2 -2
- generation_config.json +1 -1
- model.safetensors.index.json +0 -234
- modeling_internlm2.py +14 -18
- model-00001-of-00002.safetensors → pytorch_model-00001-of-00008.bin +2 -2
- model-00002-of-00002.safetensors → pytorch_model-00002-of-00008.bin +2 -2
- pytorch_model-00003-of-00008.bin +3 -0
- pytorch_model-00004-of-00008.bin +3 -0
- pytorch_model-00005-of-00008.bin +3 -0
- pytorch_model-00006-of-00008.bin +3 -0
- pytorch_model-00007-of-00008.bin +3 -0
- pytorch_model-00008-of-00008.bin +3 -0
- pytorch_model.bin.index.json +3 -0
- tokenizer_config.json +0 -26
.gitattributes
CHANGED
@@ -33,5 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
36 |
-
model
|
37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
36 |
+
tokenizer.model filter=lfs diff=lfs merge=lfs -text
|
37 |
+
pytorch_model-00003-of-00008.bin filter=lfs diff=lfs merge=lfs -text
|
38 |
+
pytorch_model-00004-of-00008.bin filter=lfs diff=lfs merge=lfs -text
|
39 |
+
pytorch_model-00006-of-00008.bin filter=lfs diff=lfs merge=lfs -text
|
40 |
+
pytorch_model-00007-of-00008.bin filter=lfs diff=lfs merge=lfs -text
|
41 |
+
pytorch_model-00008-of-00008.bin filter=lfs diff=lfs merge=lfs -text
|
42 |
+
pytorch_model-00001-of-00008.bin filter=lfs diff=lfs merge=lfs -text
|
43 |
+
pytorch_model-00002-of-00008.bin filter=lfs diff=lfs merge=lfs -text
|
44 |
+
pytorch_model-00005-of-00008.bin filter=lfs diff=lfs merge=lfs -text
|
45 |
+
pytorch_model.bin.index.json filter=lfs diff=lfs merge=lfs -text
|
config.json
CHANGED
@@ -27,8 +27,8 @@
|
|
27 |
},
|
28 |
"rope_theta": 1000000,
|
29 |
"tie_word_embeddings": false,
|
30 |
-
"torch_dtype": "
|
31 |
-
"transformers_version": "4.
|
32 |
"use_cache": true,
|
33 |
"vocab_size": 92544
|
34 |
}
|
|
|
27 |
},
|
28 |
"rope_theta": 1000000,
|
29 |
"tie_word_embeddings": false,
|
30 |
+
"torch_dtype": "float16",
|
31 |
+
"transformers_version": "4.33.2",
|
32 |
"use_cache": true,
|
33 |
"vocab_size": 92544
|
34 |
}
|
generation_config.json
CHANGED
@@ -3,5 +3,5 @@
|
|
3 |
"bos_token_id": 1,
|
4 |
"eos_token_id": 2,
|
5 |
"pad_token_id": 2,
|
6 |
-
"transformers_version": "4.
|
7 |
}
|
|
|
3 |
"bos_token_id": 1,
|
4 |
"eos_token_id": 2,
|
5 |
"pad_token_id": 2,
|
6 |
+
"transformers_version": "4.33.2"
|
7 |
}
|
model.safetensors.index.json
DELETED
@@ -1,234 +0,0 @@
|
|
1 |
-
{
|
2 |
-
"metadata": {
|
3 |
-
"total_size": 15475417088
|
4 |
-
},
|
5 |
-
"weight_map": {
|
6 |
-
"model.layers.0.attention.wo.weight": "model-00001-of-00002.safetensors",
|
7 |
-
"model.layers.0.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
8 |
-
"model.layers.0.attention_norm.weight": "model-00001-of-00002.safetensors",
|
9 |
-
"model.layers.0.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
10 |
-
"model.layers.0.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
11 |
-
"model.layers.0.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
12 |
-
"model.layers.0.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
13 |
-
"model.layers.1.attention.wo.weight": "model-00001-of-00002.safetensors",
|
14 |
-
"model.layers.1.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
15 |
-
"model.layers.1.attention_norm.weight": "model-00001-of-00002.safetensors",
|
16 |
-
"model.layers.1.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
17 |
-
"model.layers.1.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
18 |
-
"model.layers.1.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
19 |
-
"model.layers.1.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
20 |
-
"model.layers.10.attention.wo.weight": "model-00001-of-00002.safetensors",
|
21 |
-
"model.layers.10.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
22 |
-
"model.layers.10.attention_norm.weight": "model-00001-of-00002.safetensors",
|
23 |
-
"model.layers.10.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
24 |
-
"model.layers.10.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
25 |
-
"model.layers.10.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
26 |
-
"model.layers.10.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
27 |
-
"model.layers.11.attention.wo.weight": "model-00001-of-00002.safetensors",
|
28 |
-
"model.layers.11.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
29 |
-
"model.layers.11.attention_norm.weight": "model-00001-of-00002.safetensors",
|
30 |
-
"model.layers.11.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
31 |
-
"model.layers.11.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
32 |
-
"model.layers.11.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
33 |
-
"model.layers.11.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
34 |
-
"model.layers.12.attention.wo.weight": "model-00001-of-00002.safetensors",
|
35 |
-
"model.layers.12.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
36 |
-
"model.layers.12.attention_norm.weight": "model-00001-of-00002.safetensors",
|
37 |
-
"model.layers.12.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
38 |
-
"model.layers.12.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
39 |
-
"model.layers.12.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
40 |
-
"model.layers.12.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
41 |
-
"model.layers.13.attention.wo.weight": "model-00001-of-00002.safetensors",
|
42 |
-
"model.layers.13.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
43 |
-
"model.layers.13.attention_norm.weight": "model-00001-of-00002.safetensors",
|
44 |
-
"model.layers.13.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
45 |
-
"model.layers.13.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
46 |
-
"model.layers.13.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
47 |
-
"model.layers.13.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
48 |
-
"model.layers.14.attention.wo.weight": "model-00001-of-00002.safetensors",
|
49 |
-
"model.layers.14.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
50 |
-
"model.layers.14.attention_norm.weight": "model-00001-of-00002.safetensors",
|
51 |
-
"model.layers.14.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
52 |
-
"model.layers.14.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
53 |
-
"model.layers.14.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
54 |
-
"model.layers.14.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
55 |
-
"model.layers.15.attention.wo.weight": "model-00001-of-00002.safetensors",
|
56 |
-
"model.layers.15.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
57 |
-
"model.layers.15.attention_norm.weight": "model-00001-of-00002.safetensors",
|
58 |
-
"model.layers.15.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
59 |
-
"model.layers.15.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
60 |
-
"model.layers.15.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
61 |
-
"model.layers.15.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
62 |
-
"model.layers.16.attention.wo.weight": "model-00001-of-00002.safetensors",
|
63 |
-
"model.layers.16.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
64 |
-
"model.layers.16.attention_norm.weight": "model-00001-of-00002.safetensors",
|
65 |
-
"model.layers.16.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
66 |
-
"model.layers.16.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
67 |
-
"model.layers.16.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
68 |
-
"model.layers.16.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
69 |
-
"model.layers.17.attention.wo.weight": "model-00001-of-00002.safetensors",
|
70 |
-
"model.layers.17.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
71 |
-
"model.layers.17.attention_norm.weight": "model-00001-of-00002.safetensors",
|
72 |
-
"model.layers.17.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
73 |
-
"model.layers.17.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
74 |
-
"model.layers.17.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
75 |
-
"model.layers.17.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
76 |
-
"model.layers.18.attention.wo.weight": "model-00001-of-00002.safetensors",
|
77 |
-
"model.layers.18.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
78 |
-
"model.layers.18.attention_norm.weight": "model-00001-of-00002.safetensors",
|
79 |
-
"model.layers.18.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
80 |
-
"model.layers.18.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
81 |
-
"model.layers.18.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
82 |
-
"model.layers.18.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
83 |
-
"model.layers.19.attention.wo.weight": "model-00001-of-00002.safetensors",
|
84 |
-
"model.layers.19.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
85 |
-
"model.layers.19.attention_norm.weight": "model-00001-of-00002.safetensors",
|
86 |
-
"model.layers.19.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
87 |
-
"model.layers.19.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
88 |
-
"model.layers.19.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
89 |
-
"model.layers.19.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
90 |
-
"model.layers.2.attention.wo.weight": "model-00001-of-00002.safetensors",
|
91 |
-
"model.layers.2.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
92 |
-
"model.layers.2.attention_norm.weight": "model-00001-of-00002.safetensors",
|
93 |
-
"model.layers.2.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
94 |
-
"model.layers.2.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
95 |
-
"model.layers.2.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
96 |
-
"model.layers.2.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
97 |
-
"model.layers.20.attention.wo.weight": "model-00001-of-00002.safetensors",
|
98 |
-
"model.layers.20.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
99 |
-
"model.layers.20.attention_norm.weight": "model-00001-of-00002.safetensors",
|
100 |
-
"model.layers.20.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
101 |
-
"model.layers.20.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
102 |
-
"model.layers.20.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
103 |
-
"model.layers.20.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
104 |
-
"model.layers.21.attention.wo.weight": "model-00002-of-00002.safetensors",
|
105 |
-
"model.layers.21.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
106 |
-
"model.layers.21.attention_norm.weight": "model-00002-of-00002.safetensors",
|
107 |
-
"model.layers.21.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
|
108 |
-
"model.layers.21.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
|
109 |
-
"model.layers.21.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
|
110 |
-
"model.layers.21.ffn_norm.weight": "model-00002-of-00002.safetensors",
|
111 |
-
"model.layers.22.attention.wo.weight": "model-00002-of-00002.safetensors",
|
112 |
-
"model.layers.22.attention.wqkv.weight": "model-00002-of-00002.safetensors",
|
113 |
-
"model.layers.22.attention_norm.weight": "model-00002-of-00002.safetensors",
|
114 |
-
"model.layers.22.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
|
115 |
-
"model.layers.22.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
|
116 |
-
"model.layers.22.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
|
117 |
-
"model.layers.22.ffn_norm.weight": "model-00002-of-00002.safetensors",
|
118 |
-
"model.layers.23.attention.wo.weight": "model-00002-of-00002.safetensors",
|
119 |
-
"model.layers.23.attention.wqkv.weight": "model-00002-of-00002.safetensors",
|
120 |
-
"model.layers.23.attention_norm.weight": "model-00002-of-00002.safetensors",
|
121 |
-
"model.layers.23.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
|
122 |
-
"model.layers.23.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
|
123 |
-
"model.layers.23.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
|
124 |
-
"model.layers.23.ffn_norm.weight": "model-00002-of-00002.safetensors",
|
125 |
-
"model.layers.24.attention.wo.weight": "model-00002-of-00002.safetensors",
|
126 |
-
"model.layers.24.attention.wqkv.weight": "model-00002-of-00002.safetensors",
|
127 |
-
"model.layers.24.attention_norm.weight": "model-00002-of-00002.safetensors",
|
128 |
-
"model.layers.24.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
|
129 |
-
"model.layers.24.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
|
130 |
-
"model.layers.24.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
|
131 |
-
"model.layers.24.ffn_norm.weight": "model-00002-of-00002.safetensors",
|
132 |
-
"model.layers.25.attention.wo.weight": "model-00002-of-00002.safetensors",
|
133 |
-
"model.layers.25.attention.wqkv.weight": "model-00002-of-00002.safetensors",
|
134 |
-
"model.layers.25.attention_norm.weight": "model-00002-of-00002.safetensors",
|
135 |
-
"model.layers.25.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
|
136 |
-
"model.layers.25.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
|
137 |
-
"model.layers.25.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
|
138 |
-
"model.layers.25.ffn_norm.weight": "model-00002-of-00002.safetensors",
|
139 |
-
"model.layers.26.attention.wo.weight": "model-00002-of-00002.safetensors",
|
140 |
-
"model.layers.26.attention.wqkv.weight": "model-00002-of-00002.safetensors",
|
141 |
-
"model.layers.26.attention_norm.weight": "model-00002-of-00002.safetensors",
|
142 |
-
"model.layers.26.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
|
143 |
-
"model.layers.26.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
|
144 |
-
"model.layers.26.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
|
145 |
-
"model.layers.26.ffn_norm.weight": "model-00002-of-00002.safetensors",
|
146 |
-
"model.layers.27.attention.wo.weight": "model-00002-of-00002.safetensors",
|
147 |
-
"model.layers.27.attention.wqkv.weight": "model-00002-of-00002.safetensors",
|
148 |
-
"model.layers.27.attention_norm.weight": "model-00002-of-00002.safetensors",
|
149 |
-
"model.layers.27.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
|
150 |
-
"model.layers.27.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
|
151 |
-
"model.layers.27.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
|
152 |
-
"model.layers.27.ffn_norm.weight": "model-00002-of-00002.safetensors",
|
153 |
-
"model.layers.28.attention.wo.weight": "model-00002-of-00002.safetensors",
|
154 |
-
"model.layers.28.attention.wqkv.weight": "model-00002-of-00002.safetensors",
|
155 |
-
"model.layers.28.attention_norm.weight": "model-00002-of-00002.safetensors",
|
156 |
-
"model.layers.28.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
|
157 |
-
"model.layers.28.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
|
158 |
-
"model.layers.28.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
|
159 |
-
"model.layers.28.ffn_norm.weight": "model-00002-of-00002.safetensors",
|
160 |
-
"model.layers.29.attention.wo.weight": "model-00002-of-00002.safetensors",
|
161 |
-
"model.layers.29.attention.wqkv.weight": "model-00002-of-00002.safetensors",
|
162 |
-
"model.layers.29.attention_norm.weight": "model-00002-of-00002.safetensors",
|
163 |
-
"model.layers.29.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
|
164 |
-
"model.layers.29.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
|
165 |
-
"model.layers.29.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
|
166 |
-
"model.layers.29.ffn_norm.weight": "model-00002-of-00002.safetensors",
|
167 |
-
"model.layers.3.attention.wo.weight": "model-00001-of-00002.safetensors",
|
168 |
-
"model.layers.3.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
169 |
-
"model.layers.3.attention_norm.weight": "model-00001-of-00002.safetensors",
|
170 |
-
"model.layers.3.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
171 |
-
"model.layers.3.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
172 |
-
"model.layers.3.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
173 |
-
"model.layers.3.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
174 |
-
"model.layers.30.attention.wo.weight": "model-00002-of-00002.safetensors",
|
175 |
-
"model.layers.30.attention.wqkv.weight": "model-00002-of-00002.safetensors",
|
176 |
-
"model.layers.30.attention_norm.weight": "model-00002-of-00002.safetensors",
|
177 |
-
"model.layers.30.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
|
178 |
-
"model.layers.30.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
|
179 |
-
"model.layers.30.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
|
180 |
-
"model.layers.30.ffn_norm.weight": "model-00002-of-00002.safetensors",
|
181 |
-
"model.layers.31.attention.wo.weight": "model-00002-of-00002.safetensors",
|
182 |
-
"model.layers.31.attention.wqkv.weight": "model-00002-of-00002.safetensors",
|
183 |
-
"model.layers.31.attention_norm.weight": "model-00002-of-00002.safetensors",
|
184 |
-
"model.layers.31.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
|
185 |
-
"model.layers.31.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
|
186 |
-
"model.layers.31.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
|
187 |
-
"model.layers.31.ffn_norm.weight": "model-00002-of-00002.safetensors",
|
188 |
-
"model.layers.4.attention.wo.weight": "model-00001-of-00002.safetensors",
|
189 |
-
"model.layers.4.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
190 |
-
"model.layers.4.attention_norm.weight": "model-00001-of-00002.safetensors",
|
191 |
-
"model.layers.4.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
192 |
-
"model.layers.4.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
193 |
-
"model.layers.4.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
194 |
-
"model.layers.4.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
195 |
-
"model.layers.5.attention.wo.weight": "model-00001-of-00002.safetensors",
|
196 |
-
"model.layers.5.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
197 |
-
"model.layers.5.attention_norm.weight": "model-00001-of-00002.safetensors",
|
198 |
-
"model.layers.5.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
199 |
-
"model.layers.5.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
200 |
-
"model.layers.5.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
201 |
-
"model.layers.5.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
202 |
-
"model.layers.6.attention.wo.weight": "model-00001-of-00002.safetensors",
|
203 |
-
"model.layers.6.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
204 |
-
"model.layers.6.attention_norm.weight": "model-00001-of-00002.safetensors",
|
205 |
-
"model.layers.6.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
206 |
-
"model.layers.6.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
207 |
-
"model.layers.6.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
208 |
-
"model.layers.6.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
209 |
-
"model.layers.7.attention.wo.weight": "model-00001-of-00002.safetensors",
|
210 |
-
"model.layers.7.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
211 |
-
"model.layers.7.attention_norm.weight": "model-00001-of-00002.safetensors",
|
212 |
-
"model.layers.7.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
213 |
-
"model.layers.7.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
214 |
-
"model.layers.7.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
215 |
-
"model.layers.7.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
216 |
-
"model.layers.8.attention.wo.weight": "model-00001-of-00002.safetensors",
|
217 |
-
"model.layers.8.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
218 |
-
"model.layers.8.attention_norm.weight": "model-00001-of-00002.safetensors",
|
219 |
-
"model.layers.8.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
220 |
-
"model.layers.8.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
221 |
-
"model.layers.8.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
222 |
-
"model.layers.8.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
223 |
-
"model.layers.9.attention.wo.weight": "model-00001-of-00002.safetensors",
|
224 |
-
"model.layers.9.attention.wqkv.weight": "model-00001-of-00002.safetensors",
|
225 |
-
"model.layers.9.attention_norm.weight": "model-00001-of-00002.safetensors",
|
226 |
-
"model.layers.9.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
|
227 |
-
"model.layers.9.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
|
228 |
-
"model.layers.9.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
|
229 |
-
"model.layers.9.ffn_norm.weight": "model-00001-of-00002.safetensors",
|
230 |
-
"model.norm.weight": "model-00002-of-00002.safetensors",
|
231 |
-
"model.tok_embeddings.weight": "model-00001-of-00002.safetensors",
|
232 |
-
"output.weight": "model-00002-of-00002.safetensors"
|
233 |
-
}
|
234 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
modeling_internlm2.py
CHANGED
@@ -133,7 +133,7 @@ class InternLM2RotaryEmbedding(nn.Module):
|
|
133 |
def forward(self, x, seq_len=None):
|
134 |
# x: [bs, num_attention_heads, seq_len, head_size]
|
135 |
if seq_len > self.max_seq_len_cached:
|
136 |
-
self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=
|
137 |
|
138 |
return (
|
139 |
self.cos_cached[:seq_len].to(dtype=x.dtype),
|
@@ -196,20 +196,10 @@ def rotate_half(x):
|
|
196 |
|
197 |
|
198 |
def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
|
199 |
-
|
200 |
-
|
201 |
-
|
202 |
-
|
203 |
-
sin = sin.unsqueeze(0).unsqueeze(0).expand(len(position_ids), -1, -1, -1)
|
204 |
-
if q.size(2) == 1:
|
205 |
-
q_embed = (q * cos[:, :, -1, :]) + (rotate_half(q) * sin[:, :, -1, :])
|
206 |
-
else:
|
207 |
-
q_embed = (q * cos) + (rotate_half(q) * sin)
|
208 |
-
|
209 |
-
if k.size(2) == 1:
|
210 |
-
k_embed = (k * cos[:, :, -1, :]) + (rotate_half(k) * sin[:, :, -1, :])
|
211 |
-
else:
|
212 |
-
k_embed = (k * cos) + (rotate_half(k) * sin)
|
213 |
|
214 |
return q_embed, k_embed
|
215 |
|
@@ -289,8 +279,15 @@ class InternLM2Attention(nn.Module):
|
|
289 |
base=self.config.rope_theta,
|
290 |
scaling_factor=scaling_factor
|
291 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
292 |
else:
|
293 |
-
raise ValueError("Currently we only support rotary embedding's type being 'dynamic'.")
|
294 |
return self.rotary_emb
|
295 |
|
296 |
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
|
@@ -1032,7 +1029,6 @@ class InternLM2ForCausalLM(InternLM2PreTrainedModel):
|
|
1032 |
for record in history:
|
1033 |
prompt += f"""[UNUSED_TOKEN_146]user\n{record[0]}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n{record[1]}[UNUSED_TOKEN_145]\n"""
|
1034 |
prompt += f"""[UNUSED_TOKEN_146]user\n{query}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n"""
|
1035 |
-
print(prompt)
|
1036 |
return tokenizer([prompt], return_tensors="pt")
|
1037 |
|
1038 |
@torch.no_grad()
|
@@ -1268,5 +1264,5 @@ class InternLM2ForSequenceClassification(InternLM2PreTrainedModel):
|
|
1268 |
logits=pooled_logits,
|
1269 |
past_key_values=transformer_outputs.past_key_values,
|
1270 |
hidden_states=transformer_outputs.hidden_states,
|
1271 |
-
attentions=transformer_outputs,
|
1272 |
)
|
|
|
133 |
def forward(self, x, seq_len=None):
|
134 |
# x: [bs, num_attention_heads, seq_len, head_size]
|
135 |
if seq_len > self.max_seq_len_cached:
|
136 |
+
self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=torch.float32)
|
137 |
|
138 |
return (
|
139 |
self.cos_cached[:seq_len].to(dtype=x.dtype),
|
|
|
196 |
|
197 |
|
198 |
def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
|
199 |
+
cos = cos[position_ids].unsqueeze(1)
|
200 |
+
sin = sin[position_ids].unsqueeze(1)
|
201 |
+
q_embed = (q * cos) + (rotate_half(q) * sin)
|
202 |
+
k_embed = (k * cos) + (rotate_half(k) * sin)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
203 |
|
204 |
return q_embed, k_embed
|
205 |
|
|
|
279 |
base=self.config.rope_theta,
|
280 |
scaling_factor=scaling_factor
|
281 |
)
|
282 |
+
elif scaling_type == "linear":
|
283 |
+
self.rotary_emb = InternLM2LinearScalingRotaryEmbedding(
|
284 |
+
self.head_dim,
|
285 |
+
max_position_embeddings=self.max_position_embeddings,
|
286 |
+
base=self.config.rope_theta,
|
287 |
+
scaling_factor=scaling_factor
|
288 |
+
)
|
289 |
else:
|
290 |
+
raise ValueError("Currently we only support rotary embedding's type being 'dynamic' or 'linear'.")
|
291 |
return self.rotary_emb
|
292 |
|
293 |
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
|
|
|
1029 |
for record in history:
|
1030 |
prompt += f"""[UNUSED_TOKEN_146]user\n{record[0]}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n{record[1]}[UNUSED_TOKEN_145]\n"""
|
1031 |
prompt += f"""[UNUSED_TOKEN_146]user\n{query}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n"""
|
|
|
1032 |
return tokenizer([prompt], return_tensors="pt")
|
1033 |
|
1034 |
@torch.no_grad()
|
|
|
1264 |
logits=pooled_logits,
|
1265 |
past_key_values=transformer_outputs.past_key_values,
|
1266 |
hidden_states=transformer_outputs.hidden_states,
|
1267 |
+
attentions=transformer_outputs.attentions,
|
1268 |
)
|
model-00001-of-00002.safetensors → pytorch_model-00001-of-00008.bin
RENAMED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:fdd03c1ea040b17ce992eb303d0824f5694932902d0c3fa57b5a8d1df2dc8082
|
3 |
+
size 1949342053
|
model-00002-of-00002.safetensors → pytorch_model-00002-of-00008.bin
RENAMED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:377d66327bcc2dc4b3e5f4e90f24b428c55832ece07194990248647ea65c2db5
|
3 |
+
size 1946249825
|
pytorch_model-00003-of-00008.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:98aa01b7ffd7901699b4588b5e6b8f753e66c1874f8b855c36b23de2c95ae9d8
|
3 |
+
size 1979786923
|
pytorch_model-00004-of-00008.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9362abdb2afd077e579a7860ad570769255c2d64af2b0831413578c77a44d779
|
3 |
+
size 1946249889
|
pytorch_model-00005-of-00008.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:56761cf09eb95e676bce5c1dc53fd4f8c317775d146fe2313dcb7b81c48cfa39
|
3 |
+
size 1979786987
|
pytorch_model-00006-of-00008.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:7a825cc1e85ffc038efaed5a06ae398f424916bdd768bcf62d7fb4bc552d71ef
|
3 |
+
size 1946249889
|
pytorch_model-00007-of-00008.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:224105eee8f19409994aea890cc6cc9da16ef4278ad831fdd4e9fcf86f925256
|
3 |
+
size 1979786987
|
pytorch_model-00008-of-00008.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:0ed276d23e8a4ab9ec5bafe491cb7147b4b51232a66ce0b9f304bc8a5430d82f
|
3 |
+
size 1748039973
|
pytorch_model.bin.index.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a8a1efb6998624330a0564f9bba63eb8ccae0ad54a6d0176c64f2eb30721f2b5
|
3 |
+
size 18179
|
tokenizer_config.json
CHANGED
@@ -1,30 +1,4 @@
|
|
1 |
{
|
2 |
-
"added_tokens_decoder": {
|
3 |
-
"0": {
|
4 |
-
"content": "<unk>",
|
5 |
-
"lstrip": false,
|
6 |
-
"normalized": false,
|
7 |
-
"rstrip": false,
|
8 |
-
"single_word": false,
|
9 |
-
"special": true
|
10 |
-
},
|
11 |
-
"1": {
|
12 |
-
"content": "<s>",
|
13 |
-
"lstrip": false,
|
14 |
-
"normalized": false,
|
15 |
-
"rstrip": false,
|
16 |
-
"single_word": false,
|
17 |
-
"special": true
|
18 |
-
},
|
19 |
-
"2": {
|
20 |
-
"content": "</s>",
|
21 |
-
"lstrip": false,
|
22 |
-
"normalized": false,
|
23 |
-
"rstrip": false,
|
24 |
-
"single_word": false,
|
25 |
-
"special": true
|
26 |
-
}
|
27 |
-
},
|
28 |
"auto_map": {
|
29 |
"AutoTokenizer": [
|
30 |
"tokenization_internlm.InternLMTokenizer",
|
|
|
1 |
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2 |
"auto_map": {
|
3 |
"AutoTokenizer": [
|
4 |
"tokenization_internlm.InternLMTokenizer",
|