Fix - potential bug in checkpoint files #5

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00001-of-00002.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00002-of-00002.bin filter=lfs diff=lfs merge=lfs -text

midm_bitext_tokenization.py

@@ -122,10 +122,6 @@ class Midm_bitext_Tokenizer(PreTrainedTokenizer):
 
         self.vocab_file = vocab_file
         self._extra_ids = extra_ids
-        ##JHS
-        #print(**self.sp_model_kwargs)
-        #import pdb
-        #pdb.set_trace()
         
         self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.Load(vocab_file)

modeling_midm.py

@@ -106,19 +106,15 @@ class MidmAttention(nn.Module):
         assert self.scale_attn_by_inverse_layer_idx != self.scale_qk_by_inverse_layer_idx
 
         if self.is_cross_attention:
-            self.c_attn = nn.Linear(self.embed_dim, 2 * self.embed_dim)
+            self.c_attn = nn.Linear(self.embed_dim, 2 * self.embed_dim, bias=False)
             nn.init.normal_(self.c_attn.weight, std=0.02)
-            nn.init.zeros_(self.c_attn.bias)
-            self.q_attn = nn.Linear(self.embed_dim, self.embed_dim)
+            self.q_attn = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
             nn.init.normal_(self.q_attn.weight, std=0.02)
-            nn.init.zeros_(self.q_attn.bias)
         else:
-            self.c_attn = nn.Linear(self.embed_dim, 3 * self.embed_dim)
+            self.c_attn = nn.Linear(self.embed_dim, 3 * self.embed_dim, bias=False)
             nn.init.normal_(self.c_attn.weight, std=0.02)
-            nn.init.zeros_(self.c_attn.bias)
-        self.c_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.c_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
         nn.init.normal_(self.c_proj.weight, std=0.02)
-        nn.init.zeros_(self.c_proj.bias)
 
         self.attn_dropout = nn.Dropout(config.attn_pdrop)
         self.resid_dropout = nn.Dropout(config.resid_pdrop)
@@ -311,14 +307,12 @@ class MidmMLP(nn.Module):
         embed_dim = config.hidden_size
         self.kt_glu = config.activation_function in ['silu']
         if self.kt_glu:
-            self.c_fc = nn.Linear(embed_dim, intermediate_size * 2)
+            self.c_fc = nn.Linear(embed_dim, intermediate_size * 2, bias=False)
         else:
-            self.c_fc = nn.Linear(embed_dim, intermediate_size)
+            self.c_fc = nn.Linear(embed_dim, intermediate_size, bias=False)
         nn.init.normal_(self.c_fc.weight, std=0.02)
-        nn.init.zeros_(self.c_fc.bias)
-        self.c_proj = nn.Linear(intermediate_size, embed_dim)
+        self.c_proj = nn.Linear(intermediate_size, embed_dim, bias=False)
         nn.init.normal_(self.c_proj.weight, std=0.02)
-        nn.init.zeros_(self.c_proj.bias)
 
         if config.activation_function == 'silu':
             self.act = torch.nn.functional.silu

pytorch_model-00001-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8cd9de9e2ddafd6d1c19d81a02f4cb7b25d61f74fca0de6daa6a509078f4d7e
-size 9968116437
+oid sha256:80fc80d4e22b110b5ff384a3a07565d8dbf22983b3834c622897de0f0a72a23b
+size 9966406423

pytorch_model-00002-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddc4f7593179c0c90845273359d28a1575d3ee030a07da413ccbc95aab23ca8e
-size 6217417531
+oid sha256:8942236584d165e6e593f9c73b07300415a9e197e6ab0dc4daf99dbe417198d2
+size 6216391794

pytorch_model.bin.index.json

@@ -1,456 +1,328 @@
 {
   "metadata": {
-    "total_size": 16185393280
+    "total_size": 16182689920
   },
   "weight_map": {
     "lm_head.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.0.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.0.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.0.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.0.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.0.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.0.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.0.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.0.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.0.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.0.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.0.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.0.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.0.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.0.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.1.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.1.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.1.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.1.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.1.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.1.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.1.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.1.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.1.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.1.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.1.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.1.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.1.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.1.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.10.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.10.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.10.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.10.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.10.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.10.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.10.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.10.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.10.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.10.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.10.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.10.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.10.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.10.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.11.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.11.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.11.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.11.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.11.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.11.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.11.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.11.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.11.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.11.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.11.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.11.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.11.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.11.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.12.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.12.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.12.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.12.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.12.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.12.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.12.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.12.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.12.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.12.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.12.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.12.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.12.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.12.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.13.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.13.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.13.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.13.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.13.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.13.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.13.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.13.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.13.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.13.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.13.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.13.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.13.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.13.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.14.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.14.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.14.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.14.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.14.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.14.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.14.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.14.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.14.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.14.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.14.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.14.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.14.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.14.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.15.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.15.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.15.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.15.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.15.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.15.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.15.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.15.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.15.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.15.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.15.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.15.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.15.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.15.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.16.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.16.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.16.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.16.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.16.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.16.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.16.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.16.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.16.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.16.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.16.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.16.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.16.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.16.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.17.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.17.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.17.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.17.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.17.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.17.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.17.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.17.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.17.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.17.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.17.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.17.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.17.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.17.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.18.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.18.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.18.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.18.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.18.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.18.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.18.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.18.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.18.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.18.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.18.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.18.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.18.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.18.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.19.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.19.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.19.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.19.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.19.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.19.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.19.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.19.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.19.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.19.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.19.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.19.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.19.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.19.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.2.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.2.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.2.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.2.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.2.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.2.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.2.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.2.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.2.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.2.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.2.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.2.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.2.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.2.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.20.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.20.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.20.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.20.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.20.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.20.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.20.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.20.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.20.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.20.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.20.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.20.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.20.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.20.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.21.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.21.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.21.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.21.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.21.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.21.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.21.ln_1.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.21.ln_1.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.21.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.21.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.21.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.21.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.21.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.21.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.22.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.22.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.22.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.22.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.22.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.22.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.22.ln_1.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.22.ln_1.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.22.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.22.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.22.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.22.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.22.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.22.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.23.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.23.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.23.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.23.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.23.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.23.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.23.ln_1.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.23.ln_1.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.23.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.23.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.23.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.23.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.23.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.23.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.24.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.24.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.24.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.24.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.24.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.24.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.24.ln_1.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.24.ln_1.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.24.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.24.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.24.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.24.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.24.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.24.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.25.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.25.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.25.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.25.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.25.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.25.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.25.ln_1.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.25.ln_1.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.25.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.25.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.25.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.25.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.25.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.25.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.26.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.26.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.26.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.26.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.26.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.26.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.26.ln_1.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.26.ln_1.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.26.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.26.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.26.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.26.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.26.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.26.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.27.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.27.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.27.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.27.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.27.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.27.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.27.ln_1.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.27.ln_1.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.27.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.27.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.27.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.27.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.27.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.27.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.28.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.28.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.28.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.28.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.28.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.28.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.28.ln_1.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.28.ln_1.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.28.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.28.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.28.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.28.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.28.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.28.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.29.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.29.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.29.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.29.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.29.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.29.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.29.ln_1.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.29.ln_1.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.29.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.29.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.29.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.29.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.29.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.29.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.3.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.3.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.3.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.3.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.3.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.3.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.3.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.3.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.3.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.3.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.3.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.3.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.3.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.3.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.30.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.30.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.30.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.30.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.30.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.30.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.30.ln_1.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.30.ln_1.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.30.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.30.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.30.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.30.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.30.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.30.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.31.attn.bias": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.31.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.31.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.31.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.31.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.31.attn.masked_bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.31.ln_1.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.31.ln_1.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.31.ln_2.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.31.ln_2.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.31.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.31.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
-    "transformer.h.31.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.h.31.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
     "transformer.h.4.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.4.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.4.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.4.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.4.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.4.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.4.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.4.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.4.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.4.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.4.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.4.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.4.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.4.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.5.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.5.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.5.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.5.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.5.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.5.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.5.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.5.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.5.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.5.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.5.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.5.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.5.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.5.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.6.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.6.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.6.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.6.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.6.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.6.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.6.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.6.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.6.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.6.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.6.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.6.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.6.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.6.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.7.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.7.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.7.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.7.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.7.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.7.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.7.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.7.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.7.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.7.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.7.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.7.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.7.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.7.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.8.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.8.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.8.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.8.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.8.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.8.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.8.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.8.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.8.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.8.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.8.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.8.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.8.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.8.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.9.attn.bias": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.9.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.9.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.9.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.9.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.9.attn.masked_bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.9.ln_1.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.9.ln_1.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.h.9.ln_2.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.9.ln_2.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.9.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.9.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
-    "transformer.h.9.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
     "transformer.h.9.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
     "transformer.ln_f.bias": "pytorch_model-00002-of-00002.bin",
     "transformer.ln_f.weight": "pytorch_model-00002-of-00002.bin",