Update Diffusers T5 weights

.huggingface/.gitignore

@@ -0,0 +1 @@
+*

.huggingface/download/.gitattributes.metadata

@@ -0,0 +1,3 @@
+1d5ece72b189f7322a5405e9682711ffe9580378
+aafb9f7bec760b90674d44433d49b8ea46d6cb9e
+1730831164.6254377

.huggingface/download/README.md.metadata

@@ -0,0 +1,3 @@
+1d5ece72b189f7322a5405e9682711ffe9580378
+30ae5c954ea1067c62d7fc3447832e17512eb2c2
+1730831164.8530679

.huggingface/download/assets/grid.gif.metadata

@@ -0,0 +1,3 @@
+1d5ece72b189f7322a5405e9682711ffe9580378
+85588586e345d5302112486b34a59c34240ab759c3048f0abc3868d8f3fdaa5d
+1730831165.5925496

.huggingface/download/decoder.safetensors.metadata

@@ -0,0 +1,3 @@
+1d5ece72b189f7322a5405e9682711ffe9580378
+641920faaf20e5404ddb5553ce3e295c21ed9b4bc5f6fe7c930811b84099cb14
+1730831176.4544442

.huggingface/download/dit.safetensors.metadata

@@ -0,0 +1,3 @@
+1d5ece72b189f7322a5405e9682711ffe9580378
+5c273a0d4fbb0de835e31b7cb5228089ba0ee54d4c3a087e81d354cef3d8bbf1
+1730831387.0177512

.huggingface/download/encoder.safetensors.metadata

@@ -0,0 +1,3 @@
+1d5ece72b189f7322a5405e9682711ffe9580378
+d3a8827a66b58a479d97420a9bf77e59078d88f538298469d8db28c37bd556ae
+1730831389.0487227

README.md

@@ -27,7 +27,7 @@ Install using [uv](https://github.com/astral-sh/uv):
 
 ```bash
 git clone https://github.com/genmoai/models
-cd models 
+cd models
 pip install uv
 uv venv .venv
 source .venv/bin/activate
@@ -133,7 +133,7 @@ pipe.enable_vae_tiling()
 prompt = "Close-up of a chameleon's eye, with its scaly skin changing color. Ultra high resolution 4k."
 
 with torch.autocast("cuda", torch.bfloat16, cache_enabled=False):
-      frames = pipe(prompt, num_frames=84).frames[0]
+      frames = pipe(prompt, num_frames=85).frames[0]
 
 export_to_video(frames, "mochi.mp4", fps=30)
 ```
@@ -154,7 +154,7 @@ pipe.enable_model_cpu_offload()
 pipe.enable_vae_tiling()
 
 prompt = "Close-up of a chameleon's eye, with its scaly skin changing color. Ultra high resolution 4k."
-frames = pipe(prompt, num_frames=84).frames[0]
+frames = pipe(prompt, num_frames=85).frames[0]
 
 export_to_video(frames, "mochi.mp4", fps=30)
 ```
@@ -164,20 +164,20 @@ To learn more check out the [Diffusers](https://huggingface.co/docs/diffusers/ma
 
 ## Model Architecture
 
-Mochi 1 represents a significant advancement in open-source video generation, featuring a 10 billion parameter diffusion model built on our novel Asymmetric Diffusion Transformer (AsymmDiT) architecture. Trained entirely from scratch, it is the largest video generative model ever openly released. And best of all, it’s a simple, hackable architecture. Additionally, we are releasing an inference harness that includes an efficient context parallel implementation. 
+Mochi 1 represents a significant advancement in open-source video generation, featuring a 10 billion parameter diffusion model built on our novel Asymmetric Diffusion Transformer (AsymmDiT) architecture. Trained entirely from scratch, it is the largest video generative model ever openly released. And best of all, it’s a simple, hackable architecture. Additionally, we are releasing an inference harness that includes an efficient context parallel implementation.
 
-Alongside Mochi, we are open-sourcing our video AsymmVAE. We use an asymmetric encoder-decoder structure to build an efficient high quality compression model. Our AsymmVAE causally compresses videos to a 128x smaller size, with an 8x8 spatial and a 6x temporal compression to a 12-channel latent space. 
+Alongside Mochi, we are open-sourcing our video AsymmVAE. We use an asymmetric encoder-decoder structure to build an efficient high quality compression model. Our AsymmVAE causally compresses videos to a 128x smaller size, with an 8x8 spatial and a 6x temporal compression to a 12-channel latent space.
 
 ### AsymmVAE Model Specs
-|Params <br> Count | Enc Base <br>  Channels | Dec Base <br> Channels |Latent <br> Dim | Spatial <br> Compression | Temporal <br> Compression | 
+|Params <br> Count | Enc Base <br>  Channels | Dec Base <br> Channels |Latent <br> Dim | Spatial <br> Compression | Temporal <br> Compression |
 |:--:|:--:|:--:|:--:|:--:|:--:|
-|362M   | 64  | 128  | 12   | 8x8   | 6x   | 
+|362M   | 64  | 128  | 12   | 8x8   | 6x   |
 
 An AsymmDiT efficiently processes user prompts alongside compressed video tokens by streamlining text processing and focusing neural network capacity on visual reasoning. AsymmDiT jointly attends to text and visual tokens with multi-modal self-attention and learns separate MLP layers for each modality, similar to Stable Diffusion 3. However, our visual stream has nearly 4 times as many parameters as the text stream via a larger hidden dimension. To unify the modalities in self-attention, we use non-square QKV and output projection layers. This asymmetric design reduces inference memory requirements.
 Many modern diffusion models use multiple pretrained language models to represent user prompts. In contrast, Mochi 1 simply encodes prompts with a single T5-XXL language model.
 
 ### AsymmDiT Model Specs
-|Params <br> Count | Num <br> Layers | Num <br> Heads | Visual <br> Dim | Text <br> Dim | Visual <br> Tokens | Text <br> Tokens | 
+|Params <br> Count | Num <br> Layers | Num <br> Heads | Visual <br> Dim | Text <br> Dim | Visual <br> Tokens | Text <br> Tokens |
 |:--:|:--:|:--:|:--:|:--:|:--:|:--:|
 |10B   | 48   | 24   | 3072   | 1536   | 44520   |   256   |
 

text_encoder/config.json

@@ -1,4 +1,5 @@
 {
+  "_name_or_path": "google/t5-v1_1-xxl",
   "architectures": [
     "T5EncoderModel"
   ],
@@ -24,7 +25,7 @@
   "relative_attention_max_distance": 128,
   "relative_attention_num_buckets": 32,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float32",
   "transformers_version": "4.45.2",
   "use_cache": true,
   "vocab_size": 32128

text_encoder/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a68b2c8c080696a10109612a649bc69330991ecfea65930ccfdfbdb011f2686
+size 4989319680

text_encoder/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8ed6556d7507e38af5b428c605fb2a6f2bdb7e80bd481308b865f7a40c551ca
+size 4999830656

text_encoder/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c831635f83041f83faf0024b39c6ecb21b45d70dd38a63ea5bac6c7c6e5e558c
+size 4865612720

text_encoder/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02a5f2d69205be92ad48fe5d712d38c2ff55627969116aeffc58bd75a28da468
+size 4194506688

text_encoder/model.safetensors.index.json

@@ -1,226 +1,226 @@
 {
   "metadata": {
-    "total_size": 9524621312
+    "total_size": 19049242624
   },
   "weight_map": {
-    "encoder.block.0.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.0.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.0.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.0.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.0.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.0.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.0.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.0.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.0.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.1.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.1.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.1.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.1.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.1.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.1.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.1.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.1.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.1.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.10.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.10.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.10.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.10.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.10.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.10.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.10.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.10.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.10.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.11.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.11.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.11.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.11.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.11.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.11.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.11.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.11.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.11.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.12.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.12.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.12.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.12.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.12.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.12.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.12.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.12.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.12.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.13.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.13.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.13.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.13.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.13.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.13.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.13.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.13.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.13.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.14.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.14.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.14.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.14.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.14.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.14.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.14.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.14.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.14.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.15.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.15.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.15.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.15.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.15.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.15.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.15.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.15.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.15.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.16.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.16.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.16.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.16.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.16.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.16.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.16.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.16.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.16.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.17.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.17.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.17.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.17.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.17.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.17.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.17.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.17.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.17.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.18.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.18.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.18.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.18.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.18.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.18.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.18.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.18.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.18.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.19.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.19.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.19.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.19.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.19.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.19.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.19.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.19.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.19.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.2.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.2.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.2.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.2.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.2.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.2.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.2.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.2.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.2.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.20.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.20.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.20.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.20.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.20.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.20.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.20.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.20.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.20.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.21.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.21.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.21.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.21.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.21.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.21.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.21.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.21.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.21.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.22.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.22.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.22.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.22.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.22.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.22.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.22.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.22.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.22.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.23.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.23.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.23.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.23.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.23.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.23.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.23.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.23.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.23.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
-    "encoder.block.3.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.3.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.3.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.3.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.3.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.3.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.3.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.3.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.3.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.4.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.4.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.4.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.4.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.4.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.4.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.4.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.4.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.4.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.5.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.5.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.5.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.5.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.5.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.5.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.5.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.5.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.5.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.6.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.6.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.6.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.6.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.6.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.6.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.6.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.6.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.6.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.7.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.7.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.7.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.7.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.7.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.7.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.7.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.7.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.7.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.8.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.8.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.8.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.8.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.8.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.8.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.8.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.8.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.8.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.9.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.9.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.9.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.9.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.9.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.9.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.9.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.9.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
-    "encoder.block.9.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
-    "encoder.final_layer_norm.weight": "model-00002-of-00002.safetensors",
-    "shared.weight": "model-00001-of-00002.safetensors"
+    "encoder.block.0.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.0.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.0.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.0.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.0.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.0.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.0.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.0.layer.1.DenseReluDense.wo.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.0.layer.1.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.1.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.1.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.1.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.1.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.1.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.1.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.1.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.1.layer.1.DenseReluDense.wo.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.1.layer.1.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.10.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.10.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.10.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.10.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.10.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.10.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.10.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.10.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.10.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.11.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.11.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.11.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.11.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.11.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.11.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.11.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.11.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.11.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.12.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.12.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.12.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.12.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.12.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.12.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.12.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.12.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.12.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.13.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.13.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.13.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.13.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.13.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.13.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.13.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.13.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.13.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.14.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.14.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.14.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.14.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.14.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.14.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.14.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.14.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.14.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.15.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.15.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.15.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.15.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.15.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.15.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.15.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.15.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.15.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.16.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.16.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.16.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.16.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.16.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.16.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.16.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.16.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.16.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.17.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.17.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.17.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.17.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.17.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.17.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.17.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.17.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.17.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.18.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.18.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.18.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.18.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.18.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.18.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
+    "encoder.block.18.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.18.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.18.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.19.layer.0.SelfAttention.k.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.19.layer.0.SelfAttention.o.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.19.layer.0.SelfAttention.q.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.19.layer.0.SelfAttention.v.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.19.layer.0.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.19.layer.1.DenseReluDense.wi_0.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.19.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.19.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.19.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.2.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.2.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.2.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.2.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.2.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.2.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.2.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.2.layer.1.DenseReluDense.wo.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.2.layer.1.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.20.layer.0.SelfAttention.k.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.20.layer.0.SelfAttention.o.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.20.layer.0.SelfAttention.q.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.20.layer.0.SelfAttention.v.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.20.layer.0.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.20.layer.1.DenseReluDense.wi_0.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.20.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.20.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.20.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.21.layer.0.SelfAttention.k.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.21.layer.0.SelfAttention.o.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.21.layer.0.SelfAttention.q.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.21.layer.0.SelfAttention.v.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.21.layer.0.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.21.layer.1.DenseReluDense.wi_0.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.21.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.21.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.21.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.22.layer.0.SelfAttention.k.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.22.layer.0.SelfAttention.o.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.22.layer.0.SelfAttention.q.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.22.layer.0.SelfAttention.v.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.22.layer.0.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.22.layer.1.DenseReluDense.wi_0.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.22.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.22.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.22.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.23.layer.0.SelfAttention.k.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.23.layer.0.SelfAttention.o.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.23.layer.0.SelfAttention.q.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.23.layer.0.SelfAttention.v.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.23.layer.0.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.23.layer.1.DenseReluDense.wi_0.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.23.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.23.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.23.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.block.3.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.3.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.3.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.3.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.3.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.3.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.3.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.3.layer.1.DenseReluDense.wo.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.3.layer.1.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.4.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.4.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.4.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.4.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.4.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.4.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.4.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.4.layer.1.DenseReluDense.wo.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.4.layer.1.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.5.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.5.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.5.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.5.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.5.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.5.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.5.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
+    "encoder.block.5.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.5.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.6.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.6.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.6.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.6.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.6.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.6.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.6.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.6.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.6.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.7.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.7.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.7.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.7.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.7.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.7.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.7.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.7.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.7.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.8.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.8.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.8.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.8.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.8.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.8.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.8.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.8.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.8.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.9.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.9.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.9.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.9.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.9.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.9.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.9.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.9.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
+    "encoder.block.9.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
+    "encoder.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "shared.weight": "model-00001-of-00004.safetensors"
   }
 }

tokenizer/tokenizer_config.json

@@ -932,7 +932,7 @@
   "eos_token": "</s>",
   "extra_ids": 100,
   "legacy": true,
-  "model_max_length": 256,
+  "model_max_length": 512,
   "pad_token": "<pad>",
   "sp_model_kwargs": {},
   "tokenizer_class": "T5Tokenizer",