Upload 9 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,1000 @@
+{
+  "architectures": [
+    "DeciLMForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_decilm.DeciLMConfig",
+    "AutoModelForCausalLM": "modeling_decilm.DeciLMForCausalLM"
+  },
+  "block_configs": [
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 16,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 32,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 32,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 64,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 64,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 32,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 32,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 64,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 32,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 32,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": false,
+        "replace_with_linear": true
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "replace_with_linear": false
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false
+      }
+    }
+  ],
+  "bos_token_id": 128000,
+  "eos_token_id": 128009,
+  "hidden_act": "silu",
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "nemotron-nas",
+  "num_attention_heads": 64,
+  "num_hidden_layers": 80,
+  "num_key_value_heads": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "vocab_size": 128256
+}

configuration_decilm.py

@@ -0,0 +1,99 @@
+# coding=utf-8
+# Copyright 2024 Nvidia Corporation. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import dataclasses
+import warnings
+from dataclasses import dataclass, MISSING
+from functools import partial
+from typing import Optional, Dict, Any
+
+from .transformers_4_44_2__configuration_llama import LlamaConfig
+from .transformers_4_44_2__modeling_rope_utils import \
+    rope_config_validation  # fake import to make AutoConfig infer the dependency
+
+
+class DeciLMConfig(LlamaConfig):
+    model_type = "nemotron-nas"
+
+    def __init__(
+            self,
+            block_configs: list[dict] | list["BlockConfig"] = None,
+            **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.intermediate_size = None
+        self.num_key_value_heads = None
+
+        if block_configs is not None:
+            assert len(block_configs) == self.num_hidden_layers
+            if isinstance(block_configs[0], dict):
+                block_configs = [BlockConfig(**conf) for conf in block_configs]
+        self.block_configs: list[BlockConfig] = block_configs
+
+    def to_dict(self) -> Dict[str, Any]:
+        self_dict = super().to_dict()
+        if self.block_configs is not None:
+            self_dict["block_configs"] = [dataclasses.asdict(conf) for conf in self.block_configs]
+        return self_dict
+
+
+@partial(dataclass, frozen=True, eq=True, unsafe_hash=True, order=True)
+class AttentionConfig:
+    no_op: bool = False
+    replace_with_linear: bool = False
+    n_heads_in_group: Optional[int] = None
+
+    def __post_init__(self):
+        assert not (self.no_op and self.replace_with_linear)
+        if self.no_op or self.replace_with_linear:
+            object.__setattr__(self, 'n_heads_in_group', None)  # __setattr__ to overcome frozen=True
+        else:
+            assert self.n_heads_in_group is not None
+
+
+@partial(dataclass, frozen=True, eq=True, unsafe_hash=True, order=True)
+class FFNConfig:
+    no_op: bool = False
+    replace_with_linear: bool = False
+    ffn_mult: Optional[float] = None
+
+    def __post_init__(self):
+        assert not (self.no_op and self.replace_with_linear)
+        if self.no_op or self.replace_with_linear:
+            object.__setattr__(self, 'ffn_mult', None)  # __setattr__ to overcome frozen=True
+        else:
+            assert self.ffn_mult is not None
+
+
+@partial(dataclass, frozen=True, eq=True, unsafe_hash=True, order=True)
+class BlockConfig:
+    attention: AttentionConfig = MISSING
+    ffn: FFNConfig = MISSING
+
+    def __post_init__(self):
+        """
+        Init subblock dataclasses from dicts
+        """
+        for subblock_name in dataclasses.fields(self):
+            subblock_config = getattr(self, subblock_name.name)
+            if isinstance(subblock_config, dict):
+                subblock_fields = [field.name for field in dataclasses.fields(subblock_name.type)]
+                unsupported_fields = [field_name for field_name in subblock_config.keys()
+                                      if field_name not in subblock_fields]
+                if len(unsupported_fields) > 0:
+                    warnings.warn(f"Removed unsupported fields {unsupported_fields} from {subblock_name.type.__name__}")
+                subblock_config = {k: v for k, v in subblock_config.items() if k not in unsupported_fields}
+                object.__setattr__(self, subblock_name.name,
+                                   subblock_name.type(**subblock_config))  # __setattr__ to overcome frozen=True

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "transformers_version": "4.51.3"
+}

model.safetensors.index.json

@@ -0,0 +1,636 @@
+{
+  "metadata": {
+    "total_size": 103002030080
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00022-of-00022.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00022.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00022.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00022.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00022.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00022.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00022.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00022.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00022.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00004-of-00022.safetensors",
+    "model.layers.11.self_attn.linear_attn.weight": "model-00004-of-00022.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00004-of-00022.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00004-of-00022.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00004-of-00022.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00022.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00022.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00022.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00022.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00022.safetensors",
+    "model.layers.15.self_attn.linear_attn.weight": "model-00004-of-00022.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00022.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00022.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00022.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00005-of-00022.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00005-of-00022.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00005-of-00022.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00022.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00005-of-00022.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00006-of-00022.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00022.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00022.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00022.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00022.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00006-of-00022.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00006-of-00022.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00006-of-00022.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00006-of-00022.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00006-of-00022.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00006-of-00022.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00007-of-00022.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00007-of-00022.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00007-of-00022.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00007-of-00022.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00007-of-00022.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00007-of-00022.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00008-of-00022.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00008-of-00022.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00008-of-00022.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00008-of-00022.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00008-of-00022.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00008-of-00022.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00008-of-00022.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00009-of-00022.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00009-of-00022.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00009-of-00022.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00009-of-00022.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00022.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00022.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00009-of-00022.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00009-of-00022.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00009-of-00022.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00010-of-00022.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00010-of-00022.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00010-of-00022.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00010-of-00022.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00010-of-00022.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00011-of-00022.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00010-of-00022.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00011-of-00022.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00011-of-00022.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00011-of-00022.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00011-of-00022.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00011-of-00022.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00011-of-00022.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00012-of-00022.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00012-of-00022.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00012-of-00022.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00012-of-00022.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00012-of-00022.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00012-of-00022.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00013-of-00022.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00013-of-00022.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00022.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00022.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00013-of-00022.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00013-of-00022.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00013-of-00022.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00013-of-00022.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00013-of-00022.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00014-of-00022.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00014-of-00022.safetensors",
+    "model.layers.42.self_attn.linear_attn.weight": "model-00014-of-00022.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00014-of-00022.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00014-of-00022.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00014-of-00022.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00014-of-00022.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00014-of-00022.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00015-of-00022.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00015-of-00022.safetensors",
+    "model.layers.45.self_attn.linear_attn.weight": "model-00015-of-00022.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00015-of-00022.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00015-of-00022.safetensors",
+    "model.layers.46.self_attn.linear_attn.weight": "model-00015-of-00022.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00015-of-00022.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00015-of-00022.safetensors",
+    "model.layers.47.self_attn.linear_attn.weight": "model-00015-of-00022.safetensors",
+    "model.layers.48.input_layernorm.weight": "model-00015-of-00022.safetensors",
+    "model.layers.48.mlp.down_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.48.mlp.gate_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.48.mlp.up_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.48.post_attention_layernorm.weight": "model-00015-of-00022.safetensors",
+    "model.layers.48.self_attn.linear_attn.weight": "model-00015-of-00022.safetensors",
+    "model.layers.49.input_layernorm.weight": "model-00015-of-00022.safetensors",
+    "model.layers.49.mlp.down_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.49.mlp.gate_proj.weight": "model-00015-of-00022.safetensors",
+    "model.layers.49.mlp.up_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.49.post_attention_layernorm.weight": "model-00015-of-00022.safetensors",
+    "model.layers.49.self_attn.linear_attn.weight": "model-00015-of-00022.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00022.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00022.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00022.safetensors",
+    "model.layers.50.mlp.down_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.50.mlp.gate_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.50.mlp.up_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.50.post_attention_layernorm.weight": "model-00016-of-00022.safetensors",
+    "model.layers.51.input_layernorm.weight": "model-00016-of-00022.safetensors",
+    "model.layers.51.mlp.down_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.51.mlp.gate_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.51.mlp.up_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.51.post_attention_layernorm.weight": "model-00016-of-00022.safetensors",
+    "model.layers.51.self_attn.linear_attn.weight": "model-00016-of-00022.safetensors",
+    "model.layers.52.input_layernorm.weight": "model-00016-of-00022.safetensors",
+    "model.layers.52.mlp.down_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.52.mlp.gate_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.52.mlp.up_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.52.post_attention_layernorm.weight": "model-00016-of-00022.safetensors",
+    "model.layers.52.self_attn.k_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.52.self_attn.o_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.52.self_attn.q_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.52.self_attn.v_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.53.mlp.down_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.53.mlp.gate_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.53.mlp.up_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.53.post_attention_layernorm.weight": "model-00016-of-00022.safetensors",
+    "model.layers.54.input_layernorm.weight": "model-00016-of-00022.safetensors",
+    "model.layers.54.mlp.down_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.54.mlp.gate_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.54.mlp.up_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.54.post_attention_layernorm.weight": "model-00016-of-00022.safetensors",
+    "model.layers.54.self_attn.linear_attn.weight": "model-00016-of-00022.safetensors",
+    "model.layers.55.mlp.down_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.55.mlp.gate_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.55.mlp.up_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.55.post_attention_layernorm.weight": "model-00016-of-00022.safetensors",
+    "model.layers.56.input_layernorm.weight": "model-00016-of-00022.safetensors",
+    "model.layers.56.mlp.down_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.56.mlp.gate_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.56.mlp.up_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.56.post_attention_layernorm.weight": "model-00016-of-00022.safetensors",
+    "model.layers.56.self_attn.k_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.56.self_attn.o_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.56.self_attn.q_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.56.self_attn.v_proj.weight": "model-00016-of-00022.safetensors",
+    "model.layers.57.input_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.57.mlp.down_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.57.mlp.gate_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.57.mlp.up_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.57.post_attention_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.57.self_attn.linear_attn.weight": "model-00017-of-00022.safetensors",
+    "model.layers.58.mlp.down_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.58.mlp.gate_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.58.mlp.up_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.58.post_attention_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.59.input_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.59.mlp.down_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.59.mlp.gate_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.59.mlp.up_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.59.post_attention_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.59.self_attn.linear_attn.weight": "model-00017-of-00022.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00022.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00003-of-00022.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.60.input_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.60.mlp.down_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.60.mlp.gate_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.60.mlp.up_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.60.post_attention_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.60.self_attn.linear_attn.weight": "model-00017-of-00022.safetensors",
+    "model.layers.61.mlp.down_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.61.mlp.gate_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.61.mlp.up_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.61.post_attention_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.62.mlp.down_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.62.mlp.gate_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.62.mlp.up_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.62.post_attention_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.63.input_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.63.mlp.down_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.63.mlp.gate_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.63.mlp.up_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.63.post_attention_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.63.self_attn.linear_attn.weight": "model-00017-of-00022.safetensors",
+    "model.layers.64.mlp.down_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.64.mlp.gate_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.64.mlp.up_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.64.post_attention_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.65.mlp.down_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.65.mlp.gate_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.65.mlp.up_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.65.post_attention_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.66.input_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.66.mlp.down_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.66.mlp.gate_proj.weight": "model-00017-of-00022.safetensors",
+    "model.layers.66.mlp.up_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.66.post_attention_layernorm.weight": "model-00017-of-00022.safetensors",
+    "model.layers.66.self_attn.linear_attn.weight": "model-00017-of-00022.safetensors",
+    "model.layers.67.input_layernorm.weight": "model-00018-of-00022.safetensors",
+    "model.layers.67.mlp.down_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.67.mlp.gate_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.67.mlp.up_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.67.post_attention_layernorm.weight": "model-00018-of-00022.safetensors",
+    "model.layers.67.self_attn.linear_attn.weight": "model-00018-of-00022.safetensors",
+    "model.layers.68.input_layernorm.weight": "model-00018-of-00022.safetensors",
+    "model.layers.68.mlp.down_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.68.mlp.gate_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.68.mlp.up_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.68.post_attention_layernorm.weight": "model-00018-of-00022.safetensors",
+    "model.layers.68.self_attn.linear_attn.weight": "model-00018-of-00022.safetensors",
+    "model.layers.69.input_layernorm.weight": "model-00018-of-00022.safetensors",
+    "model.layers.69.mlp.down_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.69.mlp.gate_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.69.mlp.up_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.69.post_attention_layernorm.weight": "model-00018-of-00022.safetensors",
+    "model.layers.69.self_attn.linear_attn.weight": "model-00018-of-00022.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00003-of-00022.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00003-of-00022.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.70.input_layernorm.weight": "model-00018-of-00022.safetensors",
+    "model.layers.70.mlp.down_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.70.mlp.gate_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.70.mlp.up_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.70.post_attention_layernorm.weight": "model-00018-of-00022.safetensors",
+    "model.layers.70.self_attn.k_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.70.self_attn.o_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.70.self_attn.q_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.70.self_attn.v_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.71.input_layernorm.weight": "model-00018-of-00022.safetensors",
+    "model.layers.71.mlp.down_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.71.mlp.gate_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.71.mlp.up_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.71.post_attention_layernorm.weight": "model-00018-of-00022.safetensors",
+    "model.layers.71.self_attn.k_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.71.self_attn.o_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.71.self_attn.q_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.71.self_attn.v_proj.weight": "model-00018-of-00022.safetensors",
+    "model.layers.72.input_layernorm.weight": "model-00019-of-00022.safetensors",
+    "model.layers.72.mlp.down_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.72.mlp.gate_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.72.mlp.up_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.72.post_attention_layernorm.weight": "model-00019-of-00022.safetensors",
+    "model.layers.72.self_attn.k_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.72.self_attn.o_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.72.self_attn.q_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.72.self_attn.v_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.73.input_layernorm.weight": "model-00019-of-00022.safetensors",
+    "model.layers.73.mlp.down_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.73.mlp.gate_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.73.mlp.up_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.73.post_attention_layernorm.weight": "model-00019-of-00022.safetensors",
+    "model.layers.73.self_attn.k_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.73.self_attn.o_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.73.self_attn.q_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.73.self_attn.v_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.74.input_layernorm.weight": "model-00019-of-00022.safetensors",
+    "model.layers.74.mlp.down_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.74.mlp.gate_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.74.mlp.up_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.74.post_attention_layernorm.weight": "model-00019-of-00022.safetensors",
+    "model.layers.74.self_attn.k_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.74.self_attn.o_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.74.self_attn.q_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.74.self_attn.v_proj.weight": "model-00019-of-00022.safetensors",
+    "model.layers.75.input_layernorm.weight": "model-00020-of-00022.safetensors",
+    "model.layers.75.mlp.down_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.75.mlp.gate_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.75.mlp.up_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.75.post_attention_layernorm.weight": "model-00020-of-00022.safetensors",
+    "model.layers.75.self_attn.k_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.75.self_attn.o_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.75.self_attn.q_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.75.self_attn.v_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.76.input_layernorm.weight": "model-00020-of-00022.safetensors",
+    "model.layers.76.mlp.down_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.76.mlp.gate_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.76.mlp.up_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.76.post_attention_layernorm.weight": "model-00020-of-00022.safetensors",
+    "model.layers.76.self_attn.k_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.76.self_attn.o_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.76.self_attn.q_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.76.self_attn.v_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.77.input_layernorm.weight": "model-00020-of-00022.safetensors",
+    "model.layers.77.mlp.down_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.77.mlp.gate_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.77.mlp.up_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.77.post_attention_layernorm.weight": "model-00020-of-00022.safetensors",
+    "model.layers.77.self_attn.k_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.77.self_attn.o_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.77.self_attn.q_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.77.self_attn.v_proj.weight": "model-00020-of-00022.safetensors",
+    "model.layers.78.input_layernorm.weight": "model-00021-of-00022.safetensors",
+    "model.layers.78.mlp.down_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.78.mlp.gate_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.78.mlp.up_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.78.post_attention_layernorm.weight": "model-00021-of-00022.safetensors",
+    "model.layers.78.self_attn.k_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.78.self_attn.o_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.78.self_attn.q_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.78.self_attn.v_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.79.input_layernorm.weight": "model-00021-of-00022.safetensors",
+    "model.layers.79.mlp.down_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.79.mlp.gate_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.79.mlp.up_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.79.post_attention_layernorm.weight": "model-00021-of-00022.safetensors",
+    "model.layers.79.self_attn.k_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.79.self_attn.o_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.79.self_attn.q_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.79.self_attn.v_proj.weight": "model-00021-of-00022.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00022.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00022.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00022.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00022.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00022.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00022.safetensors",
+    "model.norm.weight": "model-00021-of-00022.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb46edf6ecfb53e620af2687d3aea9faee9038924b264033c32683c7d9e9891f
+size 17209921

tokenizer_config.json

@@ -0,0 +1,2064 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}

transformers_4_44_2__configuration_llama.py

@@ -0,0 +1,203 @@
+# coding=utf-8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""LLaMA model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from .transformers_4_44_2__modeling_rope_utils import rope_config_validation
+
+
+class LlamaConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`LlamaModel`]. It is used to instantiate an LLaMA
+    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+    defaults will yield a similar configuration to that of the LLaMA-7B.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 32000):
+            Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`LlamaModel`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 11008):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer decoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used with. Llama 1 supports up to 2048 tokens,
+            Llama 2 up to 4096, CodeLlama up to 16384.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            Padding token id.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            Beginning of stream token id.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            End of stream token id.
+        pretraining_tp (`int`, *optional*, defaults to 1):
+            Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
+            document](https://huggingface.co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism) to
+            understand more about it. This value is necessary to ensure exact reproducibility of the pretraining
+            results. Please refer to [this issue](https://github.com/pytorch/pytorch/issues/76232).
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
+            and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
+            accordingly.
+            Expected contents:
+                `rope_type` (`str`):
+                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
+                    'llama3'], with 'default' being the original RoPE implementation.
+                `factor` (`float`, *optional*):
+                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
+                    most scaling types, a `factor` of x will enable the model to handle sequences of length x *
+                    original maximum pre-trained length.
+                `original_max_position_embeddings` (`int`, *optional*):
+                    Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
+                    pretraining.
+                `attention_factor` (`float`, *optional*):
+                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
+                    computation. If unspecified, it defaults to value recommended by the implementation, using the
+                    `factor` field to infer the suggested value.
+                `beta_fast` (`float`, *optional*):
+                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
+                    ramp function. If unspecified, it defaults to 32.
+                `beta_slow` (`float`, *optional*):
+                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
+                    ramp function. If unspecified, it defaults to 1.
+                `short_factor` (`List[float]`, *optional*):
+                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<
+                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+                    size divided by the number of attention heads divided by 2
+                `long_factor` (`List[float]`, *optional*):
+                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<
+                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+                    size divided by the number of attention heads divided by 2
+                `low_freq_factor` (`float`, *optional*):
+                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
+                `high_freq_factor` (`float`, *optional*):
+                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
+        attention_bias (`bool`, *optional*, defaults to `False`):
+            Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        mlp_bias (`bool`, *optional*, defaults to `False`):
+            Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers.
+
+    ```python
+    >>> from transformers import LlamaModel, LlamaConfig
+
+    >>> # Initializing a LLaMA llama-7b style configuration
+    >>> configuration = LlamaConfig()
+
+    >>> # Initializing a model from the llama-7b style configuration
+    >>> model = LlamaModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "llama"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=32000,
+        hidden_size=4096,
+        intermediate_size=11008,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=None,
+        hidden_act="silu",
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=1,
+        eos_token_id=2,
+        pretraining_tp=1,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        mlp_bias=False,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.pretraining_tp = pretraining_tp
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+        self.mlp_bias = mlp_bias
+
+        # Validate the correctness of rotary position embeddings parameters
+        # BC: if there is a 'type' field, move it to 'rope_type'.
+        if self.rope_scaling is not None and "type" in self.rope_scaling:
+            self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+        rope_config_validation(self)
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

transformers_4_44_2__modeling_rope_utils.py

@@ -0,0 +1,559 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from typing import Optional, Tuple
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import is_torch_available, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+if is_torch_available():
+    import torch
+
+
+def _compute_default_rope_parameters(
+    config: Optional[PretrainedConfig] = None,
+    device: Optional["torch.device"] = None,
+    seq_len: Optional[int] = None,
+    **rope_kwargs,
+) -> Tuple["torch.Tensor", float]:
+    """
+    Computes the inverse frequencies according to the original RoPE implementation
+    Args:
+        config ([`~transformers.PretrainedConfig`]):
+            The model configuration.
+        device (`torch.device`):
+            The device to use for initialization of the inverse frequencies.
+        seq_len (`int`, *optional*):
+            The current sequence length. Unused for this type of RoPE.
+        rope_kwargs (`Dict`, *optional*):
+            BC compatibility with the previous RoPE class instantiation, will be removed in v4.45.
+    Returns:
+        Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
+        post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
+    """
+    if config is not None and len(rope_kwargs) > 0:
+        raise ValueError(
+            "Unexpected arguments: `**rope_kwargs` and `config` are mutually exclusive in "
+            f"`_compute_default_rope_parameters`, got `rope_kwargs`={rope_kwargs} and `config`={config}"
+        )
+    if len(rope_kwargs) > 0:
+        base = rope_kwargs["base"]
+        dim = rope_kwargs["dim"]
+    elif config is not None:
+        base = config.rope_theta
+        partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
+        head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+        dim = int(head_dim * partial_rotary_factor)
+
+    attention_factor = 1.0  # Unused in this type of RoPE
+
+    # Compute the inverse frequencies
+    inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.int64).float().to(device) / dim))
+    return inv_freq, attention_factor
+
+
+def _compute_linear_scaling_rope_parameters(
+    config: Optional[PretrainedConfig] = None,
+    device: Optional["torch.device"] = None,
+    seq_len: Optional[int] = None,
+    **rope_kwargs,
+) -> Tuple["torch.Tensor", float]:
+    """
+    Computes the inverse frequencies with linear scaling. Credits to the Reddit user /u/kaiokendev
+    Args:
+        config ([`~transformers.PretrainedConfig`]):
+            The model configuration.
+        device (`torch.device`):
+            The device to use for initialization of the inverse frequencies.
+        seq_len (`int`, *optional*):
+            The current sequence length. Unused for this type of RoPE.
+        rope_kwargs (`Dict`, *optional*):
+            BC compatibility with the previous RoPE class instantiation, will be removed in v4.45.
+    Returns:
+        Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
+        post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
+    """
+    if config is not None and len(rope_kwargs) > 0:
+        raise ValueError(
+            "Unexpected arguments: `**rope_kwargs` and `config` are mutually exclusive in "
+            f"`_compute_linear_scaling_rope_parameters`, got `rope_kwargs`={rope_kwargs} and `config`={config}"
+        )
+    if len(rope_kwargs) > 0:
+        factor = rope_kwargs["factor"]
+    elif config is not None:
+        factor = config.rope_scaling["factor"]
+
+    # Gets the default RoPE parameters
+    inv_freq, attention_factor = _compute_default_rope_parameters(config, device, seq_len, **rope_kwargs)
+
+    # Then applies linear scaling to the frequencies.
+    # NOTE: originally, scaling was applied to the position_ids. However, we get `embs = inv_freq @ position_ids`, so
+    # applying scaling to the inverse frequencies is equivalent.
+    inv_freq /= factor
+    return inv_freq, attention_factor
+
+
+def _compute_dynamic_ntk_parameters(
+    config: Optional[PretrainedConfig] = None,
+    device: Optional["torch.device"] = None,
+    seq_len: Optional[int] = None,
+    **rope_kwargs,
+) -> Tuple["torch.Tensor", float]:
+    """
+    Computes the inverse frequencies with NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla
+    Args:
+        config ([`~transformers.PretrainedConfig`]):
+            The model configuration.
+        device (`torch.device`):
+            The device to use for initialization of the inverse frequencies.
+        seq_len (`int`, *optional*):
+            The current sequence length, used to update the dynamic RoPE at inference time.
+        rope_kwargs (`Dict`, *optional*):
+            BC compatibility with the previous RoPE class instantiation, will be removed in v4.45.
+    Returns:
+        Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
+        post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
+    """
+    # TODO (joao): use the new `original_max_position_embeddings` from rope_scaling
+    if config is not None and len(rope_kwargs) > 0:
+        raise ValueError(
+            "Unexpected arguments: `**rope_kwargs` and `config` are mutually exclusive in "
+            f"`_compute_dynamic_ntk_parameters`, got `rope_kwargs`={rope_kwargs} and `config`={config}"
+        )
+    if len(rope_kwargs) > 0:
+        base = rope_kwargs["base"]
+        dim = rope_kwargs["dim"]
+        max_position_embeddings = rope_kwargs["max_position_embeddings"]
+        factor = rope_kwargs["factor"]
+    elif config is not None:
+        base = config.rope_theta
+        partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
+        head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+        dim = int(head_dim * partial_rotary_factor)
+        max_position_embeddings = config.max_position_embeddings
+        factor = config.rope_scaling["factor"]
+
+    attention_factor = 1.0  # Unused in this type of RoPE
+
+    # seq_len: default to max_position_embeddings, e.g. at init time
+    seq_len = seq_len if seq_len is not None and seq_len > max_position_embeddings else max_position_embeddings
+
+    # Compute the inverse frequencies
+    base = base * ((factor * seq_len / max_position_embeddings) - (factor - 1)) ** (dim / (dim - 2))
+    inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.int64).float().to(device) / dim))
+    return inv_freq, attention_factor
+
+
+def _compute_yarn_parameters(
+    config: PretrainedConfig, device: "torch.device", seq_len: Optional[int] = None, **rope_kwargs
+) -> Tuple["torch.Tensor", float]:
+    """
+    Computes the inverse frequencies with NTK scaling. Please refer to the
+    [original paper](https://arxiv.org/abs/2309.00071)
+    Args:
+        config ([`~transformers.PretrainedConfig`]):
+            The model configuration.
+        device (`torch.device`):
+            The device to use for initialization of the inverse frequencies.
+        seq_len (`int`, *optional*):
+            The current sequence length. Unused for this type of RoPE.
+        rope_kwargs (`Dict`, *optional*):
+            BC compatibility with the previous RoPE class instantiation, will be removed in v4.45.
+    Returns:
+        Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
+        post-processing scaling factor applied to the computed cos/sin.
+    """
+    # No need to keep BC with yarn, unreleased when this new pattern was created.
+    if len(rope_kwargs) > 0:
+        raise ValueError(
+            f"Unexpected arguments: `**rope_kwargs` should be unset in `_compute_yarn_parameters`, got {rope_kwargs}"
+        )
+
+    base = config.rope_theta
+    partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
+    head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+    dim = int(head_dim * partial_rotary_factor)
+    max_position_embeddings = config.max_position_embeddings
+    factor = config.rope_scaling["factor"]
+
+    # Sets the attention factor as suggested in the paper
+    attention_factor = config.rope_scaling.get("attention_factor")
+    if attention_factor is None:
+        attention_factor = 0.1 * math.log(factor) + 1.0
+
+    # Optional config options
+    # beta_fast/beta_slow: as suggested in the paper, default to 32/1 (correspondingly)
+    beta_fast = config.rope_scaling.get("beta_fast") or 32
+    beta_slow = config.rope_scaling.get("beta_slow") or 1
+
+    # Compute the inverse frequencies
+    def find_correction_dim(num_rotations, dim, base, max_position_embeddings):
+        """Inverse dimension formula to find the dimension based on the number of rotations"""
+        return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / (2 * math.log(base))
+
+    def find_correction_range(low_rot, high_rot, dim, base, max_position_embeddings):
+        """Find dimension range bounds based on rotations"""
+        low = math.floor(find_correction_dim(low_rot, dim, base, max_position_embeddings))
+        high = math.ceil(find_correction_dim(high_rot, dim, base, max_position_embeddings))
+        return max(low, 0), min(high, dim - 1)
+
+    def linear_ramp_factor(min, max, dim):
+        if min == max:
+            max += 0.001  # Prevent singularity
+
+        linear_func = (torch.arange(dim, dtype=torch.float32) - min) / (max - min)
+        ramp_func = torch.clamp(linear_func, 0, 1)
+        return ramp_func
+
+    # Note on variable naming: "interpolation" comes from the original technique, where we interpolate the position IDs
+    # to expand the possible context length. In other words, interpolation = apply scaling factor.
+    pos_freqs = base ** (torch.arange(0, dim, 2).float().to(device) / dim)
+    inv_freq_extrapolation = 1.0 / pos_freqs
+    inv_freq_interpolation = 1.0 / (factor * pos_freqs)
+
+    low, high = find_correction_range(beta_fast, beta_slow, dim, base, max_position_embeddings)
+
+    # Get n-dimensional rotational scaling corrected for extrapolation
+    inv_freq_extrapolation_factor = 1 - linear_ramp_factor(low, high, dim // 2).float().to(device)
+    inv_freq = (
+        inv_freq_interpolation * (1 - inv_freq_extrapolation_factor)
+        + inv_freq_extrapolation * inv_freq_extrapolation_factor
+    )
+
+    return inv_freq, attention_factor
+
+
+def _compute_longrope_parameters(
+    config: PretrainedConfig, device: "torch.device", seq_len: Optional[int] = None, **rope_kwargs
+) -> Tuple["torch.Tensor", float]:
+    """
+    Computes the inverse frequencies with LongRoPE scaling. Please refer to the
+    [original implementation](https://github.com/microsoft/LongRoPE)
+    Args:
+        config ([`~transformers.PretrainedConfig`]):
+            The model configuration.
+        device (`torch.device`):
+            The device to use for initialization of the inverse frequencies.
+        seq_len (`int`, *optional*):
+            The current sequence length. Unused for this type of RoPE.
+        rope_kwargs (`Dict`, *optional*):
+            BC compatibility with the previous RoPE class instantiation, will be removed in v4.45.
+    Returns:
+        Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
+        post-processing scaling factor applied to the computed cos/sin.
+    """
+    # TODO (joao): use the new `original_max_position_embeddings` from rope_scaling
+    # No need to keep BC with longrope, unreleased when this new pattern was created.
+    if len(rope_kwargs) > 0:
+        raise ValueError(
+            "Unexpected arguments: `**rope_kwargs` should be unset in `_compute_longrope_parameters`, got "
+            f"{rope_kwargs}"
+        )
+
+    base = config.rope_theta
+    partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
+    head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+    dim = int(head_dim * partial_rotary_factor)
+    long_factor = config.rope_scaling["long_factor"]
+    short_factor = config.rope_scaling["short_factor"]
+    factor = config.rope_scaling.get("factor")
+    attention_factor = config.rope_scaling.get("attention_factor")
+
+    # NOTE: Phi3 (and potentially other models) modify `max_position_embeddings` and have a
+    # `original_max_position_embeddings` field containing the pretrained value. They use the ratio between these two
+    # values to compute the default attention scaling factor, instead of using `factor`.
+    if hasattr(config, "original_max_position_embeddings"):
+        max_position_embeddings = config.original_max_position_embeddings
+        expanded_max_position_embeddings = config.max_position_embeddings
+        factor = expanded_max_position_embeddings / max_position_embeddings
+    else:
+        max_position_embeddings = config.max_position_embeddings
+        expanded_max_position_embeddings = max_position_embeddings * factor
+
+    # Sets the attention factor as suggested in the paper
+    if attention_factor is None:
+        if factor <= 1.0:
+            attention_factor = 1.0
+        else:
+            attention_factor = math.sqrt(1 + math.log(factor) / math.log(max_position_embeddings))
+
+    # Compute the inverse frequencies -- scaled based on the target sequence length
+    if expanded_max_position_embeddings > max_position_embeddings:
+        ext_factors = torch.tensor(long_factor, dtype=torch.float32, device=device)
+    else:
+        ext_factors = torch.tensor(short_factor, dtype=torch.float32, device=device)
+    inv_freq_shape = torch.arange(0, dim, 2, dtype=torch.int64, device=device).float() / dim
+    inv_freq = 1.0 / (ext_factors * base**inv_freq_shape)
+
+    return inv_freq, attention_factor
+
+
+def _compute_llama3_parameters(
+    config: PretrainedConfig, device: "torch.device", seq_len: Optional[int] = None, **rope_kwargs
+) -> Tuple["torch.Tensor", float]:
+    """
+    Computes the inverse frequencies for llama 3.1.
+
+    Args:
+        config ([`~transformers.PretrainedConfig`]):
+            The model configuration.
+        device (`torch.device`):
+            The device to use for initialization of the inverse frequencies.
+        seq_len (`int`, *optional*):
+            The current sequence length. Unused for this type of RoPE.
+        rope_kwargs (`Dict`, *optional*):
+            BC compatibility with the previous RoPE class instantiation, will be removed in v4.45.
+    Returns:
+        Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
+        post-processing scaling factor applied to the computed cos/sin.
+    """
+    # Gets the default RoPE parameters
+    inv_freq, attention_factor = _compute_default_rope_parameters(config, device, seq_len, **rope_kwargs)
+
+    factor = config.rope_scaling["factor"]  # `8` in the original implementation
+    low_freq_factor = config.rope_scaling["low_freq_factor"]  # `1` in the original implementation
+    high_freq_factor = config.rope_scaling["high_freq_factor"]  # `4` in the original implementation
+    old_context_len = config.rope_scaling["original_max_position_embeddings"]  # `8192` in the original implementation
+
+    low_freq_wavelen = old_context_len / low_freq_factor
+    high_freq_wavelen = old_context_len / high_freq_factor
+
+    wavelen = 2 * math.pi / inv_freq
+    # wavelen < high_freq_wavelen: do nothing
+    # wavelen > low_freq_wavelen: divide by factor
+    inv_freq_llama = torch.where(wavelen > low_freq_wavelen, inv_freq / factor, inv_freq)
+    # otherwise: interpolate between the two, using a smooth factor
+    smooth_factor = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)
+    smoothed_inv_freq = (1 - smooth_factor) * inv_freq_llama / factor + smooth_factor * inv_freq_llama
+    is_medium_freq = ~(wavelen < high_freq_wavelen) * ~(wavelen > low_freq_wavelen)
+    inv_freq_llama = torch.where(is_medium_freq, smoothed_inv_freq, inv_freq_llama)
+
+    return inv_freq_llama, attention_factor
+
+
+# This maps the "rope_type" string field in rope config to the corresponding function to compute the RoPE parameters
+# from the model config. You can append new {'rope_type': callable} pairs to this dictionary to enable custom RoPE
+# parameterizations, as long as the callable has the same signature.
+ROPE_INIT_FUNCTIONS = {
+    "default": _compute_default_rope_parameters,
+    "linear": _compute_linear_scaling_rope_parameters,
+    "dynamic": _compute_dynamic_ntk_parameters,
+    "yarn": _compute_yarn_parameters,
+    "longrope": _compute_longrope_parameters,
+    "llama3": _compute_llama3_parameters,
+}
+
+
+def _check_received_keys(rope_type: str, received_keys: set, required_keys: set, optional_keys: Optional[set] = None):
+    """Compare the received keys in `config.rope_scaling` against the expected and optional keys"""
+    # BC: "rope_type" was originally "type" -- let's gracefully handle it
+    if "rope_type" not in received_keys and "type" in received_keys:
+        received_keys -= {"type"}
+        received_keys.add("rope_type")
+
+    missing_keys = required_keys - received_keys
+    if missing_keys:
+        raise KeyError(f"Missing required keys in `rope_scaling` for 'rope_type'='{rope_type}': {missing_keys}")
+
+    if optional_keys is not None:
+        unused_keys = received_keys - required_keys - optional_keys
+    else:
+        unused_keys = received_keys - required_keys
+    if unused_keys:
+        logger.warning(f"Unrecognized keys in `rope_scaling` for 'rope_type'='{rope_type}': {unused_keys}")
+
+
+def _validate_default_rope_parameters(config: PretrainedConfig):
+    rope_scaling = config.rope_scaling
+    rope_type = rope_scaling.get("rope_type", rope_scaling.get("type", None))  # BC: "rope_type" was originally "type"
+    required_keys = {"rope_type"}
+    received_keys = set(rope_scaling.keys())
+    _check_received_keys(rope_type, received_keys, required_keys)
+
+
+def _validate_linear_scaling_rope_parameters(config: PretrainedConfig):
+    rope_scaling = config.rope_scaling
+    rope_type = rope_scaling.get("rope_type", rope_scaling.get("type", None))  # BC: "rope_type" was originally "type"
+    required_keys = {"rope_type", "factor"}
+    received_keys = set(rope_scaling.keys())
+    _check_received_keys(rope_type, received_keys, required_keys)
+
+    factor = rope_scaling["factor"]
+    if factor is None or not isinstance(factor, float) or factor < 1.0:
+        logger.warning(f"`rope_scaling`'s factor field must be a float >= 1, got {factor}")
+
+
+def _validate_dynamic_scaling_rope_parameters(config: PretrainedConfig):
+    rope_scaling = config.rope_scaling
+    rope_type = rope_scaling.get("rope_type", rope_scaling.get("type", None))  # BC: "rope_type" was originally "type"
+    required_keys = {"rope_type", "factor"}
+    # TODO (joao): update logic for the inclusion of `original_max_position_embeddings`
+    optional_keys = {"original_max_position_embeddings"}
+    received_keys = set(rope_scaling.keys())
+    _check_received_keys(rope_type, received_keys, required_keys, optional_keys)
+
+    factor = rope_scaling["factor"]
+    if factor is None or not isinstance(factor, float) or factor < 1.0:
+        logger.warning(f"`rope_scaling`'s factor field must be a float >= 1, got {factor}")
+
+
+def _validate_yarn_parameters(config: PretrainedConfig):
+    rope_scaling = config.rope_scaling
+    rope_type = rope_scaling.get("rope_type", rope_scaling.get("type", None))  # BC: "rope_type" was originally "type"
+    required_keys = {"rope_type", "factor"}
+    optional_keys = {"attention_factor", "beta_fast", "beta_slow"}
+    received_keys = set(rope_scaling.keys())
+    _check_received_keys(rope_type, received_keys, required_keys, optional_keys)
+
+    factor = rope_scaling["factor"]
+    if factor is None or not isinstance(factor, float) or factor < 1.0:
+        logger.warning(f"`rope_scaling`'s factor field must be a float >= 1, got {factor}")
+
+    attention_factor = rope_scaling.get("attention_factor")
+    if attention_factor is not None and (not isinstance(attention_factor, float) or attention_factor < 0):
+        logger.warning(
+            f"`rope_scaling`'s attention_factor field must be a float greater than 0, got {attention_factor}"
+        )
+    beta_fast = rope_scaling.get("beta_fast")
+    if beta_fast is not None and not isinstance(beta_fast, float):
+        logger.warning(f"`rope_scaling`'s beta_fast field must be a float, got {beta_fast}")
+    beta_slow = rope_scaling.get("beta_slow")
+    if beta_slow is not None and not isinstance(beta_slow, float):
+        logger.warning(f"`rope_scaling`'s beta_slow field must be a float, got {beta_slow}")
+
+    if (beta_fast or 32) < (beta_slow or 1):
+        logger.warning(
+            f"`rope_scaling`'s beta_fast field must be greater than beta_slow, got beta_fast={beta_fast} "
+            f"(defaults to 32 if None) and beta_slow={beta_slow} (defaults to 1 if None)"
+        )
+
+
+def _validate_longrope_parameters(config: PretrainedConfig):
+    rope_scaling = config.rope_scaling
+    rope_type = rope_scaling.get("rope_type", rope_scaling.get("type", None))  # BC: "rope_type" was originally "type"
+    required_keys = {"rope_type", "short_factor", "long_factor"}
+    # TODO (joao): update logic for the inclusion of `original_max_position_embeddings`
+    optional_keys = {"attention_factor", "factor", "original_max_position_embeddings"}
+    received_keys = set(rope_scaling.keys())
+    _check_received_keys(rope_type, received_keys, required_keys, optional_keys)
+
+    partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
+    head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+    dim = int(head_dim * partial_rotary_factor)
+
+    short_factor = rope_scaling.get("short_factor")
+    if not isinstance(short_factor, list) and all(isinstance(x, (int, float)) for x in short_factor):
+        logger.warning(f"`rope_scaling`'s short_factor field must be a list of numbers, got {short_factor}")
+    if not len(short_factor) == dim // 2:
+        logger.warning(f"`rope_scaling`'s short_factor field must have length {dim // 2}, got {len(short_factor)}")
+
+    long_factor = rope_scaling.get("long_factor")
+    if not isinstance(long_factor, list) and all(isinstance(x, (int, float)) for x in long_factor):
+        logger.warning(f"`rope_scaling`'s long_factor field must be a list of numbers, got {long_factor}")
+    if not len(long_factor) == dim // 2:
+        logger.warning(f"`rope_scaling`'s long_factor field must have length {dim // 2}, got {len(long_factor)}")
+
+    # Handle Phi3 divergence: prefer the use of `attention_factor` and/or `factor` over
+    # `original_max_position_embeddings` to compute internal variables. The latter lives outside `rope_scaling` and is
+    # unique to longrope (= undesirable)
+    if hasattr(config, "original_max_position_embeddings"):
+        logger.warning_once(
+            "This model has set a `original_max_position_embeddings` field, to be used together with "
+            "`max_position_embeddings` to determine a scaling factor. Please set the `factor` field of `rope_scaling`"
+            "with this ratio instead -- we recommend the use of this field over `original_max_position_embeddings`, "
+            "as it is compatible with most model architectures."
+        )
+    else:
+        factor = rope_scaling.get("factor")
+        if factor is None:
+            logger.warning("Missing required keys in `rope_scaling`: 'factor'")
+        elif not isinstance(factor, float) or factor < 1.0:
+            logger.warning(f"`rope_scaling`'s factor field must be a float >= 1, got {factor}")
+
+        attention_factor = rope_scaling.get("attention_factor")
+        if attention_factor is not None and not isinstance(attention_factor, float) or attention_factor < 0:
+            logger.warning(
+                f"`rope_scaling`'s attention_factor field must be a float greater than 0, got {attention_factor}"
+            )
+
+
+def _validate_llama3_parameters(config: PretrainedConfig):
+    rope_scaling = config.rope_scaling
+    rope_type = rope_scaling.get("rope_type", rope_scaling.get("type", None))  # BC: "rope_type" was originally "type"
+    required_keys = {"rope_type", "factor", "original_max_position_embeddings", "low_freq_factor", "high_freq_factor"}
+    received_keys = set(rope_scaling.keys())
+    _check_received_keys(rope_type, received_keys, required_keys)
+
+    factor = rope_scaling["factor"]
+    if factor is None or not isinstance(factor, float) or factor < 1.0:
+        logger.warning(f"`rope_scaling`'s factor field must be a float >= 1, got {factor}")
+
+    low_freq_factor = rope_scaling["low_freq_factor"]
+    high_freq_factor = rope_scaling["high_freq_factor"]
+    if low_freq_factor is None or not isinstance(low_freq_factor, float):
+        logger.warning(f"`rope_scaling`'s low_freq_factor field must be a float, got {low_freq_factor}")
+    if high_freq_factor is None or not isinstance(high_freq_factor, float):
+        logger.warning(f"`rope_scaling`'s high_freq_factor field must be a float, got {high_freq_factor}")
+    if high_freq_factor <= low_freq_factor:
+        logger.warning(
+            "`rope_scaling`'s high_freq_factor field must be greater than low_freq_factor, got high_freq_factor="
+            f"{high_freq_factor} and low_freq_factor={low_freq_factor}"
+        )
+
+    original_max_position_embeddings = rope_scaling["original_max_position_embeddings"]
+    if original_max_position_embeddings is None or not isinstance(original_max_position_embeddings, int):
+        logger.warning(
+            "`rope_scaling`'s original_max_position_embeddings field must be an integer, got "
+            f"{original_max_position_embeddings}"
+        )
+    if original_max_position_embeddings >= config.max_position_embeddings:
+        logger.warning(
+            "`rope_scaling`'s original_max_position_embeddings field must be less than max_position_embeddings, got "
+            f"{original_max_position_embeddings} and max_position_embeddings={config.max_position_embeddings}"
+        )
+
+
+# Like `ROPE_INIT_FUNCTIONS`, this validation function mapping can be dynamically updated for custom RoPE types.
+ROPE_VALIDATION_FUNCTIONS = {
+    "default": _validate_default_rope_parameters,
+    "linear": _validate_linear_scaling_rope_parameters,
+    "dynamic": _validate_dynamic_scaling_rope_parameters,
+    "yarn": _validate_yarn_parameters,
+    "longrope": _validate_longrope_parameters,
+    "llama3": _validate_llama3_parameters,
+}
+
+
+def rope_config_validation(config: PretrainedConfig):
+    """
+    Validate the RoPE config arguments, given a `PretrainedConfig` object
+    """
+    rope_scaling = getattr(config, "rope_scaling", None)  # not a default parameter in `PretrainedConfig`
+    if rope_scaling is None:
+        return
+
+    # BC: "rope_type" was originally "type"
+    rope_type = rope_scaling.get("rope_type", rope_scaling.get("type", "default"))
+    validation_fn = ROPE_VALIDATION_FUNCTIONS.get(rope_type)
+    if validation_fn is not None:
+        validation_fn(config)
+    else:
+        logger.warning(
+            f"Missing validation function mapping in `ROPE_VALIDATION_FUNCTIONS` for 'rope_type'='{rope_type}'"
+        )