Upload LongLlamaForCausalLM

config.json

@@ -1,24 +1,38 @@
 {
-  "_name_or_path": "openlm-research/open_llama_3b_v2",
+  "_name_or_path": "syzymon/long_llama_3b_instruct",
   "architectures": [
-    "LlamaForCausalLM"
+    "LongLlamaForCausalLM"
   ],
+  "auto_map": {
+    "AutoConfig": "syzymon/long_llama_3b_instruct--configuration_longllama.LongLlamaConfig",
+    "AutoModel": "syzymon/long_llama_3b_instruct--modeling_longllama.LongLlamaModel",
+    "AutoModelForCausalLM": "syzymon/long_llama_3b_instruct--modeling_longllama.LongLlamaForCausalLM",
+    "AutoModelForSequenceClassification": "syzymon/long_llama_3b_instruct--modeling_longllama.LongLlamaForSequenceClassification"
+  },
   "bos_token_id": 1,
   "eos_token_id": 2,
+  "gradient_checkpoint_every_ith": 1,
   "hidden_act": "silu",
   "hidden_size": 3200,
   "initializer_range": 0.02,
   "intermediate_size": 8640,
+  "last_context_length": 1024,
   "max_position_embeddings": 2048,
-  "model_type": "llama",
+  "mem_attention_grouping": null,
+  "mem_dtype": "bfloat16",
+  "mem_layers": [
+    6,
+    12,
+    18
+  ],
+  "mem_positionals": true,
+  "model_type": "longllama",
   "num_attention_heads": 32,
   "num_hidden_layers": 26,
-  "num_key_value_heads": 32,
   "pad_token_id": 0,
-  "pretraining_tp": 1,
   "rms_norm_eps": 1e-06,
-  "rope_scaling": null,
   "tie_word_embeddings": false,
+  "torch_attention": false,
   "torch_dtype": "float32",
   "transformers_version": "4.32.0.dev0",
   "use_cache": true,

pytorch_model-00001-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:415502eebda29a3b0ab24bfb0ed994248bb76dfdbb343fe17ef7ecd7093df51f
-size 2969744687
+oid sha256:907751722ee5d3371a6f0d424e2c9a267d7428fcc9b9b7a7e029d138cfe1b17e
+size 2969747401

pytorch_model-00002-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e50b0488017b082e1626674fb6fde181fc8b4fc4fcc0724b0e31cb75e549442f
-size 2973868379
+oid sha256:5909dd80234c548c3c4e2fb1d84d0d7d2c5d39d749f9596a8e5cfebc44aa22aa
+size 2973871623

pytorch_model-00003-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7339d8ba2e5042d3772a6977f9d9fe114b2a69575ec9dfabbff0dcdbf87c3878
-size 2973868443
+oid sha256:dbbb1f3f7d980e94f4ee9f721713c648ae8664dbd111620633bcd9aeaf06b1b7
+size 2973871687

pytorch_model-00004-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:587e87eed898a771eaf72033fa6a8d264115b4338246037f5c67487a4886efc2
-size 2973868443
+oid sha256:885cc675d2fe901453a5faf3d39544e8e85bb2654046903b8d9a2b0c11cc2b06
+size 2973871687

pytorch_model-00005-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb532c05770fbd249eaba8bb5132473d6544d38208bbce1a04a5090574712323
-size 1814626933
+oid sha256:f548dd3cd3fda7cad7214fbe114ff7821e6e7b2414abb1029d06892ba21d875a
+size 1814628523

pytorch_model.bin.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 13705894400
+    "total_size": 13705899600
   },
   "weight_map": {
     "lm_head.weight": "pytorch_model-00005-of-00005.bin",
@@ -13,6 +13,7 @@
     "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
+    "model.layers.0.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
     "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
@@ -22,6 +23,7 @@
     "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
+    "model.layers.1.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
     "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.10.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.10.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -31,6 +33,7 @@
     "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
+    "model.layers.10.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.11.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.11.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -40,6 +43,7 @@
     "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
+    "model.layers.11.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.12.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.12.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -49,6 +53,7 @@
     "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
+    "model.layers.12.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.13.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.13.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -58,6 +63,7 @@
     "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
+    "model.layers.13.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.14.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.14.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -67,6 +73,7 @@
     "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
+    "model.layers.14.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.15.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.15.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -76,6 +83,7 @@
     "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
+    "model.layers.15.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.16.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.16.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -85,6 +93,7 @@
     "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
+    "model.layers.16.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.17.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.17.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -94,6 +103,7 @@
     "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
+    "model.layers.17.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.18.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.18.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -103,6 +113,7 @@
     "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
+    "model.layers.18.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.19.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.19.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -112,6 +123,7 @@
     "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
+    "model.layers.19.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
@@ -121,6 +133,7 @@
     "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
+    "model.layers.2.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
     "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.20.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.20.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -130,6 +143,7 @@
     "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
+    "model.layers.20.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.21.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.21.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -139,6 +153,7 @@
     "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
+    "model.layers.21.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.22.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.22.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -148,6 +163,7 @@
     "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
+    "model.layers.22.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.23.input_layernorm.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.23.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin",
@@ -157,6 +173,7 @@
     "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
+    "model.layers.23.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin",
     "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.24.input_layernorm.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.24.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin",
@@ -166,6 +183,7 @@
     "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00005-of-00005.bin",
+    "model.layers.24.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin",
     "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.25.input_layernorm.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.25.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin",
@@ -175,6 +193,7 @@
     "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00005-of-00005.bin",
+    "model.layers.25.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin",
     "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
@@ -184,6 +203,7 @@
     "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
+    "model.layers.3.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
     "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
@@ -193,6 +213,7 @@
     "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
+    "model.layers.4.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
     "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.5.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.5.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -202,6 +223,7 @@
     "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
+    "model.layers.5.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.6.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.6.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -211,6 +233,7 @@
     "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
+    "model.layers.6.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.7.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.7.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -220,6 +243,7 @@
     "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
+    "model.layers.7.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.8.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.8.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -229,6 +253,7 @@
     "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
+    "model.layers.8.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.9.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.9.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -238,6 +263,7 @@
     "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
+    "model.layers.9.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.norm.weight": "pytorch_model-00005-of-00005.bin"
   }