Upload Llama4ForCausalLM

config.json

@@ -6,7 +6,7 @@
   "attention_chunk_size": 8192,
   "attention_dropout": 0.0,
   "attn_scale": 0.1,
-  "attn_temperature_tuning": true,
+  "attn_temperature_tuning": 4,
   "bos_token_id": 2,
   "cache_implementation": "hybrid",
   "eos_token_id": 1,
@@ -54,7 +54,7 @@
   "num_attention_heads": 16,
   "num_experts_per_tok": 1,
   "num_hidden_layers": 28,
-  "num_key_value_heads": 8,
+  "num_key_value_heads": 16,
   "num_local_experts": 16,
   "output_router_logits": false,
   "pad_token_id": 0,
@@ -65,8 +65,8 @@
   "router_jitter_noise": 0.0,
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.52.4",
+  "transformers_version": "4.51.3",
   "use_cache": true,
-  "use_qk_norm": false,
+  "use_qk_norm": true,
   "vocab_size": 262144
 }

generation_config.json

@@ -4,5 +4,5 @@
   "cache_implementation": "hybrid",
   "eos_token_id": 1,
   "pad_token_id": 0,
-  "transformers_version": "4.52.4"
+  "transformers_version": "4.51.3"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6cd045d8249d33cc9f0f85f9fb65acbc6f8844a3cbd9206d262769d97a02b80
-size 1417821464
+oid sha256:4d5b9374055ac2ec373f39998f435a4866053127773528bb51e7327ae2e8b76e
+size 1535262008