Upload RetNetForCausalLM

config.json

@@ -5,9 +5,12 @@
   ],
   "attn_mode": "fused_chunk",
   "bos_token_id": 1,
+  "conv_size": 4,
+  "elementwise_affine": true,
   "eos_token_id": 2,
   "expand_k": 1,
   "expand_v": 2,
+  "feature_map": null,
   "fuse_cross_entropy": true,
   "fuse_norm": true,
   "hidden_act": "swish",
@@ -17,12 +20,17 @@
   "intermediate_size": null,
   "max_position_embeddings": 2048,
   "model_type": "retnet",
+  "norm_eps": 1e-06,
   "num_heads": 8,
   "num_hidden_layers": 24,
+  "num_kv_heads": null,
   "rms_norm_eps": 1e-06,
+  "share_conv_kernel": true,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.39.2",
+  "transformers_version": "4.40.2",
   "use_cache": true,
+  "use_output_gate": true,
+  "use_short_conv": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.39.2"
+  "transformers_version": "4.40.2"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89334566f60914511481d2768743ae43a330ced47b2fc08591fc57e053840c9d
+oid sha256:0692b44dbc3850d56fa98aceaeb15529852eb0c49b7cb98986000ad3d708e189
 size 2703481256

tokenizer.json

@@ -134,6 +134,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,