retnet-xsum_model_70k-28_1M

README.md

@@ -15,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [](https://huggingface.co/) on the xsum dataset.
 It achieves the following results on the evaluation set:
-- Loss: 4.5288
+- Loss: 4.0200
 
 ## Model description
 
@@ -35,31 +35,29 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0006
-- train_batch_size: 1
-- eval_batch_size: 1
+- train_batch_size: 8
+- eval_batch_size: 8
 - seed: 42
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 4
+- total_train_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 10
-- num_epochs: 10
+- num_epochs: 8
 - mixed_precision_training: Native AMP
 
 ### Training results
 
 | Training Loss | Epoch | Step  | Validation Loss |
 |:-------------:|:-----:|:-----:|:---------------:|
-| 5.5013        | 1.0   | 2500  | 4.8933          |
-| 4.5772        | 2.0   | 5000  | 4.5701          |
-| 4.1583        | 3.0   | 7500  | 4.4502          |
-| 3.8363        | 4.0   | 10000 | 4.3856          |
-| 3.5506        | 5.0   | 12500 | 4.3816          |
-| 3.2924        | 6.0   | 15000 | 4.4053          |
-| 3.0678        | 7.0   | 17500 | 4.4400          |
-| 2.8893        | 8.0   | 20000 | 4.4869          |
-| 2.7668        | 9.0   | 22500 | 4.5172          |
-| 2.7036        | 10.0  | 25000 | 4.5288          |
+| 5.3257        | 1.0   | 2187  | 4.6412          |
+| 4.5863        | 2.0   | 4375  | 4.3474          |
+| 4.3703        | 3.0   | 6562  | 4.2111          |
+| 4.2404        | 4.0   | 8750  | 4.1213          |
+| 4.1568        | 5.0   | 10937 | 4.0673          |
+| 4.0975        | 6.0   | 13125 | 4.0371          |
+| 4.0618        | 7.0   | 15312 | 4.0219          |
+| 4.045         | 8.0   | 17496 | 4.0200          |
 
 
 ### Framework versions

config.json

@@ -1,17 +1,17 @@
 {
-  "activation_dropout": 0.0,
+  "activation_dropout": 0.1,
   "activation_fn": "swish",
   "architectures": [
     "RetNetForCausalLM"
   ],
-  "decoder_embed_dim": 512,
-  "decoder_ffn_embed_dim": 864,
-  "decoder_layers": 4,
+  "decoder_embed_dim": 256,
+  "decoder_ffn_embed_dim": 432,
+  "decoder_layers": 3,
   "decoder_normalize_before": true,
   "decoder_retention_heads": 2,
-  "decoder_value_embed_dim": 864,
+  "decoder_value_embed_dim": 432,
   "deepnorm": false,
-  "drop_path_rate": 0.0,
+  "drop_path_rate": 0.1,
   "dropout": 0.1,
   "eos_token_id": 50256,
   "forward_impl": "parallel",
@@ -23,7 +23,7 @@
   "no_scale_embedding": false,
   "output_retentions": false,
   "pad_token_id": 50257,
-  "recurrent_chunk_size": 512,
+  "recurrent_chunk_size": 256,
   "subln": true,
   "tie_word_embeddings": false,
   "torch_dtype": "float32",

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:26e8b5ce8a75937cbe2bfedd62db58c48a622baaf6948a10dfc5afd7e4d65e2c
-size 256743288
+oid sha256:ec44e3271173c6e75a66acc989aa19a83850d775155a5a968a248b07ee0cd3da
+size 112478520

runs/Nov03_22-11-48_cbf29f69c7ee/events.out.tfevents.1699049509.cbf29f69c7ee.2454.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c8681c92acdca48d5c00bced2a5e295a7e8f1e94892319125a30623875b8a0d
+size 8207

runs/Nov03_22-11-48_cbf29f69c7ee/events.out.tfevents.1699067684.cbf29f69c7ee.2454.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0462915650419b4b9e2fe38a407d605f91531f643c1f577bc213d80f880a3cf
+size 364

tokenizer.json

@@ -1,7 +1,16 @@
 {
   "version": "1.0",
   "truncation": null,
-  "padding": null,
+  "padding": {
+    "strategy": {
+      "Fixed": 1024
+    },
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 50257,
+    "pad_type_id": 0,
+    "pad_token": "<|pad|>"
+  },
   "added_tokens": [
     {
       "id": 50256,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f079f397e15b5a36b5952f2298243ba98bc70f542dda8e1bebe1c8869c260cc
+oid sha256:011be80e171330f986658f62208876e42ea8718c3e05fb87230fcb064c1711b8
 size 4536