Training in progress, step 500

config.json

@@ -1,48 +1,125 @@
 {
-  "_name_or_path": "mnaylor/mega-base-wikitext",
-  "activation": "silu",
-  "add_lm_hidden_dense_layer": false,
-  "add_token_type_embeddings": false,
+  "_name_or_path": "google/pegasus-large",
+  "activation_dropout": 0.1,
+  "activation_function": "relu",
+  "add_bias_logits": false,
+  "add_final_layer_norm": true,
   "architectures": [
-    "MegaForCausalLM"
+    "PegasusForCausalLM"
   ],
-  "attention_activation": "softmax",
-  "attention_probs_dropout_prob": 0.1,
-  "bidirectional": false,
+  "attention_dropout": 0.1,
   "bos_token_id": 0,
-  "chunk_size": -1,
-  "classifier_dropout": null,
-  "dropout_prob": 0.1,
-  "ema_beta_range": 0.02,
-  "ema_delta_alpha_range": 0.2,
-  "ema_gamma_omega_range": 1.0,
-  "ema_projection_size": 16,
-  "eos_token_id": 2,
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 128,
-  "initializer_range": 0.02,
-  "intermediate_size": 256,
+  "classif_dropout": 0.0,
+  "classifier_dropout": 0.0,
+  "d_model": 1024,
+  "decoder_attention_heads": 16,
+  "decoder_ffn_dim": 4096,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 16,
+  "decoder_start_token_id": 0,
+  "dropout": 0.1,
+  "encoder_attention_heads": 16,
+  "encoder_ffn_dim": 4096,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 16,
+  "eos_token_id": 1,
+  "extra_pos_embeddings": 1,
+  "force_bos_token_to_be_generated": false,
+  "forced_eos_token_id": 1,
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
   "is_decoder": true,
-  "max_positions": 10000,
-  "model_type": "mega",
-  "nffn_activation_dropout_prob": 0.0,
-  "nffn_hidden_size": 256,
-  "norm_affine": true,
-  "normalization_type": "scalenorm",
-  "normalize_before_ffn": true,
-  "normalize_before_mega": true,
-  "num_attention_heads": 1,
-  "num_hidden_layers": 4,
-  "pad_token_id": 1,
-  "relative_positional_bias": "rotary",
-  "shared_representation_size": 64,
+  "is_encoder_decoder": false,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "length_penalty": 0.8,
+  "max_length": 256,
+  "max_position_embeddings": 1024,
+  "model_type": "pegasus",
+  "normalize_before": true,
+  "normalize_embedding": false,
+  "num_beams": 8,
+  "num_hidden_layers": 16,
+  "pad_token_id": 0,
+  "scale_embedding": true,
+  "static_position_embeddings": true,
+  "task_specific_params": {
+    "summarization_aeslc": {
+      "length_penalty": 0.6,
+      "max_length": 32,
+      "max_position_embeddings": 512
+    },
+    "summarization_arxiv": {
+      "length_penalty": 0.8,
+      "max_length": 256,
+      "max_position_embeddings": 1024
+    },
+    "summarization_big_patent": {
+      "length_penalty": 0.7,
+      "max_length": 256,
+      "max_position_embeddings": 1024
+    },
+    "summarization_billsum": {
+      "length_penalty": 0.6,
+      "max_length": 256,
+      "max_position_embeddings": 1024
+    },
+    "summarization_cnn_dailymail": {
+      "length_penalty": 0.8,
+      "max_length": 128,
+      "max_position_embeddings": 1024
+    },
+    "summarization_gigaword": {
+      "length_penalty": 0.6,
+      "max_length": 32,
+      "max_position_embeddings": 128
+    },
+    "summarization_large": {
+      "length_penalty": 0.8,
+      "max_length": 256,
+      "max_position_embeddings": 1024
+    },
+    "summarization_multi_news": {
+      "length_penalty": 0.8,
+      "max_length": 256,
+      "max_position_embeddings": 1024
+    },
+    "summarization_newsroom": {
+      "length_penalty": 0.8,
+      "max_length": 128,
+      "max_position_embeddings": 512
+    },
+    "summarization_pubmed": {
+      "length_penalty": 0.8,
+      "max_length": 256,
+      "max_position_embeddings": 1024
+    },
+    "summarization_reddit_tifu": {
+      "length_penalty": 0.6,
+      "max_length": 128,
+      "max_position_embeddings": 512
+    },
+    "summarization_wikihow": {
+      "length_penalty": 0.6,
+      "max_length": 256,
+      "max_position_embeddings": 512
+    },
+    "summarization_xsum": {
+      "length_penalty": 0.8,
+      "max_length": 64,
+      "max_position_embeddings": 512
+    }
+  },
   "torch_dtype": "float32",
   "transformers_version": "4.30.0",
-  "truncation": null,
-  "type_vocab_size": 2,
   "use_cache": true,
-  "use_chunking": false,
-  "use_feature_dropout": false,
-  "use_normalized_ffn": true,
-  "vocab_size": 50265
+  "vocab_size": 96103
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b722f4c58c9c4f2a68578fd19cbbfe0cc03f6d3a95639e12acf91a2f9b51c78b
-size 29203129
+oid sha256:adb9a39b93b38fa6f7e8552865ffc976d1cc40c0d2e18ab4873e20828bfb6a66
+size 1472971290

runs/Apr27_05-25-56_c93ac8136d01/events.out.tfevents.1714195559.c93ac8136d01.1850.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41f62828d44aae7e7b392ac1f6a8e12a8362ab7bcdafada3b003d7f66f6d428b
-size 6337
+oid sha256:3b6410bf2fe66f275f88d1ea568dc6b0fcc08e9fb618a8043a6cf1f2d7ee1c79
+size 6962

runs/Apr27_05-25-56_c93ac8136d01/events.out.tfevents.1714195809.c93ac8136d01.1850.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:629e491ffd6fcdeb8c10df9def8b092d0fdd5864566a8521b22b514fd9b848e5
+size 359

runs/Apr27_05-36-46_c93ac8136d01/events.out.tfevents.1714196210.c93ac8136d01.1850.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b20551ed87d6749823e389a5f13a7a53e906e691c666560245a3ba177110b8e
+size 6665

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee4cbc38cb5a81b6648f0866e2afa564d4b6d7fab3b73343137252b14cee19f7
+oid sha256:a10e960aa41a195a658e95225e72d92fa69a09ee27700190c845f0ff9e4debe9
 size 4408