update with +2 epochs ft

.gitattributes

@@ -9,10 +9,14 @@
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "pszemraj/long-t5-tglobal-large-pubmed-3k-booksum-16384-WIP6",
+  "_name_or_path": "pszemraj/long-t5-tglobal-large-pubmed-3k-booksum-16384-WIP8",
   "architectures": [
     "LongT5ForConditionalGeneration"
   ],

latest

@@ -1 +1 @@
-global_step150
+global_step125

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:92ab44b991c09f6bc82409f7899997e00371195237de8016934fcb74aa027de4
+oid sha256:9367f037d78d7b661094b03d8b1849046c4e5cdf531566d6d241b7acc40aae13
 size 3132548907

rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:661796353532b01dfcf9f907b7fc04fac2c5b189850da5882dad7779f8759631
+oid sha256:c28d31b3833d0cddbaa6171d231e7ed1b0895c2340b0293c734298c4433b9101
 size 14503

tokenizer_config.json

@@ -103,7 +103,7 @@
   ],
   "eos_token": "</s>",
   "extra_ids": 100,
-  "name_or_path": "pszemraj/long-t5-tglobal-large-pubmed-3k-booksum-16384-WIP6",
+  "name_or_path": "pszemraj/long-t5-tglobal-large-pubmed-3k-booksum-16384-WIP8",
   "pad_token": "<pad>",
   "special_tokens_map_file": null,
   "tokenizer_class": "T5Tokenizer",

trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.8219074598677998,
-  "global_step": 150,
+  "epoch": 1.5197355996222852,
+  "global_step": 125,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -10,457 +10,379 @@
     {
       "epoch": 0.02,
       "learning_rate": 0.0004,
-      "loss": 2.8174,
+      "loss": 2.6298,
       "step": 2
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0008,
-      "loss": 2.8801,
+      "loss": 2.6348,
       "step": 4
     },
     {
       "epoch": 0.07,
-      "learning_rate": 0.0009999575185316995,
-      "loss": 2.8676,
+      "learning_rate": 0.0009999024041442454,
+      "loss": 2.6855,
       "step": 6
     },
     {
       "epoch": 0.1,
-      "learning_rate": 0.0009996177100962712,
-      "loss": 2.8733,
+      "learning_rate": 0.0009991218658821608,
+      "loss": 2.6779,
       "step": 8
     },
     {
       "epoch": 0.12,
-      "learning_rate": 0.000998938324184584,
-      "loss": 2.9105,
+      "learning_rate": 0.0009975620080758321,
+      "loss": 2.7265,
       "step": 10
     },
     {
       "epoch": 0.15,
-      "learning_rate": 0.0009979198225579969,
-      "loss": 2.9035,
+      "learning_rate": 0.000995225266258058,
+      "loss": 2.7332,
       "step": 12
     },
     {
       "epoch": 0.17,
-      "learning_rate": 0.0009965628974662145,
-      "loss": 2.8884,
+      "learning_rate": 0.0009921152889737985,
+      "loss": 2.7308,
       "step": 14
     },
     {
       "epoch": 0.19,
-      "learning_rate": 0.00099486847117678,
-      "loss": 2.879,
+      "learning_rate": 0.0009882369320834067,
+      "loss": 2.7706,
       "step": 16
     },
     {
       "epoch": 0.22,
-      "learning_rate": 0.0009928376953482344,
-      "loss": 2.9521,
+      "learning_rate": 0.0009835962511807786,
+      "loss": 2.7455,
       "step": 18
     },
     {
       "epoch": 0.24,
-      "learning_rate": 0.0009904719502473634,
-      "loss": 2.9526,
+      "learning_rate": 0.0009782004921382612,
+      "loss": 2.7819,
       "step": 20
     },
     {
       "epoch": 0.27,
-      "learning_rate": 0.0009877728438110645,
-      "loss": 2.9211,
+      "learning_rate": 0.0009720580797930845,
+      "loss": 2.7122,
       "step": 22
     },
     {
       "epoch": 0.29,
-      "learning_rate": 0.0009847422105534738,
-      "loss": 2.8804,
+      "learning_rate": 0.0009651786047929772,
+      "loss": 2.7141,
       "step": 24
     },
     {
       "epoch": 0.31,
-      "learning_rate": 0.000981382110319093,
-      "loss": 2.9045,
+      "learning_rate": 0.0009575728086215092,
+      "loss": 2.694,
       "step": 26
     },
     {
       "epoch": 0.34,
-      "learning_rate": 0.0009776948268827657,
-      "loss": 2.9369,
+      "learning_rate": 0.00094925256682654,
+      "loss": 2.6981,
       "step": 28
     },
     {
       "epoch": 0.36,
-      "learning_rate": 0.0009736828663974526,
-      "loss": 2.9049,
+      "learning_rate": 0.0009402308704779598,
+      "loss": 2.6848,
       "step": 30
     },
     {
       "epoch": 0.39,
-      "learning_rate": 0.0009693489556908641,
-      "loss": 2.9069,
+      "learning_rate": 0.0009305218058836777,
+      "loss": 2.6498,
       "step": 32
     },
     {
       "epoch": 0.41,
-      "learning_rate": 0.0009646960404121041,
-      "loss": 2.9257,
+      "learning_rate": 0.0009201405325955221,
+      "loss": 2.7059,
       "step": 34
     },
     {
       "epoch": 0.44,
-      "learning_rate": 0.0009597272830295876,
-      "loss": 2.8909,
+      "learning_rate": 0.0009091032597394012,
+      "loss": 2.7016,
       "step": 36
     },
     {
       "epoch": 0.46,
-      "learning_rate": 0.00095444606068159,
-      "loss": 2.9512,
+      "learning_rate": 0.0008974272207066767,
+      "loss": 2.6599,
       "step": 38
     },
     {
       "epoch": 0.48,
-      "learning_rate": 0.0009488559628808938,
-      "loss": 3.0004,
+      "learning_rate": 0.0008851306462462688,
+      "loss": 2.6922,
       "step": 40
     },
     {
       "epoch": 0.51,
-      "learning_rate": 0.0009429607890750862,
-      "loss": 3.1118,
+      "learning_rate": 0.0008722327359995064,
+      "loss": 2.6473,
       "step": 42
     },
     {
       "epoch": 0.53,
-      "learning_rate": 0.0009367645460641715,
-      "loss": 3.082,
+      "learning_rate": 0.0008587536285221655,
+      "loss": 2.7236,
       "step": 44
     },
     {
       "epoch": 0.56,
-      "learning_rate": 0.0009302714452772515,
-      "loss": 3.0348,
+      "learning_rate": 0.000844714369840506,
+      "loss": 2.6924,
       "step": 46
     },
     {
       "epoch": 0.58,
-      "learning_rate": 0.0009234858999101231,
-      "loss": 3.0918,
+      "learning_rate": 0.0008301368805903988,
+      "loss": 2.6428,
       "step": 48
     },
     {
       "epoch": 0.6,
-      "learning_rate": 0.0009164125219257418,
-      "loss": 3.0042,
+      "learning_rate": 0.0008150439217908557,
+      "loss": 2.6161,
       "step": 50
     },
     {
       "epoch": 0.63,
-      "learning_rate": 0.000909056118919587,
-      "loss": 2.971,
+      "learning_rate": 0.0007994590593054001,
+      "loss": 2.6721,
       "step": 52
     },
     {
       "epoch": 0.65,
-      "learning_rate": 0.0009014216908520619,
-      "loss": 2.9717,
+      "learning_rate": 0.0007834066270467691,
+      "loss": 2.6506,
       "step": 54
     },
     {
       "epoch": 0.68,
-      "learning_rate": 0.0008935144266501469,
-      "loss": 3.0012,
+      "learning_rate": 0.0007669116889823954,
+      "loss": 2.6787,
       "step": 56
     },
     {
       "epoch": 0.7,
-      "learning_rate": 0.0008853397006806182,
-      "loss": 2.9478,
+      "learning_rate": 0.00075,
+      "loss": 2.6172,
       "step": 58
     },
     {
       "epoch": 0.73,
-      "learning_rate": 0.0008769030690972262,
-      "loss": 2.9312,
+      "learning_rate": 0.0007326979656943906,
+      "loss": 2.6071,
       "step": 60
     },
     {
       "epoch": 0.75,
-      "learning_rate": 0.0008682102660643197,
-      "loss": 2.898,
+      "learning_rate": 0.0007150326011382603,
+      "loss": 2.6513,
       "step": 62
     },
     {
       "epoch": 0.77,
-      "learning_rate": 0.0008592671998594795,
-      "loss": 2.8673,
+      "learning_rate": 0.0006970314887013584,
+      "loss": 2.679,
       "step": 64
     },
     {
       "epoch": 0.8,
-      "learning_rate": 0.0008500799488578119,
-      "loss": 2.8984,
+      "learning_rate": 0.0006787227349838946,
+      "loss": 2.6879,
       "step": 66
     },
     {
       "epoch": 0.82,
-      "learning_rate": 0.0008406547574006325,
-      "loss": 2.8778,
+      "learning_rate": 0.0006601349269314187,
+      "loss": 2.6459,
       "step": 68
     },
     {
       "epoch": 0.85,
-      "learning_rate": 0.0008309980315513443,
-      "loss": 2.8978,
+      "learning_rate": 0.0006412970871996995,
+      "loss": 2.6268,
       "step": 70
     },
     {
       "epoch": 0.87,
-      "learning_rate": 0.0008211163347414004,
-      "loss": 2.8657,
+      "learning_rate": 0.0006222386288392914,
+      "loss": 2.615,
       "step": 72
     },
     {
       "epoch": 0.89,
-      "learning_rate": 0.0008110163833093049,
-      "loss": 2.881,
+      "learning_rate": 0.0006029893093705492,
+      "loss": 2.6541,
       "step": 74
     },
     {
       "epoch": 0.92,
-      "learning_rate": 0.0008007050419356898,
-      "loss": 2.8425,
+      "learning_rate": 0.0005835791843207916,
+      "loss": 2.6697,
       "step": 76
     },
     {
       "epoch": 0.94,
-      "learning_rate": 0.0007901893189775639,
-      "loss": 2.899,
+      "learning_rate": 0.0005640385602961634,
+      "loss": 2.6017,
       "step": 78
     },
     {
       "epoch": 0.97,
-      "learning_rate": 0.0007794763617049123,
-      "loss": 2.8116,
+      "learning_rate": 0.0005443979476614675,
+      "loss": 2.658,
       "step": 80
     },
     {
       "epoch": 0.99,
-      "learning_rate": 0.0007685734514428766,
-      "loss": 2.8995,
+      "learning_rate": 0.0005246880129018515,
+      "loss": 2.5922,
       "step": 82
     },
     {
       "epoch": 1.02,
-      "learning_rate": 0.0007574879986228245,
-      "loss": 3.874,
+      "learning_rate": 0.0005049395307407329,
+      "loss": 3.5292,
       "step": 84
     },
     {
       "epoch": 1.05,
-      "learning_rate": 0.000746227537745667,
-      "loss": 2.7812,
+      "learning_rate": 0.0004851833360887201,
+      "loss": 2.5377,
       "step": 86
     },
     {
       "epoch": 1.07,
-      "learning_rate": 0.0007347997222608493,
-      "loss": 2.7706,
+      "learning_rate": 0.0004654502758985611,
+      "loss": 2.6096,
       "step": 88
     },
     {
       "epoch": 1.1,
-      "learning_rate": 0.0007232123193644956,
-      "loss": 2.727,
+      "learning_rate": 0.00044577116100128736,
+      "loss": 2.5112,
       "step": 90
     },
     {
       "epoch": 1.12,
-      "learning_rate": 0.0007114732047202433,
-      "loss": 2.7466,
+      "learning_rate": 0.00042617671799875947,
+      "loss": 2.5469,
       "step": 92
     },
     {
       "epoch": 1.15,
-      "learning_rate": 0.000699590357106354,
-      "loss": 2.7615,
+      "learning_rate": 0.0004066975412877255,
+      "loss": 2.5364,
       "step": 94
     },
     {
       "epoch": 1.17,
-      "learning_rate": 0.0006875718529927403,
-      "loss": 2.7312,
+      "learning_rate": 0.00038736404529030255,
+      "loss": 2.5132,
       "step": 96
     },
     {
       "epoch": 1.19,
-      "learning_rate": 0.0006754258610515948,
-      "loss": 2.7192,
+      "learning_rate": 0.0003682064169654663,
+      "loss": 2.5339,
       "step": 98
     },
     {
       "epoch": 1.22,
-      "learning_rate": 0.0006631606366053507,
-      "loss": 2.7178,
+      "learning_rate": 0.0003492545686756986,
+      "loss": 2.5136,
       "step": 100
     },
     {
       "epoch": 1.24,
-      "learning_rate": 0.0006507845160157477,
-      "loss": 2.7426,
+      "learning_rate": 0.00033053809148238423,
+      "loss": 2.5043,
       "step": 102
     },
     {
       "epoch": 1.27,
-      "learning_rate": 0.0006383059110178205,
-      "loss": 2.78,
+      "learning_rate": 0.00031208620894288106,
+      "loss": 2.5174,
       "step": 104
     },
     {
       "epoch": 1.29,
-      "learning_rate": 0.0006257333030026537,
-      "loss": 2.801,
+      "learning_rate": 0.0002939277314814041,
+      "loss": 2.479,
       "step": 106
     },
     {
       "epoch": 1.31,
-      "learning_rate": 0.0006130752372527982,
-      "loss": 2.8101,
+      "learning_rate": 0.0002760910114049686,
+      "loss": 2.5554,
       "step": 108
     },
     {
       "epoch": 1.34,
-      "learning_rate": 0.0006003403171342563,
-      "loss": 2.8212,
+      "learning_rate": 0.00025860389863462764,
+      "loss": 2.5007,
       "step": 110
     },
     {
       "epoch": 1.36,
-      "learning_rate": 0.0005875371982489958,
-      "loss": 2.8073,
+      "learning_rate": 0.00024149369722112717,
+      "loss": 2.493,
       "step": 112
     },
     {
       "epoch": 1.39,
-      "learning_rate": 0.0005746745825519538,
-      "loss": 2.7252,
+      "learning_rate": 0.00022478712271287087,
+      "loss": 2.4715,
       "step": 114
     },
     {
       "epoch": 1.41,
-      "learning_rate": 0.0005617612124365409,
-      "loss": 2.783,
+      "learning_rate": 0.00020851026044276406,
+      "loss": 2.5359,
       "step": 116
     },
     {
       "epoch": 1.44,
-      "learning_rate": 0.0005488058647926577,
-      "loss": 2.7034,
+      "learning_rate": 0.00019268852479906146,
+      "loss": 2.5447,
       "step": 118
     },
     {
       "epoch": 1.46,
-      "learning_rate": 0.0005358173450412649,
-      "loss": 2.7864,
+      "learning_rate": 0.00017734661954381753,
+      "loss": 2.5141,
       "step": 120
     },
     {
       "epoch": 1.48,
-      "learning_rate": 0.0005228044811495631,
-      "loss": 2.7136,
+      "learning_rate": 0.00016250849924089484,
+      "loss": 2.558,
       "step": 122
     },
     {
       "epoch": 1.51,
-      "learning_rate": 0.0005097761176308471,
-      "loss": 2.762,
+      "learning_rate": 0.00014819733185375534,
+      "loss": 2.5372,
       "step": 124
-    },
-    {
-      "epoch": 1.53,
-      "learning_rate": 0.0004967411095331149,
-      "loss": 2.7633,
-      "step": 126
-    },
-    {
-      "epoch": 1.56,
-      "learning_rate": 0.0004837083164205159,
-      "loss": 2.7863,
-      "step": 128
-    },
-    {
-      "epoch": 1.58,
-      "learning_rate": 0.00047068659635173026,
-      "loss": 2.7523,
-      "step": 130
-    },
-    {
-      "epoch": 1.6,
-      "learning_rate": 0.00045768479985937196,
-      "loss": 2.6834,
-      "step": 132
-    },
-    {
-      "epoch": 1.63,
-      "learning_rate": 0.0004447117639345052,
-      "loss": 2.7107,
-      "step": 134
-    },
-    {
-      "epoch": 1.65,
-      "learning_rate": 0.0004317763060203664,
-      "loss": 2.7554,
-      "step": 136
-    },
-    {
-      "epoch": 1.68,
-      "learning_rate": 0.0004188872180193723,
-      "loss": 2.7709,
-      "step": 138
-    },
-    {
-      "epoch": 1.7,
-      "learning_rate": 0.00040605326031748645,
-      "loss": 2.7199,
-      "step": 140
-    },
-    {
-      "epoch": 1.73,
-      "learning_rate": 0.0003932831558300074,
-      "loss": 2.702,
-      "step": 142
-    },
-    {
-      "epoch": 1.75,
-      "learning_rate": 0.0003805855840728246,
-      "loss": 2.7447,
-      "step": 144
-    },
-    {
-      "epoch": 1.77,
-      "learning_rate": 0.00036796917526317154,
-      "loss": 2.7329,
-      "step": 146
-    },
-    {
-      "epoch": 1.8,
-      "learning_rate": 0.0003554425044538868,
-      "loss": 2.7188,
-      "step": 148
-    },
-    {
-      "epoch": 1.82,
-      "learning_rate": 0.00034301408570516746,
-      "loss": 2.7646,
-      "step": 150
     }
   ],
-  "max_steps": 246,
-  "num_train_epochs": 3,
-  "total_flos": 1.4229744023408476e+18,
+  "max_steps": 164,
+  "num_train_epochs": 2,
+  "total_flos": 1.1869674526642012e+18,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69a6570fe353bddee22ecfde7f0ce22259d3ebe6bcb4e301b37e008be4d9697d
+oid sha256:77577491aa076140acff879d10d90d737ea2b8587725db272bc6a06fcd09d690
 size 4527