Upload folder using huggingface_hub

README.md

@@ -12,34 +12,47 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# Model Card: LLama 2 - Version 7b (Embedding + Output + 1 Hidden Layer)
+# core2
 
-## Overview
+This model is a fine-tuned version of [./core2](https://huggingface.co/./core2) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.7997
+- Accuracy: 0.4028
 
-- **Link to Training Progress:** [WandB Training Progress](https://wandb.ai/inteligenciaartificialcursos/huggingface/runs/qpsun6iq?workspace=user-inteligenciaartificialcursos)
+## Model description
 
-- **Model Name:** LLama 2 - Version 7b
+More information needed
 
-- **Total Parameters:** 446 million
+## Intended uses & limitations
 
-## Training Data
+More information needed
 
-The model has been trained with the following sequence of datasets:
+## Training and evaluation data
 
-1. **GPT-2 Data (Done):** The initial training phase involves GPT-2 data and is currently in the finalization stage.
+More information needed
 
-2. **Wikipedia QA in Markdown (In Progress):** The model's training will continue with Wikipedia question-answering data in Markdown format.
+## Training procedure
 
-3. **QA with Rhetoric (Future Stages):** The model will further be fine-tuned with question-answering data generated from various LLama models, incorporating rhetorical elements.
+### Training hyperparameters
 
-## Model Description
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 1
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 64
+- total_train_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 1.0
 
-The LLama 2 - Version 7b model is a powerful language model with a total of 446 million parameters. It utilizes embeddings, an output layer, and one hidden layer to perform a wide range of natural language processing tasks. The training is conducted in multiple stages, each focused on different datasets and objectives.
+### Training results
 
-## Disclaimer
 
-This model card provides an overview of the LLama 2 - Version 7b model, its training data, and intended use cases. Keep in mind that the model's performance may vary depending on the specific task or dataset. Users are encouraged to evaluate the model's suitability for their applications and exercise caution when using it in real-world scenarios.
 
-For any further inquiries or issues related to this model, please contact the model developers through the provided training progress link.
+### Framework versions
 
----
+- Transformers 4.34.0.dev0
+- Pytorch 2.0.1+cu117
+- Datasets 2.14.5
+- Tokenizers 0.13.3

all_results.json

@@ -1,15 +1,15 @@
 {
     "epoch": 1.0,
-    "eval_accuracy": 0.4077155652549501,
-    "eval_loss": 2.7608211040496826,
-    "eval_runtime": 1.7213,
+    "eval_accuracy": 0.40275220320231575,
+    "eval_loss": 2.7997472286224365,
+    "eval_runtime": 1.756,
     "eval_samples": 129,
-    "eval_samples_per_second": 74.942,
-    "eval_steps_per_second": 9.876,
-    "perplexity": 15.81282159096841,
-    "train_loss": 2.7717256223521947,
-    "train_runtime": 3102.8464,
-    "train_samples": 119233,
-    "train_samples_per_second": 38.427,
-    "train_steps_per_second": 1.201
+    "eval_samples_per_second": 73.461,
+    "eval_steps_per_second": 9.681,
+    "perplexity": 16.44049056038686,
+    "train_loss": 2.7764307147395,
+    "train_runtime": 11751.6473,
+    "train_samples": 455985,
+    "train_samples_per_second": 38.802,
+    "train_steps_per_second": 0.606
 }

eval_results.json

@@ -1,10 +1,10 @@
 {
     "epoch": 1.0,
-    "eval_accuracy": 0.4077155652549501,
-    "eval_loss": 2.7608211040496826,
-    "eval_runtime": 1.7213,
+    "eval_accuracy": 0.40275220320231575,
+    "eval_loss": 2.7997472286224365,
+    "eval_runtime": 1.756,
     "eval_samples": 129,
-    "eval_samples_per_second": 74.942,
-    "eval_steps_per_second": 9.876,
-    "perplexity": 15.81282159096841
+    "eval_samples_per_second": 73.461,
+    "eval_steps_per_second": 9.681,
+    "perplexity": 16.44049056038686
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:451f74d625df52f59037a15db1a90de07168091802624afed9235768562eba88
+oid sha256:e80035c32e620d674f75f7fbd6b7c57764e37e87d5777f59a2be270c24a7d943
 size 929067029

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 2.7717256223521947,
-    "train_runtime": 3102.8464,
-    "train_samples": 119233,
-    "train_samples_per_second": 38.427,
-    "train_steps_per_second": 1.201
+    "train_loss": 2.7764307147395,
+    "train_runtime": 11751.6473,
+    "train_samples": 455985,
+    "train_samples_per_second": 38.802,
+    "train_steps_per_second": 0.606
 }

trainer_state.json

@@ -1,472 +1,880 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9999916130601427,
+  "epoch": 0.9998925403247914,
   "eval_steps": 500,
-  "global_step": 3726,
+  "global_step": 7124,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.01,
-      "learning_rate": 9.86580783682233e-05,
-      "loss": 3.6768,
+      "learning_rate": 9.92981471083661e-05,
+      "loss": 2.9958,
       "step": 50
     },
     {
-      "epoch": 0.03,
-      "learning_rate": 9.731615673644659e-05,
-      "loss": 3.2865,
+      "epoch": 0.01,
+      "learning_rate": 9.859629421673217e-05,
+      "loss": 2.9669,
       "step": 100
     },
     {
-      "epoch": 0.04,
-      "learning_rate": 9.597423510466989e-05,
-      "loss": 3.1518,
+      "epoch": 0.02,
+      "learning_rate": 9.789444132509826e-05,
+      "loss": 2.958,
       "step": 150
     },
     {
-      "epoch": 0.05,
-      "learning_rate": 9.463231347289319e-05,
-      "loss": 3.1091,
+      "epoch": 0.03,
+      "learning_rate": 9.719258843346435e-05,
+      "loss": 2.9459,
       "step": 200
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 9.329039184111649e-05,
-      "loss": 3.0456,
+      "epoch": 0.04,
+      "learning_rate": 9.649073554183044e-05,
+      "loss": 2.9461,
       "step": 250
     },
     {
-      "epoch": 0.08,
-      "learning_rate": 9.194847020933978e-05,
-      "loss": 3.0357,
+      "epoch": 0.04,
+      "learning_rate": 9.578888265019651e-05,
+      "loss": 2.9109,
       "step": 300
     },
     {
-      "epoch": 0.09,
-      "learning_rate": 9.060654857756307e-05,
-      "loss": 2.9846,
+      "epoch": 0.05,
+      "learning_rate": 9.50870297585626e-05,
+      "loss": 2.9316,
       "step": 350
     },
     {
-      "epoch": 0.11,
-      "learning_rate": 8.926462694578636e-05,
-      "loss": 2.9834,
+      "epoch": 0.06,
+      "learning_rate": 9.43851768669287e-05,
+      "loss": 2.8693,
       "step": 400
     },
     {
-      "epoch": 0.12,
-      "learning_rate": 8.792270531400967e-05,
-      "loss": 2.9385,
+      "epoch": 0.06,
+      "learning_rate": 9.368332397529478e-05,
+      "loss": 2.8988,
       "step": 450
     },
     {
-      "epoch": 0.13,
-      "learning_rate": 8.658078368223296e-05,
-      "loss": 2.8967,
+      "epoch": 0.07,
+      "learning_rate": 9.298147108366086e-05,
+      "loss": 2.8858,
       "step": 500
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 8.523886205045626e-05,
-      "loss": 2.886,
+      "epoch": 0.08,
+      "learning_rate": 9.227961819202695e-05,
+      "loss": 2.8717,
       "step": 550
     },
     {
-      "epoch": 0.16,
-      "learning_rate": 8.389694041867955e-05,
-      "loss": 2.8671,
+      "epoch": 0.08,
+      "learning_rate": 9.157776530039304e-05,
+      "loss": 2.8852,
       "step": 600
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 8.255501878690284e-05,
-      "loss": 2.8547,
+      "epoch": 0.09,
+      "learning_rate": 9.087591240875913e-05,
+      "loss": 2.8702,
       "step": 650
     },
     {
-      "epoch": 0.19,
-      "learning_rate": 8.121309715512614e-05,
-      "loss": 2.8396,
+      "epoch": 0.1,
+      "learning_rate": 9.017405951712522e-05,
+      "loss": 2.8506,
       "step": 700
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 7.987117552334944e-05,
-      "loss": 2.8531,
+      "epoch": 0.11,
+      "learning_rate": 8.947220662549131e-05,
+      "loss": 2.8793,
       "step": 750
     },
     {
-      "epoch": 0.21,
-      "learning_rate": 7.852925389157274e-05,
-      "loss": 2.8196,
+      "epoch": 0.11,
+      "learning_rate": 8.87703537338574e-05,
+      "loss": 2.857,
       "step": 800
     },
     {
-      "epoch": 0.23,
-      "learning_rate": 7.718733225979603e-05,
-      "loss": 2.7921,
+      "epoch": 0.12,
+      "learning_rate": 8.806850084222347e-05,
+      "loss": 2.8538,
       "step": 850
     },
     {
-      "epoch": 0.24,
-      "learning_rate": 7.584541062801933e-05,
-      "loss": 2.7596,
+      "epoch": 0.13,
+      "learning_rate": 8.736664795058956e-05,
+      "loss": 2.8592,
       "step": 900
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 7.450348899624262e-05,
-      "loss": 2.7918,
+      "epoch": 0.13,
+      "learning_rate": 8.666479505895565e-05,
+      "loss": 2.859,
       "step": 950
     },
     {
-      "epoch": 0.27,
-      "learning_rate": 7.316156736446593e-05,
-      "loss": 2.7553,
+      "epoch": 0.14,
+      "learning_rate": 8.596294216732174e-05,
+      "loss": 2.8183,
       "step": 1000
     },
     {
-      "epoch": 0.28,
-      "learning_rate": 7.181964573268921e-05,
-      "loss": 2.7914,
+      "epoch": 0.15,
+      "learning_rate": 8.526108927568782e-05,
+      "loss": 2.8456,
       "step": 1050
     },
     {
-      "epoch": 0.3,
-      "learning_rate": 7.047772410091251e-05,
-      "loss": 2.7924,
+      "epoch": 0.15,
+      "learning_rate": 8.455923638405391e-05,
+      "loss": 2.803,
       "step": 1100
     },
     {
-      "epoch": 0.31,
-      "learning_rate": 6.91358024691358e-05,
-      "loss": 2.7823,
+      "epoch": 0.16,
+      "learning_rate": 8.385738349242e-05,
+      "loss": 2.8414,
       "step": 1150
     },
     {
-      "epoch": 0.32,
-      "learning_rate": 6.77938808373591e-05,
-      "loss": 2.7437,
+      "epoch": 0.17,
+      "learning_rate": 8.315553060078609e-05,
+      "loss": 2.8162,
       "step": 1200
     },
     {
-      "epoch": 0.34,
-      "learning_rate": 6.64519592055824e-05,
-      "loss": 2.7404,
+      "epoch": 0.18,
+      "learning_rate": 8.245367770915216e-05,
+      "loss": 2.8404,
       "step": 1250
     },
     {
-      "epoch": 0.35,
-      "learning_rate": 6.51100375738057e-05,
-      "loss": 2.7318,
+      "epoch": 0.18,
+      "learning_rate": 8.175182481751825e-05,
+      "loss": 2.801,
       "step": 1300
     },
     {
-      "epoch": 0.36,
-      "learning_rate": 6.376811594202898e-05,
-      "loss": 2.708,
+      "epoch": 0.19,
+      "learning_rate": 8.104997192588434e-05,
+      "loss": 2.818,
       "step": 1350
     },
     {
-      "epoch": 0.38,
-      "learning_rate": 6.242619431025228e-05,
-      "loss": 2.7579,
+      "epoch": 0.2,
+      "learning_rate": 8.034811903425043e-05,
+      "loss": 2.8279,
       "step": 1400
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 6.108427267847558e-05,
-      "loss": 2.7037,
+      "epoch": 0.2,
+      "learning_rate": 7.96462661426165e-05,
+      "loss": 2.8261,
       "step": 1450
     },
     {
-      "epoch": 0.4,
-      "learning_rate": 5.9742351046698876e-05,
-      "loss": 2.7326,
+      "epoch": 0.21,
+      "learning_rate": 7.89444132509826e-05,
+      "loss": 2.7759,
       "step": 1500
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 5.8400429414922176e-05,
-      "loss": 2.7252,
+      "epoch": 0.22,
+      "learning_rate": 7.824256035934869e-05,
+      "loss": 2.7991,
       "step": 1550
     },
     {
-      "epoch": 0.43,
-      "learning_rate": 5.705850778314546e-05,
-      "loss": 2.7263,
+      "epoch": 0.22,
+      "learning_rate": 7.754070746771476e-05,
+      "loss": 2.7823,
       "step": 1600
     },
     {
-      "epoch": 0.44,
-      "learning_rate": 5.571658615136877e-05,
-      "loss": 2.6944,
+      "epoch": 0.23,
+      "learning_rate": 7.683885457608085e-05,
+      "loss": 2.7843,
       "step": 1650
     },
     {
-      "epoch": 0.46,
-      "learning_rate": 5.4374664519592054e-05,
-      "loss": 2.7292,
+      "epoch": 0.24,
+      "learning_rate": 7.613700168444694e-05,
+      "loss": 2.7736,
       "step": 1700
     },
     {
-      "epoch": 0.47,
-      "learning_rate": 5.3032742887815354e-05,
-      "loss": 2.7156,
+      "epoch": 0.25,
+      "learning_rate": 7.543514879281303e-05,
+      "loss": 2.8135,
       "step": 1750
     },
     {
-      "epoch": 0.48,
-      "learning_rate": 5.1690821256038647e-05,
-      "loss": 2.6852,
+      "epoch": 0.25,
+      "learning_rate": 7.47332959011791e-05,
+      "loss": 2.7873,
       "step": 1800
     },
     {
-      "epoch": 0.5,
-      "learning_rate": 5.0348899624261946e-05,
-      "loss": 2.6922,
+      "epoch": 0.26,
+      "learning_rate": 7.40314430095452e-05,
+      "loss": 2.777,
       "step": 1850
     },
     {
-      "epoch": 0.51,
-      "learning_rate": 4.9006977992485246e-05,
-      "loss": 2.7217,
+      "epoch": 0.27,
+      "learning_rate": 7.332959011791129e-05,
+      "loss": 2.7542,
       "step": 1900
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 4.766505636070854e-05,
-      "loss": 2.6923,
+      "epoch": 0.27,
+      "learning_rate": 7.262773722627737e-05,
+      "loss": 2.7759,
       "step": 1950
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 4.632313472893183e-05,
-      "loss": 2.6993,
+      "epoch": 0.28,
+      "learning_rate": 7.192588433464346e-05,
+      "loss": 2.7772,
       "step": 2000
     },
     {
-      "epoch": 0.55,
-      "learning_rate": 4.498121309715513e-05,
-      "loss": 2.7156,
+      "epoch": 0.29,
+      "learning_rate": 7.122403144300955e-05,
+      "loss": 2.7689,
       "step": 2050
     },
     {
-      "epoch": 0.56,
-      "learning_rate": 4.3639291465378424e-05,
-      "loss": 2.6932,
+      "epoch": 0.29,
+      "learning_rate": 7.052217855137564e-05,
+      "loss": 2.8015,
       "step": 2100
     },
     {
-      "epoch": 0.58,
-      "learning_rate": 4.2297369833601716e-05,
-      "loss": 2.714,
+      "epoch": 0.3,
+      "learning_rate": 6.982032565974172e-05,
+      "loss": 2.7655,
       "step": 2150
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 4.0955448201825016e-05,
-      "loss": 2.6921,
+      "epoch": 0.31,
+      "learning_rate": 6.911847276810781e-05,
+      "loss": 2.7728,
       "step": 2200
     },
     {
-      "epoch": 0.6,
-      "learning_rate": 3.961352657004831e-05,
-      "loss": 2.6949,
+      "epoch": 0.32,
+      "learning_rate": 6.84166198764739e-05,
+      "loss": 2.7686,
       "step": 2250
     },
     {
-      "epoch": 0.62,
-      "learning_rate": 3.82716049382716e-05,
-      "loss": 2.694,
+      "epoch": 0.32,
+      "learning_rate": 6.771476698483999e-05,
+      "loss": 2.7651,
       "step": 2300
     },
     {
-      "epoch": 0.63,
-      "learning_rate": 3.69296833064949e-05,
-      "loss": 2.6781,
+      "epoch": 0.33,
+      "learning_rate": 6.701291409320606e-05,
+      "loss": 2.7594,
       "step": 2350
     },
     {
-      "epoch": 0.64,
-      "learning_rate": 3.55877616747182e-05,
-      "loss": 2.6932,
+      "epoch": 0.34,
+      "learning_rate": 6.631106120157215e-05,
+      "loss": 2.8137,
       "step": 2400
     },
     {
-      "epoch": 0.66,
-      "learning_rate": 3.4245840042941493e-05,
-      "loss": 2.6748,
+      "epoch": 0.34,
+      "learning_rate": 6.560920830993824e-05,
+      "loss": 2.7566,
       "step": 2450
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 3.290391841116479e-05,
-      "loss": 2.6861,
+      "epoch": 0.35,
+      "learning_rate": 6.490735541830433e-05,
+      "loss": 2.7705,
       "step": 2500
     },
     {
-      "epoch": 0.68,
-      "learning_rate": 3.1561996779388086e-05,
-      "loss": 2.6891,
+      "epoch": 0.36,
+      "learning_rate": 6.420550252667041e-05,
+      "loss": 2.7873,
       "step": 2550
     },
     {
-      "epoch": 0.7,
-      "learning_rate": 3.0220075147611382e-05,
-      "loss": 2.7091,
+      "epoch": 0.36,
+      "learning_rate": 6.35036496350365e-05,
+      "loss": 2.7841,
       "step": 2600
     },
     {
-      "epoch": 0.71,
-      "learning_rate": 2.8878153515834678e-05,
-      "loss": 2.6993,
+      "epoch": 0.37,
+      "learning_rate": 6.280179674340259e-05,
+      "loss": 2.7643,
       "step": 2650
     },
     {
-      "epoch": 0.72,
-      "learning_rate": 2.753623188405797e-05,
-      "loss": 2.6827,
+      "epoch": 0.38,
+      "learning_rate": 6.209994385176868e-05,
+      "loss": 2.7838,
       "step": 2700
     },
     {
-      "epoch": 0.74,
-      "learning_rate": 2.6194310252281267e-05,
-      "loss": 2.6876,
+      "epoch": 0.39,
+      "learning_rate": 6.139809096013475e-05,
+      "loss": 2.7849,
       "step": 2750
     },
     {
-      "epoch": 0.75,
-      "learning_rate": 2.4852388620504563e-05,
-      "loss": 2.6812,
+      "epoch": 0.39,
+      "learning_rate": 6.069623806850084e-05,
+      "loss": 2.789,
       "step": 2800
     },
     {
-      "epoch": 0.76,
-      "learning_rate": 2.351046698872786e-05,
-      "loss": 2.6936,
+      "epoch": 0.4,
+      "learning_rate": 5.999438517686693e-05,
+      "loss": 2.7397,
       "step": 2850
     },
     {
-      "epoch": 0.78,
-      "learning_rate": 2.2168545356951156e-05,
-      "loss": 2.6579,
+      "epoch": 0.41,
+      "learning_rate": 5.9292532285233015e-05,
+      "loss": 2.7426,
       "step": 2900
     },
     {
-      "epoch": 0.79,
-      "learning_rate": 2.0826623725174452e-05,
-      "loss": 2.6786,
+      "epoch": 0.41,
+      "learning_rate": 5.8590679393599104e-05,
+      "loss": 2.7395,
       "step": 2950
     },
     {
-      "epoch": 0.81,
-      "learning_rate": 1.9484702093397748e-05,
-      "loss": 2.6974,
+      "epoch": 0.42,
+      "learning_rate": 5.788882650196519e-05,
+      "loss": 2.7462,
       "step": 3000
     },
     {
-      "epoch": 0.82,
-      "learning_rate": 1.814278046162104e-05,
-      "loss": 2.6818,
+      "epoch": 0.43,
+      "learning_rate": 5.718697361033128e-05,
+      "loss": 2.7756,
       "step": 3050
     },
     {
-      "epoch": 0.83,
-      "learning_rate": 1.6800858829844337e-05,
-      "loss": 2.6742,
+      "epoch": 0.44,
+      "learning_rate": 5.648512071869736e-05,
+      "loss": 2.7218,
       "step": 3100
     },
     {
-      "epoch": 0.85,
-      "learning_rate": 1.5458937198067633e-05,
-      "loss": 2.6748,
+      "epoch": 0.44,
+      "learning_rate": 5.578326782706345e-05,
+      "loss": 2.7712,
       "step": 3150
     },
     {
-      "epoch": 0.86,
-      "learning_rate": 1.4117015566290927e-05,
-      "loss": 2.6771,
+      "epoch": 0.45,
+      "learning_rate": 5.508141493542953e-05,
+      "loss": 2.768,
       "step": 3200
     },
     {
-      "epoch": 0.87,
-      "learning_rate": 1.2775093934514227e-05,
-      "loss": 2.6895,
+      "epoch": 0.46,
+      "learning_rate": 5.4379562043795615e-05,
+      "loss": 2.7623,
       "step": 3250
     },
     {
-      "epoch": 0.89,
-      "learning_rate": 1.143317230273752e-05,
-      "loss": 2.6663,
+      "epoch": 0.46,
+      "learning_rate": 5.367770915216172e-05,
+      "loss": 2.7365,
       "step": 3300
     },
     {
-      "epoch": 0.9,
-      "learning_rate": 1.0091250670960816e-05,
-      "loss": 2.6485,
+      "epoch": 0.47,
+      "learning_rate": 5.29758562605278e-05,
+      "loss": 2.7875,
       "step": 3350
     },
     {
-      "epoch": 0.91,
-      "learning_rate": 8.749329039184112e-06,
-      "loss": 2.6825,
+      "epoch": 0.48,
+      "learning_rate": 5.227400336889389e-05,
+      "loss": 2.6979,
       "step": 3400
     },
     {
-      "epoch": 0.93,
-      "learning_rate": 7.4074074074074075e-06,
-      "loss": 2.6832,
+      "epoch": 0.48,
+      "learning_rate": 5.157215047725997e-05,
+      "loss": 2.7315,
       "step": 3450
     },
     {
-      "epoch": 0.94,
-      "learning_rate": 6.065485775630704e-06,
-      "loss": 2.7134,
+      "epoch": 0.49,
+      "learning_rate": 5.087029758562606e-05,
+      "loss": 2.7189,
       "step": 3500
     },
     {
-      "epoch": 0.95,
-      "learning_rate": 4.723564143853999e-06,
-      "loss": 2.6631,
+      "epoch": 0.5,
+      "learning_rate": 5.0168444693992145e-05,
+      "loss": 2.7283,
       "step": 3550
     },
     {
-      "epoch": 0.97,
-      "learning_rate": 3.3816425120772947e-06,
-      "loss": 2.6862,
+      "epoch": 0.51,
+      "learning_rate": 4.946659180235823e-05,
+      "loss": 2.7685,
       "step": 3600
     },
     {
-      "epoch": 0.98,
-      "learning_rate": 2.0397208803005905e-06,
-      "loss": 2.6663,
+      "epoch": 0.51,
+      "learning_rate": 4.876473891072432e-05,
+      "loss": 2.7724,
       "step": 3650
     },
     {
-      "epoch": 0.99,
-      "learning_rate": 6.977992485238862e-07,
-      "loss": 2.6592,
+      "epoch": 0.52,
+      "learning_rate": 4.80628860190904e-05,
+      "loss": 2.773,
       "step": 3700
     },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.736103312745649e-05,
+      "loss": 2.7564,
+      "step": 3750
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.665918023582257e-05,
+      "loss": 2.769,
+      "step": 3800
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.595732734418866e-05,
+      "loss": 2.7544,
+      "step": 3850
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5255474452554745e-05,
+      "loss": 2.742,
+      "step": 3900
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.4553621560920834e-05,
+      "loss": 2.7558,
+      "step": 3950
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.385176866928692e-05,
+      "loss": 2.752,
+      "step": 4000
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3149915777653006e-05,
+      "loss": 2.7577,
+      "step": 4050
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.244806288601909e-05,
+      "loss": 2.7248,
+      "step": 4100
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.174620999438518e-05,
+      "loss": 2.7314,
+      "step": 4150
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.104435710275126e-05,
+      "loss": 2.7308,
+      "step": 4200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.034250421111735e-05,
+      "loss": 2.7252,
+      "step": 4250
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.964065131948344e-05,
+      "loss": 2.7444,
+      "step": 4300
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.893879842784952e-05,
+      "loss": 2.7529,
+      "step": 4350
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.823694553621561e-05,
+      "loss": 2.7312,
+      "step": 4400
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.7535092644581696e-05,
+      "loss": 2.7471,
+      "step": 4450
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6833239752947785e-05,
+      "loss": 2.7289,
+      "step": 4500
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.613138686131387e-05,
+      "loss": 2.7667,
+      "step": 4550
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.542953396967996e-05,
+      "loss": 2.7553,
+      "step": 4600
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.472768107804604e-05,
+      "loss": 2.7422,
+      "step": 4650
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.402582818641213e-05,
+      "loss": 2.7328,
+      "step": 4700
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.332397529477821e-05,
+      "loss": 2.7449,
+      "step": 4750
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.26221224031443e-05,
+      "loss": 2.7332,
+      "step": 4800
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.1920269511510385e-05,
+      "loss": 2.7529,
+      "step": 4850
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1218416619876474e-05,
+      "loss": 2.7493,
+      "step": 4900
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.0516563728242564e-05,
+      "loss": 2.7525,
+      "step": 4950
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.981471083660865e-05,
+      "loss": 2.7367,
+      "step": 5000
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9112857944974736e-05,
+      "loss": 2.726,
+      "step": 5050
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8411005053340822e-05,
+      "loss": 2.7327,
+      "step": 5100
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.770915216170691e-05,
+      "loss": 2.7545,
+      "step": 5150
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7007299270072995e-05,
+      "loss": 2.7682,
+      "step": 5200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.630544637843908e-05,
+      "loss": 2.7486,
+      "step": 5250
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5603593486805167e-05,
+      "loss": 2.7341,
+      "step": 5300
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4901740595171253e-05,
+      "loss": 2.7423,
+      "step": 5350
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.419988770353734e-05,
+      "loss": 2.743,
+      "step": 5400
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.349803481190343e-05,
+      "loss": 2.7319,
+      "step": 5450
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.279618192026951e-05,
+      "loss": 2.7555,
+      "step": 5500
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2094329028635598e-05,
+      "loss": 2.7349,
+      "step": 5550
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1392476137001684e-05,
+      "loss": 2.7447,
+      "step": 5600
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.069062324536777e-05,
+      "loss": 2.7325,
+      "step": 5650
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9988770353733856e-05,
+      "loss": 2.7334,
+      "step": 5700
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9286917462099946e-05,
+      "loss": 2.7493,
+      "step": 5750
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8585064570466032e-05,
+      "loss": 2.7298,
+      "step": 5800
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7883211678832118e-05,
+      "loss": 2.7208,
+      "step": 5850
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7181358787198204e-05,
+      "loss": 2.7383,
+      "step": 5900
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.647950589556429e-05,
+      "loss": 2.714,
+      "step": 5950
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5777653003930376e-05,
+      "loss": 2.7646,
+      "step": 6000
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5075800112296463e-05,
+      "loss": 2.7601,
+      "step": 6050
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.437394722066255e-05,
+      "loss": 2.7473,
+      "step": 6100
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3672094329028636e-05,
+      "loss": 2.7423,
+      "step": 6150
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2970241437394723e-05,
+      "loss": 2.7147,
+      "step": 6200
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2268388545760809e-05,
+      "loss": 2.7426,
+      "step": 6250
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1566535654126895e-05,
+      "loss": 2.7496,
+      "step": 6300
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0864682762492983e-05,
+      "loss": 2.7438,
+      "step": 6350
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0162829870859069e-05,
+      "loss": 2.7441,
+      "step": 6400
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.460976979225155e-06,
+      "loss": 2.7426,
+      "step": 6450
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.759124087591241e-06,
+      "loss": 2.7334,
+      "step": 6500
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.057271195957327e-06,
+      "loss": 2.7512,
+      "step": 6550
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.3554183043234135e-06,
+      "loss": 2.7391,
+      "step": 6600
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.653565412689501e-06,
+      "loss": 2.7737,
+      "step": 6650
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.9517125210555875e-06,
+      "loss": 2.7009,
+      "step": 6700
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.249859629421673e-06,
+      "loss": 2.7149,
+      "step": 6750
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.54800673778776e-06,
+      "loss": 2.7317,
+      "step": 6800
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.846153846153847e-06,
+      "loss": 2.7297,
+      "step": 6850
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.1443009545199325e-06,
+      "loss": 2.7356,
+      "step": 6900
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.442448062886019e-06,
+      "loss": 2.7414,
+      "step": 6950
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7405951712521057e-06,
+      "loss": 2.7362,
+      "step": 7000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0387422796181922e-06,
+      "loss": 2.7303,
+      "step": 7050
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.3688938798427854e-07,
+      "loss": 2.7418,
+      "step": 7100
+    },
     {
       "epoch": 1.0,
-      "step": 3726,
-      "total_flos": 2.4427952859827405e+17,
-      "train_loss": 2.7717256223521947,
-      "train_runtime": 3102.8464,
-      "train_samples_per_second": 38.427,
-      "train_steps_per_second": 1.201
+      "step": 7124,
+      "total_flos": 9.341102317413335e+17,
+      "train_loss": 2.7764307147395,
+      "train_runtime": 11751.6473,
+      "train_samples_per_second": 38.802,
+      "train_steps_per_second": 0.606
     }
   ],
   "logging_steps": 50,
-  "max_steps": 3726,
+  "max_steps": 7124,
   "num_train_epochs": 1,
-  "save_steps": 5000,
-  "total_flos": 2.4427952859827405e+17,
+  "save_steps": 2000,
+  "total_flos": 9.341102317413335e+17,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce83680fb3cb44202bb32f2eca1b7e1d9dd35259d3cb4ec2c75bd5a87a2385d3
+oid sha256:0be34ece13733d42cb106498be612b92d1ed5dd757900d449ecef81b9dfbb415
 size 4027