Upload folder using huggingface_hub

README.md

@@ -0,0 +1,56 @@
+---
+license: other
+library_name: peft
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+base_model: gabrielmbmb/Upcycled-Qwen1.5-MoE2.7B
+model-index:
+- name: first
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# first
+
+This model is a fine-tuned version of [gabrielmbmb/Upcycled-Qwen1.5-MoE2.7B](https://huggingface.co/gabrielmbmb/Upcycled-Qwen1.5-MoE2.7B) on the wiki_demo dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 3.0
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.10.0
+- Transformers 4.40.0.dev0
+- Pytorch 2.2.2+cu121
+- Datasets 2.18.0
+- Tokenizers 0.15.2

adapter_config.json

@@ -0,0 +1,36 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "gabrielmbmb/Upcycled-Qwen1.5-MoE2.7B",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "k_proj",
+    "gate",
+    "shared_expert_gate",
+    "down_proj",
+    "q_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d4b53525787211fd3e8d6586fa4ef8315bfddb31d188e356cabd1f033e98bb6
+size 507798304

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

all_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 3.0,
+    "train_loss": 4.515984590848287,
+    "train_runtime": 5513.8168,
+    "train_samples_per_second": 0.696,
+    "train_steps_per_second": 0.087
+}

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '\\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 3.0,
+    "train_loss": 4.515984590848287,
+    "train_runtime": 5513.8168,
+    "train_samples_per_second": 0.696,
+    "train_steps_per_second": 0.087
+}

trainer_log.jsonl

@@ -0,0 +1,49 @@
+{"current_steps": 10, "total_steps": 480, "loss": 8.9503, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.994647308096509e-05, "epoch": 0.06, "percentage": 2.08, "elapsed_time": "0:01:45", "remaining_time": "1:22:28"}
+{"current_steps": 20, "total_steps": 480, "loss": 8.0851, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.9786121534345265e-05, "epoch": 0.12, "percentage": 4.17, "elapsed_time": "0:03:35", "remaining_time": "1:22:28"}
+{"current_steps": 30, "total_steps": 480, "loss": 7.5574, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.951963201008076e-05, "epoch": 0.19, "percentage": 6.25, "elapsed_time": "0:05:25", "remaining_time": "1:21:16"}
+{"current_steps": 40, "total_steps": 480, "loss": 6.9461, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.914814565722671e-05, "epoch": 0.25, "percentage": 8.33, "elapsed_time": "0:07:16", "remaining_time": "1:20:00"}
+{"current_steps": 50, "total_steps": 480, "loss": 6.1526, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.867325323737765e-05, "epoch": 0.31, "percentage": 10.42, "elapsed_time": "0:09:07", "remaining_time": "1:18:27"}
+{"current_steps": 60, "total_steps": 480, "loss": 5.4116, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.8096988312782174e-05, "epoch": 0.38, "percentage": 12.5, "elapsed_time": "0:10:55", "remaining_time": "1:16:25"}
+{"current_steps": 70, "total_steps": 480, "loss": 5.0721, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.742181853831721e-05, "epoch": 0.44, "percentage": 14.58, "elapsed_time": "0:12:44", "remaining_time": "1:14:36"}
+{"current_steps": 80, "total_steps": 480, "loss": 4.7695, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.665063509461097e-05, "epoch": 0.5, "percentage": 16.67, "elapsed_time": "0:14:35", "remaining_time": "1:12:58"}
+{"current_steps": 90, "total_steps": 480, "loss": 4.6179, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.5786740307563636e-05, "epoch": 0.56, "percentage": 18.75, "elapsed_time": "0:16:28", "remaining_time": "1:11:21"}
+{"current_steps": 100, "total_steps": 480, "loss": 4.4848, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.4833833507280884e-05, "epoch": 0.62, "percentage": 20.83, "elapsed_time": "0:18:21", "remaining_time": "1:09:43"}
+{"current_steps": 110, "total_steps": 480, "loss": 4.3627, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.379599518697444e-05, "epoch": 0.69, "percentage": 22.92, "elapsed_time": "0:20:14", "remaining_time": "1:08:05"}
+{"current_steps": 120, "total_steps": 480, "loss": 4.3319, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.267766952966369e-05, "epoch": 0.75, "percentage": 25.0, "elapsed_time": "0:22:08", "remaining_time": "1:06:26"}
+{"current_steps": 130, "total_steps": 480, "loss": 4.269, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.148364537750172e-05, "epoch": 0.81, "percentage": 27.08, "elapsed_time": "0:24:02", "remaining_time": "1:04:44"}
+{"current_steps": 140, "total_steps": 480, "loss": 4.2557, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.021903572521802e-05, "epoch": 0.88, "percentage": 29.17, "elapsed_time": "0:25:56", "remaining_time": "1:03:00"}
+{"current_steps": 150, "total_steps": 480, "loss": 4.1534, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.888925582549006e-05, "epoch": 0.94, "percentage": 31.25, "elapsed_time": "0:27:51", "remaining_time": "1:01:17"}
+{"current_steps": 160, "total_steps": 480, "loss": 4.2017, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.7500000000000003e-05, "epoch": 1.0, "percentage": 33.33, "elapsed_time": "0:29:45", "remaining_time": "0:59:30"}
+{"current_steps": 170, "total_steps": 480, "loss": 4.1732, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.6057217255475034e-05, "epoch": 1.06, "percentage": 35.42, "elapsed_time": "0:31:39", "remaining_time": "0:57:44"}
+{"current_steps": 180, "total_steps": 480, "loss": 4.1358, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.456708580912725e-05, "epoch": 1.12, "percentage": 37.5, "elapsed_time": "0:33:33", "remaining_time": "0:55:56"}
+{"current_steps": 190, "total_steps": 480, "loss": 4.1514, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.303598663257904e-05, "epoch": 1.19, "percentage": 39.58, "elapsed_time": "0:35:29", "remaining_time": "0:54:10"}
+{"current_steps": 200, "total_steps": 480, "loss": 4.0925, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.147047612756302e-05, "epoch": 1.25, "percentage": 41.67, "elapsed_time": "0:37:25", "remaining_time": "0:52:23"}
+{"current_steps": 210, "total_steps": 480, "loss": 4.0496, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.9877258050403212e-05, "epoch": 1.31, "percentage": 43.75, "elapsed_time": "0:39:20", "remaining_time": "0:50:35"}
+{"current_steps": 220, "total_steps": 480, "loss": 4.0853, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.8263154805501297e-05, "epoch": 1.38, "percentage": 45.83, "elapsed_time": "0:41:16", "remaining_time": "0:48:46"}
+{"current_steps": 230, "total_steps": 480, "loss": 4.1283, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.663507823075358e-05, "epoch": 1.44, "percentage": 47.92, "elapsed_time": "0:43:11", "remaining_time": "0:46:56"}
+{"current_steps": 240, "total_steps": 480, "loss": 4.0567, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.5e-05, "epoch": 1.5, "percentage": 50.0, "elapsed_time": "0:45:07", "remaining_time": "0:45:07"}
+{"current_steps": 250, "total_steps": 480, "loss": 4.0141, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.3364921769246423e-05, "epoch": 1.56, "percentage": 52.08, "elapsed_time": "0:47:03", "remaining_time": "0:43:17"}
+{"current_steps": 260, "total_steps": 480, "loss": 4.0356, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.173684519449872e-05, "epoch": 1.62, "percentage": 54.17, "elapsed_time": "0:48:59", "remaining_time": "0:41:27"}
+{"current_steps": 270, "total_steps": 480, "loss": 4.082, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.0122741949596797e-05, "epoch": 1.69, "percentage": 56.25, "elapsed_time": "0:50:56", "remaining_time": "0:39:36"}
+{"current_steps": 280, "total_steps": 480, "loss": 4.0767, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.852952387243698e-05, "epoch": 1.75, "percentage": 58.33, "elapsed_time": "0:52:52", "remaining_time": "0:37:46"}
+{"current_steps": 290, "total_steps": 480, "loss": 3.9817, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.6964013367420966e-05, "epoch": 1.81, "percentage": 60.42, "elapsed_time": "0:54:49", "remaining_time": "0:35:55"}
+{"current_steps": 300, "total_steps": 480, "loss": 4.0618, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.5432914190872757e-05, "epoch": 1.88, "percentage": 62.5, "elapsed_time": "0:56:46", "remaining_time": "0:34:03"}
+{"current_steps": 310, "total_steps": 480, "loss": 4.0106, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.3942782744524973e-05, "epoch": 1.94, "percentage": 64.58, "elapsed_time": "0:58:43", "remaining_time": "0:32:12"}
+{"current_steps": 320, "total_steps": 480, "loss": 4.0549, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.2500000000000006e-05, "epoch": 2.0, "percentage": 66.67, "elapsed_time": "1:00:39", "remaining_time": "0:30:19"}
+{"current_steps": 330, "total_steps": 480, "loss": 3.919, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.1110744174509952e-05, "epoch": 2.06, "percentage": 68.75, "elapsed_time": "1:02:36", "remaining_time": "0:28:27"}
+{"current_steps": 340, "total_steps": 480, "loss": 4.053, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.780964274781984e-06, "epoch": 2.12, "percentage": 70.83, "elapsed_time": "1:04:33", "remaining_time": "0:26:34"}
+{"current_steps": 350, "total_steps": 480, "loss": 4.0032, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.51635462249828e-06, "epoch": 2.19, "percentage": 72.92, "elapsed_time": "1:06:30", "remaining_time": "0:24:42"}
+{"current_steps": 360, "total_steps": 480, "loss": 3.991, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.3223304703363135e-06, "epoch": 2.25, "percentage": 75.0, "elapsed_time": "1:08:27", "remaining_time": "0:22:49"}
+{"current_steps": 370, "total_steps": 480, "loss": 3.9941, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.204004813025568e-06, "epoch": 2.31, "percentage": 77.08, "elapsed_time": "1:10:25", "remaining_time": "0:20:56"}
+{"current_steps": 380, "total_steps": 480, "loss": 4.0388, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.166166492719124e-06, "epoch": 2.38, "percentage": 79.17, "elapsed_time": "1:12:22", "remaining_time": "0:19:02"}
+{"current_steps": 390, "total_steps": 480, "loss": 4.0767, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.213259692436367e-06, "epoch": 2.44, "percentage": 81.25, "elapsed_time": "1:14:20", "remaining_time": "0:17:09"}
+{"current_steps": 400, "total_steps": 480, "loss": 3.9709, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.3493649053890326e-06, "epoch": 2.5, "percentage": 83.33, "elapsed_time": "1:16:16", "remaining_time": "0:15:15"}
+{"current_steps": 410, "total_steps": 480, "loss": 3.9714, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.578181461682794e-06, "epoch": 2.56, "percentage": 85.42, "elapsed_time": "1:18:12", "remaining_time": "0:13:21"}
+{"current_steps": 420, "total_steps": 480, "loss": 4.0366, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.9030116872178316e-06, "epoch": 2.62, "percentage": 87.5, "elapsed_time": "1:20:10", "remaining_time": "0:11:27"}
+{"current_steps": 430, "total_steps": 480, "loss": 3.9307, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.3267467626223606e-06, "epoch": 2.69, "percentage": 89.58, "elapsed_time": "1:22:07", "remaining_time": "0:09:33"}
+{"current_steps": 440, "total_steps": 480, "loss": 4.0751, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.51854342773295e-07, "epoch": 2.75, "percentage": 91.67, "elapsed_time": "1:24:05", "remaining_time": "0:07:38"}
+{"current_steps": 450, "total_steps": 480, "loss": 4.0141, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.803679899192392e-07, "epoch": 2.81, "percentage": 93.75, "elapsed_time": "1:26:02", "remaining_time": "0:05:44"}
+{"current_steps": 460, "total_steps": 480, "loss": 3.9676, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.1387846565474045e-07, "epoch": 2.88, "percentage": 95.83, "elapsed_time": "1:27:59", "remaining_time": "0:03:49"}
+{"current_steps": 470, "total_steps": 480, "loss": 4.0041, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.352691903491303e-08, "epoch": 2.94, "percentage": 97.92, "elapsed_time": "1:29:56", "remaining_time": "0:01:54"}
+{"current_steps": 480, "total_steps": 480, "loss": 3.909, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "1:31:52", "remaining_time": "0:00:00"}
+{"current_steps": 480, "total_steps": 480, "loss": null, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": null, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "1:31:52", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,366 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 480,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06,
+      "grad_norm": 4.709561347961426,
+      "learning_rate": 4.994647308096509e-05,
+      "loss": 8.9503,
+      "step": 10
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 2.5953965187072754,
+      "learning_rate": 4.9786121534345265e-05,
+      "loss": 8.0851,
+      "step": 20
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 2.367886543273926,
+      "learning_rate": 4.951963201008076e-05,
+      "loss": 7.5574,
+      "step": 30
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 3.426809549331665,
+      "learning_rate": 4.914814565722671e-05,
+      "loss": 6.9461,
+      "step": 40
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 4.523465633392334,
+      "learning_rate": 4.867325323737765e-05,
+      "loss": 6.1526,
+      "step": 50
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 1.7781277894973755,
+      "learning_rate": 4.8096988312782174e-05,
+      "loss": 5.4116,
+      "step": 60
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 1.3401896953582764,
+      "learning_rate": 4.742181853831721e-05,
+      "loss": 5.0721,
+      "step": 70
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.9671522378921509,
+      "learning_rate": 4.665063509461097e-05,
+      "loss": 4.7695,
+      "step": 80
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.055595874786377,
+      "learning_rate": 4.5786740307563636e-05,
+      "loss": 4.6179,
+      "step": 90
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 0.7612900733947754,
+      "learning_rate": 4.4833833507280884e-05,
+      "loss": 4.4848,
+      "step": 100
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 0.7046266198158264,
+      "learning_rate": 4.379599518697444e-05,
+      "loss": 4.3627,
+      "step": 110
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.9548586010932922,
+      "learning_rate": 4.267766952966369e-05,
+      "loss": 4.3319,
+      "step": 120
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 0.6869709491729736,
+      "learning_rate": 4.148364537750172e-05,
+      "loss": 4.269,
+      "step": 130
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.6282512545585632,
+      "learning_rate": 4.021903572521802e-05,
+      "loss": 4.2557,
+      "step": 140
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 0.5287710428237915,
+      "learning_rate": 3.888925582549006e-05,
+      "loss": 4.1534,
+      "step": 150
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.7542024850845337,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 4.2017,
+      "step": 160
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 0.49876415729522705,
+      "learning_rate": 3.6057217255475034e-05,
+      "loss": 4.1732,
+      "step": 170
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 0.5626935958862305,
+      "learning_rate": 3.456708580912725e-05,
+      "loss": 4.1358,
+      "step": 180
+    },
+    {
+      "epoch": 1.19,
+      "grad_norm": 0.493310809135437,
+      "learning_rate": 3.303598663257904e-05,
+      "loss": 4.1514,
+      "step": 190
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 0.4654150605201721,
+      "learning_rate": 3.147047612756302e-05,
+      "loss": 4.0925,
+      "step": 200
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 0.592623770236969,
+      "learning_rate": 2.9877258050403212e-05,
+      "loss": 4.0496,
+      "step": 210
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 0.5564578771591187,
+      "learning_rate": 2.8263154805501297e-05,
+      "loss": 4.0853,
+      "step": 220
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 0.6952773332595825,
+      "learning_rate": 2.663507823075358e-05,
+      "loss": 4.1283,
+      "step": 230
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 0.5385617613792419,
+      "learning_rate": 2.5e-05,
+      "loss": 4.0567,
+      "step": 240
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 0.5663427114486694,
+      "learning_rate": 2.3364921769246423e-05,
+      "loss": 4.0141,
+      "step": 250
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 0.5520788431167603,
+      "learning_rate": 2.173684519449872e-05,
+      "loss": 4.0356,
+      "step": 260
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 0.5162128806114197,
+      "learning_rate": 2.0122741949596797e-05,
+      "loss": 4.082,
+      "step": 270
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 0.5291630625724792,
+      "learning_rate": 1.852952387243698e-05,
+      "loss": 4.0767,
+      "step": 280
+    },
+    {
+      "epoch": 1.81,
+      "grad_norm": 0.6226648092269897,
+      "learning_rate": 1.6964013367420966e-05,
+      "loss": 3.9817,
+      "step": 290
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 0.5460664629936218,
+      "learning_rate": 1.5432914190872757e-05,
+      "loss": 4.0618,
+      "step": 300
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 0.7545162439346313,
+      "learning_rate": 1.3942782744524973e-05,
+      "loss": 4.0106,
+      "step": 310
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.6207989454269409,
+      "learning_rate": 1.2500000000000006e-05,
+      "loss": 4.0549,
+      "step": 320
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 0.5338532328605652,
+      "learning_rate": 1.1110744174509952e-05,
+      "loss": 3.919,
+      "step": 330
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 0.5484297275543213,
+      "learning_rate": 9.780964274781984e-06,
+      "loss": 4.053,
+      "step": 340
+    },
+    {
+      "epoch": 2.19,
+      "grad_norm": 0.6356564164161682,
+      "learning_rate": 8.51635462249828e-06,
+      "loss": 4.0032,
+      "step": 350
+    },
+    {
+      "epoch": 2.25,
+      "grad_norm": 0.5518457889556885,
+      "learning_rate": 7.3223304703363135e-06,
+      "loss": 3.991,
+      "step": 360
+    },
+    {
+      "epoch": 2.31,
+      "grad_norm": 0.5176472067832947,
+      "learning_rate": 6.204004813025568e-06,
+      "loss": 3.9941,
+      "step": 370
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 0.5543831586837769,
+      "learning_rate": 5.166166492719124e-06,
+      "loss": 4.0388,
+      "step": 380
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 0.5504453182220459,
+      "learning_rate": 4.213259692436367e-06,
+      "loss": 4.0767,
+      "step": 390
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 0.5619158148765564,
+      "learning_rate": 3.3493649053890326e-06,
+      "loss": 3.9709,
+      "step": 400
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 0.5513697266578674,
+      "learning_rate": 2.578181461682794e-06,
+      "loss": 3.9714,
+      "step": 410
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 0.590857744216919,
+      "learning_rate": 1.9030116872178316e-06,
+      "loss": 4.0366,
+      "step": 420
+    },
+    {
+      "epoch": 2.69,
+      "grad_norm": 0.5728959441184998,
+      "learning_rate": 1.3267467626223606e-06,
+      "loss": 3.9307,
+      "step": 430
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 0.5163474082946777,
+      "learning_rate": 8.51854342773295e-07,
+      "loss": 4.0751,
+      "step": 440
+    },
+    {
+      "epoch": 2.81,
+      "grad_norm": 0.5247732996940613,
+      "learning_rate": 4.803679899192392e-07,
+      "loss": 4.0141,
+      "step": 450
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 0.5492649674415588,
+      "learning_rate": 2.1387846565474045e-07,
+      "loss": 3.9676,
+      "step": 460
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 0.5566267371177673,
+      "learning_rate": 5.352691903491303e-08,
+      "loss": 4.0041,
+      "step": 470
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.6199953556060791,
+      "learning_rate": 0.0,
+      "loss": 3.909,
+      "step": 480
+    },
+    {
+      "epoch": 3.0,
+      "step": 480,
+      "total_flos": 3.260181978788659e+17,
+      "train_loss": 4.515984590848287,
+      "train_runtime": 5513.8168,
+      "train_samples_per_second": 0.696,
+      "train_steps_per_second": 0.087
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 480,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000,
+  "total_flos": 3.260181978788659e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ee90bb77243c24436e2e183b8a39237b082d3ff29adac639257c4ee220a498b
+size 5112