update to instruct model

config.json

@@ -1,12 +1,12 @@
 {
-  "_name_or_path": "meta-llama/meta-llama-3-8B",
+  "_name_or_path": "meta-llama/meta-llama-3-8B-Instruct",
   "architectures": [
     "LlamaForTokenClassification"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
   "bos_token_id": 128000,
-  "eos_token_id": 128001,
+  "eos_token_id": 128009,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "id2label": {
@@ -20,6 +20,7 @@
     "1": 1
   },
   "max_position_embeddings": 8192,
+  "mlp_bias": false,
   "model_type": "llama",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
@@ -30,7 +31,7 @@
   "rope_theta": 500000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.2",
+  "transformers_version": "4.41.1",
   "use_cache": true,
   "vocab_size": 128256
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2c144103072514542e327fa8080bd375cb300f2d453fba9ca3aea81d0d4cf33
+oid sha256:d8cf9c4d0dd972e1a2131bfe656235ee98221679711a3beef6d46dadf0f20b5c
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9eee5f23d94405d90b7e9ff88b9443fee42f8528a658f54214c2aba7530d80c
+oid sha256:8d4782b4a69ef03845159ce1a15e272aadaaf134dc138d68f616098e8531729c
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4b8fbc5e113f69768dd8de84661ea20af8a32b734a9976144b4236c447b40ccc
+oid sha256:3acdd690e65c24f42a24581b8467af98bd3ca357444580f8012aacd2bd607921
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7098f8a123b9401d6d5b4630c5c331204a96e41ff9a55b21d17d4c039890e23b
+oid sha256:40e0d6b37568815888f7ab4c75dedaeef555e609deb2ea86901df415d70c9acb
 size 117482084

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4f6d9c910463f9577dfa0729d98e43e952097e9029a4c1360b1284cc4aac6829
+oid sha256:9a5397c647c43191a2dd8a57c46eabad77ab78eb886fb2155efb565d5e66c304
 size 35676

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1dbe76c2dc7721068d20ba0cbbc11b229d5c11383a839baef1f79fce84d9b904
+oid sha256:c525ef7132f585c38b4bfc5ae4f0355dfcc2530a841b09dae07ee95e86cb2de2
 size 14244

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b398016557f45ae11f4aaf508c18378d0be1e99dec77c80ea0fff0298f82bd0
+oid sha256:8d33b0edd5b6ac8e7325d969d8a731d29cfad089e3aa53b250b771d90d30a917
 size 1064

special_tokens_map.json

@@ -7,11 +7,11 @@
     "single_word": false
   },
   "eos_token": {
-    "content": "<|end_of_text|>",
+    "content": "<|eot_id|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|end_of_text|>"
+  "pad_token": "<|eot_id|>"
 }

tokenizer_config.json

@@ -2050,14 +2050,15 @@
     }
   },
   "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|end_of_text|>",
+  "eos_token": "<|eot_id|>",
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<|end_of_text|>",
+  "pad_token": "<|eot_id|>",
   "padding": "longest",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }

trainer_state.json

@@ -2,133 +2,186 @@
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 10.0,
-  "eval_steps": 500,
-  "global_step": 3380,
+  "eval_steps": 50,
+  "global_step": 520,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.4792899408284024,
-      "grad_norm": 3.84375,
-      "learning_rate": 8.520710059171599e-05,
-      "loss": 0.6478,
-      "step": 500
+      "epoch": 0.9615384615384616,
+      "eval_accuracy": 0.8201973056115414,
+      "eval_f1": 0.012405237767057204,
+      "eval_loss": 0.3012058436870575,
+      "eval_precision": 0.015254237288135594,
+      "eval_recall": 0.010452961672473868,
+      "eval_runtime": 2.9167,
+      "eval_samples_per_second": 31.542,
+      "eval_steps_per_second": 4.114,
+      "step": 50
     },
     {
-      "epoch": 1.4792899408284024,
-      "eval_accuracy": 0.8455031356898518,
-      "eval_f1": 0.012843946055426568,
-      "eval_loss": 0.4867057204246521,
-      "eval_precision": 0.010016179983049541,
-      "eval_recall": 0.01789647577092511,
-      "eval_runtime": 95.5701,
-      "eval_samples_per_second": 6.278,
-      "eval_steps_per_second": 0.785,
-      "step": 500
+      "epoch": 1.9230769230769231,
+      "grad_norm": 1.078125,
+      "learning_rate": 8.076923076923078e-05,
+      "loss": 0.4442,
+      "step": 100
+    },
+    {
+      "epoch": 1.9230769230769231,
+      "eval_accuracy": 0.8290018033308582,
+      "eval_f1": 0.01993355481727575,
+      "eval_loss": 0.27755603194236755,
+      "eval_precision": 0.03498542274052478,
+      "eval_recall": 0.013937282229965157,
+      "eval_runtime": 2.9435,
+      "eval_samples_per_second": 31.255,
+      "eval_steps_per_second": 4.077,
+      "step": 100
+    },
+    {
+      "epoch": 2.8846153846153846,
+      "eval_accuracy": 0.8282592553304339,
+      "eval_f1": 0.020833333333333336,
+      "eval_loss": 0.25815218687057495,
+      "eval_precision": 0.03359173126614987,
+      "eval_recall": 0.015098722415795587,
+      "eval_runtime": 2.941,
+      "eval_samples_per_second": 31.282,
+      "eval_steps_per_second": 4.08,
+      "step": 150
     },
     {
-      "epoch": 2.9585798816568047,
-      "grad_norm": 4.09375,
-      "learning_rate": 7.041420118343195e-05,
-      "loss": 0.5008,
-      "step": 1000
+      "epoch": 3.8461538461538463,
+      "grad_norm": 1.171875,
+      "learning_rate": 6.153846153846155e-05,
+      "loss": 0.2901,
+      "step": 200
     },
     {
-      "epoch": 2.9585798816568047,
-      "eval_accuracy": 0.8540407639680729,
-      "eval_f1": 0.016587026967649805,
-      "eval_loss": 0.4580599069595337,
-      "eval_precision": 0.013798775472905053,
-      "eval_recall": 0.020787444933920703,
-      "eval_runtime": 95.5633,
-      "eval_samples_per_second": 6.279,
-      "eval_steps_per_second": 0.785,
-      "step": 1000
+      "epoch": 3.8461538461538463,
+      "eval_accuracy": 0.8308051341890315,
+      "eval_f1": 0.02389078498293516,
+      "eval_loss": 0.25513756275177,
+      "eval_precision": 0.04501607717041801,
+      "eval_recall": 0.016260162601626018,
+      "eval_runtime": 2.9552,
+      "eval_samples_per_second": 31.132,
+      "eval_steps_per_second": 4.061,
+      "step": 200
     },
     {
-      "epoch": 4.437869822485207,
-      "grad_norm": 4.125,
-      "learning_rate": 5.562130177514793e-05,
-      "loss": 0.4718,
-      "step": 1500
+      "epoch": 4.8076923076923075,
+      "eval_accuracy": 0.8308051341890315,
+      "eval_f1": 0.022298456260720412,
+      "eval_loss": 0.2511464059352875,
+      "eval_precision": 0.04262295081967213,
+      "eval_recall": 0.015098722415795587,
+      "eval_runtime": 2.9617,
+      "eval_samples_per_second": 31.064,
+      "eval_steps_per_second": 4.052,
+      "step": 250
     },
     {
-      "epoch": 4.437869822485207,
-      "eval_accuracy": 0.8557155074116306,
-      "eval_f1": 0.01738934056007227,
-      "eval_loss": 0.44876301288604736,
-      "eval_precision": 0.014739663093415008,
-      "eval_recall": 0.021200440528634363,
-      "eval_runtime": 95.5719,
-      "eval_samples_per_second": 6.278,
-      "eval_steps_per_second": 0.785,
-      "step": 1500
+      "epoch": 5.769230769230769,
+      "grad_norm": 1.3515625,
+      "learning_rate": 4.230769230769231e-05,
+      "loss": 0.2775,
+      "step": 300
     },
     {
-      "epoch": 5.9171597633136095,
-      "grad_norm": 5.0,
-      "learning_rate": 4.0828402366863904e-05,
-      "loss": 0.4697,
-      "step": 2000
+      "epoch": 5.769230769230769,
+      "eval_accuracy": 0.8308051341890315,
+      "eval_f1": 0.02181208053691275,
+      "eval_loss": 0.24583899974822998,
+      "eval_precision": 0.03927492447129909,
+      "eval_recall": 0.015098722415795587,
+      "eval_runtime": 2.9496,
+      "eval_samples_per_second": 31.19,
+      "eval_steps_per_second": 4.068,
+      "step": 300
     },
     {
-      "epoch": 5.9171597633136095,
-      "eval_accuracy": 0.8567702394526796,
-      "eval_f1": 0.016934408165632433,
-      "eval_loss": 0.44674479961395264,
-      "eval_precision": 0.01463072452149514,
-      "eval_recall": 0.020099118942731278,
-      "eval_runtime": 95.576,
-      "eval_samples_per_second": 6.278,
-      "eval_steps_per_second": 0.785,
-      "step": 2000
+      "epoch": 6.730769230769231,
+      "eval_accuracy": 0.8309112124748064,
+      "eval_f1": 0.020477815699658706,
+      "eval_loss": 0.24579653143882751,
+      "eval_precision": 0.03858520900321544,
+      "eval_recall": 0.013937282229965157,
+      "eval_runtime": 2.9607,
+      "eval_samples_per_second": 31.074,
+      "eval_steps_per_second": 4.053,
+      "step": 350
     },
     {
-      "epoch": 7.396449704142012,
-      "grad_norm": 5.3125,
-      "learning_rate": 2.6035502958579882e-05,
-      "loss": 0.4663,
-      "step": 2500
+      "epoch": 7.6923076923076925,
+      "grad_norm": 1.5625,
+      "learning_rate": 2.307692307692308e-05,
+      "loss": 0.2677,
+      "step": 400
     },
     {
-      "epoch": 7.396449704142012,
-      "eval_accuracy": 0.8562428734321551,
-      "eval_f1": 0.017444150582409543,
-      "eval_loss": 0.4421093761920929,
-      "eval_precision": 0.01475207004853907,
-      "eval_recall": 0.021338105726872246,
-      "eval_runtime": 95.5992,
-      "eval_samples_per_second": 6.276,
-      "eval_steps_per_second": 0.785,
-      "step": 2500
+      "epoch": 7.6923076923076925,
+      "eval_accuracy": 0.8305929776174817,
+      "eval_f1": 0.020168067226890754,
+      "eval_loss": 0.24363110959529877,
+      "eval_precision": 0.0364741641337386,
+      "eval_recall": 0.013937282229965157,
+      "eval_runtime": 2.955,
+      "eval_samples_per_second": 31.133,
+      "eval_steps_per_second": 4.061,
+      "step": 400
     },
     {
-      "epoch": 8.875739644970414,
-      "grad_norm": 6.0,
-      "learning_rate": 1.1242603550295859e-05,
-      "loss": 0.4614,
-      "step": 3000
+      "epoch": 8.653846153846153,
+      "eval_accuracy": 0.8306990559032567,
+      "eval_f1": 0.020202020202020204,
+      "eval_loss": 0.24341881275177002,
+      "eval_precision": 0.03669724770642202,
+      "eval_recall": 0.013937282229965157,
+      "eval_runtime": 2.9545,
+      "eval_samples_per_second": 31.138,
+      "eval_steps_per_second": 4.062,
+      "step": 450
     },
     {
-      "epoch": 8.875739644970414,
-      "eval_accuracy": 0.8567559863169898,
-      "eval_f1": 0.01658185144948253,
-      "eval_loss": 0.4435156285762787,
-      "eval_precision": 0.014180929095354523,
-      "eval_recall": 0.01996145374449339,
-      "eval_runtime": 95.7263,
-      "eval_samples_per_second": 6.268,
-      "eval_steps_per_second": 0.783,
-      "step": 3000
+      "epoch": 9.615384615384615,
+      "grad_norm": 1.9453125,
+      "learning_rate": 3.846153846153847e-06,
+      "loss": 0.2664,
+      "step": 500
+    },
+    {
+      "epoch": 9.615384615384615,
+      "eval_accuracy": 0.8309112124748064,
+      "eval_f1": 0.02027027027027027,
+      "eval_loss": 0.24354620277881622,
+      "eval_precision": 0.03715170278637771,
+      "eval_recall": 0.013937282229965157,
+      "eval_runtime": 3.1385,
+      "eval_samples_per_second": 29.313,
+      "eval_steps_per_second": 3.823,
+      "step": 500
     }
   ],
-  "logging_steps": 500,
-  "max_steps": 3380,
+  "logging_steps": 100,
+  "max_steps": 520,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 10,
   "save_steps": 500,
-  "total_flos": 2.351623317247311e+18,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.476838693139221e+16,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c62084dc75fa1981b3f4998b0236d32b0af168ce250dbc7124b6db0630753d05
-size 4984
+oid sha256:d1e2a0fefa001714bb24115f77edcee86e4a522f49c3a2ee60ab00018a3414ec
+size 5112