Training in progress, epoch 9

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94e237c151f088f5ef335bdea20a800ef4fa3ba97cc2103e5c90fb26c58b560a
+oid sha256:d9d2b3d4cac18b0038832d940fe40145fbe984bb714203040ef3e853ecc77691
 size 26235704

checkpoint-30478/README.md

@@ -4,6 +4,17 @@ library_name: peft
 ## Training procedure
 
 
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - load_in_8bit: False
 - load_in_4bit: True
@@ -16,5 +27,6 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_compute_dtype: bfloat16
 ### Framework versions
 
+- PEFT 0.4.0
 
 - PEFT 0.4.0

{checkpoint-13546 → checkpoint-30478}/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b4e5fee9061991048c4490a5bf46a53acc51804defbde01a2de3910aec0ccc3
+oid sha256:de8e207806fea9e1844e1282bf91a07fb700527b9292023111a6a567896dea3b
 size 26271757

{checkpoint-13546 → checkpoint-33860}/README.md

@@ -4,17 +4,6 @@ library_name: peft
 ## Training procedure
 
 
-The following `bitsandbytes` quantization config was used during training:
-- load_in_8bit: False
-- load_in_4bit: True
-- llm_int8_threshold: 6.0
-- llm_int8_skip_modules: None
-- llm_int8_enable_fp32_cpu_offload: False
-- llm_int8_has_fp16_weight: False
-- bnb_4bit_quant_type: nf4
-- bnb_4bit_use_double_quant: True
-- bnb_4bit_compute_dtype: bfloat16
-
 The following `bitsandbytes` quantization config was used during training:
 - load_in_8bit: False
 - load_in_4bit: True
@@ -27,6 +16,5 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_compute_dtype: bfloat16
 ### Framework versions
 
-- PEFT 0.4.0
 
 - PEFT 0.4.0

{checkpoint-13546 → checkpoint-33860}/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81e441a39a1b44e95de1c018f9894ae9e41dd67f9357e05ca7fd396a95396f16
+oid sha256:d9d2b3d4cac18b0038832d940fe40145fbe984bb714203040ef3e853ecc77691
 size 26235704

{checkpoint-13546 → checkpoint-33860}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e03dd5dbf12d0ea2518ace4902adad55d2a1cb7f02cb30d2ec755585f7d19ef1
+oid sha256:18e6fa52809a7d7368080278ea5322a2d47f94a22ded3a2f8a1881e6b9358dbb
 size 52562821

{checkpoint-13546 → checkpoint-33860}/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34bb7b30caea09be5f7ab712abbfef74f79a753bccbab6e765b5973f5163b4e6
+oid sha256:8257ab56d9f27c8856724fb1225546a62367aafd45a911c8c0d5e52cb3f577a2
 size 14575

{checkpoint-13546 → checkpoint-33860}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de172a61ab602116d7ad21e9aab3af34df2feb5343e4a4175fc9f40a41de5bbc
+oid sha256:e54f8af7a488cf90c2c6be279ef394f968fad8e2f0d29e9deccd38c75c1a8187
 size 627

{checkpoint-13546 → checkpoint-33860}/trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 0.738968551158905,
-  "best_model_checkpoint": "./Zeroshot/01-12-23-NousResearch-Nous-Hermes-Llama2-13b_multilang-dataset-3.0.3-portuguese-2_epochs-10_batch_2/checkpoints/checkpoint-13546",
-  "epoch": 4.0,
-  "global_step": 13546,
+  "best_metric": 0.7249947190284729,
+  "best_model_checkpoint": "./Zeroshot/01-12-23-NousResearch-Nous-Hermes-Llama2-13b_multilang-dataset-3.0.3-portuguese-2_epochs-10_batch_2/checkpoints/checkpoint-20319",
+  "epoch": 9.998523549387272,
+  "global_step": 33860,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -200,11 +200,299 @@
       "eval_samples_per_second": 11.35,
       "eval_steps_per_second": 1.424,
       "step": 13546
+    },
+    {
+      "epoch": 4.13,
+      "learning_rate": 0.00029184183811650945,
+      "loss": 0.5125,
+      "step": 14000
+    },
+    {
+      "epoch": 4.28,
+      "learning_rate": 0.00028256605442468665,
+      "loss": 0.5143,
+      "step": 14500
+    },
+    {
+      "epoch": 4.43,
+      "learning_rate": 0.00027307094671192217,
+      "loss": 0.5171,
+      "step": 15000
+    },
+    {
+      "epoch": 4.58,
+      "learning_rate": 0.0002633817372710499,
+      "loss": 0.5133,
+      "step": 15500
+    },
+    {
+      "epoch": 4.72,
+      "learning_rate": 0.00025352416399628225,
+      "loss": 0.5155,
+      "step": 16000
+    },
+    {
+      "epoch": 4.87,
+      "learning_rate": 0.0002435244120144539,
+      "loss": 0.5217,
+      "step": 16500
+    },
+    {
+      "epoch": 5.0,
+      "eval_loss": 0.7306132912635803,
+      "eval_runtime": 132.674,
+      "eval_samples_per_second": 11.351,
+      "eval_steps_per_second": 1.425,
+      "step": 16932
+    },
+    {
+      "epoch": 5.02,
+      "learning_rate": 0.00023342937243316328,
+      "loss": 0.5015,
+      "step": 17000
+    },
+    {
+      "epoch": 5.17,
+      "learning_rate": 0.00022322540906831537,
+      "loss": 0.4297,
+      "step": 17500
+    },
+    {
+      "epoch": 5.32,
+      "learning_rate": 0.00021295975097746452,
+      "loss": 0.4336,
+      "step": 18000
+    },
+    {
+      "epoch": 5.46,
+      "learning_rate": 0.00020265966730197618,
+      "loss": 0.4382,
+      "step": 18500
+    },
+    {
+      "epoch": 5.61,
+      "learning_rate": 0.00019235251862948508,
+      "loss": 0.4357,
+      "step": 19000
+    },
+    {
+      "epoch": 5.76,
+      "learning_rate": 0.0001820862195310506,
+      "loss": 0.4411,
+      "step": 19500
+    },
+    {
+      "epoch": 5.91,
+      "learning_rate": 0.0001718469024885363,
+      "loss": 0.4411,
+      "step": 20000
+    },
+    {
+      "epoch": 6.0,
+      "eval_loss": 0.7249947190284729,
+      "eval_runtime": 132.741,
+      "eval_samples_per_second": 11.345,
+      "eval_steps_per_second": 1.424,
+      "step": 20319
+    },
+    {
+      "epoch": 6.05,
+      "learning_rate": 0.00016168236981825817,
+      "loss": 0.4086,
+      "step": 20500
+    },
+    {
+      "epoch": 6.2,
+      "learning_rate": 0.00015161962203746017,
+      "loss": 0.3539,
+      "step": 21000
+    },
+    {
+      "epoch": 6.35,
+      "learning_rate": 0.00014170511189134913,
+      "loss": 0.3628,
+      "step": 21500
+    },
+    {
+      "epoch": 6.5,
+      "learning_rate": 0.00013192544705149724,
+      "loss": 0.3737,
+      "step": 22000
+    },
+    {
+      "epoch": 6.64,
+      "learning_rate": 0.00012232661178840682,
+      "loss": 0.3739,
+      "step": 22500
+    },
+    {
+      "epoch": 6.79,
+      "learning_rate": 0.00011293410393119082,
+      "loss": 0.3781,
+      "step": 23000
+    },
+    {
+      "epoch": 6.94,
+      "learning_rate": 0.00010377287323195157,
+      "loss": 0.3773,
+      "step": 23500
+    },
+    {
+      "epoch": 7.0,
+      "eval_loss": 0.7291969656944275,
+      "eval_runtime": 132.6924,
+      "eval_samples_per_second": 11.35,
+      "eval_steps_per_second": 1.424,
+      "step": 23705
+    },
+    {
+      "epoch": 7.09,
+      "learning_rate": 9.486725509060299e-05,
+      "loss": 0.3353,
+      "step": 24000
+    },
+    {
+      "epoch": 7.23,
+      "learning_rate": 8.624090591162438e-05,
+      "loss": 0.3143,
+      "step": 24500
+    },
+    {
+      "epoch": 7.38,
+      "learning_rate": 7.79167402644616e-05,
+      "loss": 0.3206,
+      "step": 25000
+    },
+    {
+      "epoch": 7.53,
+      "learning_rate": 6.991687001449847e-05,
+      "loss": 0.3215,
+      "step": 25500
+    },
+    {
+      "epoch": 7.68,
+      "learning_rate": 6.226254558628854e-05,
+      "loss": 0.317,
+      "step": 26000
+    },
+    {
+      "epoch": 7.83,
+      "learning_rate": 5.497409951507149e-05,
+      "loss": 0.3158,
+      "step": 26500
+    },
+    {
+      "epoch": 7.97,
+      "learning_rate": 4.807089243652132e-05,
+      "loss": 0.3103,
+      "step": 27000
+    },
+    {
+      "epoch": 8.0,
+      "eval_loss": 0.7368675470352173,
+      "eval_runtime": 132.8691,
+      "eval_samples_per_second": 11.334,
+      "eval_steps_per_second": 1.422,
+      "step": 27092
+    },
+    {
+      "epoch": 8.12,
+      "learning_rate": 4.158384649176379e-05,
+      "loss": 0.28,
+      "step": 27500
+    },
+    {
+      "epoch": 8.27,
+      "learning_rate": 3.551592750827968e-05,
+      "loss": 0.2736,
+      "step": 28000
+    },
+    {
+      "epoch": 8.42,
+      "learning_rate": 2.9872348640621118e-05,
+      "loss": 0.2826,
+      "step": 28500
+    },
+    {
+      "epoch": 8.56,
+      "learning_rate": 2.468068770496861e-05,
+      "loss": 0.2811,
+      "step": 29000
+    },
+    {
+      "epoch": 8.71,
+      "learning_rate": 1.9954735549870106e-05,
+      "loss": 0.284,
+      "step": 29500
+    },
+    {
+      "epoch": 8.86,
+      "learning_rate": 1.5707045940146493e-05,
+      "loss": 0.273,
+      "step": 30000
+    },
+    {
+      "epoch": 9.0,
+      "eval_loss": 0.7521340250968933,
+      "eval_runtime": 133.117,
+      "eval_samples_per_second": 11.313,
+      "eval_steps_per_second": 1.42,
+      "step": 30478
+    },
+    {
+      "epoch": 9.01,
+      "learning_rate": 1.1948902209746626e-05,
+      "loss": 0.2787,
+      "step": 30500
+    },
+    {
+      "epoch": 9.15,
+      "learning_rate": 8.696300012677295e-06,
+      "loss": 0.2593,
+      "step": 31000
+    },
+    {
+      "epoch": 9.3,
+      "learning_rate": 5.944845796839005e-06,
+      "loss": 0.2633,
+      "step": 31500
+    },
+    {
+      "epoch": 9.45,
+      "learning_rate": 3.7088692431083593e-06,
+      "loss": 0.2595,
+      "step": 32000
+    },
+    {
+      "epoch": 9.6,
+      "learning_rate": 1.9943098791995297e-06,
+      "loss": 0.2606,
+      "step": 32500
+    },
+    {
+      "epoch": 9.74,
+      "learning_rate": 8.057221682808091e-07,
+      "loss": 0.2535,
+      "step": 33000
+    },
+    {
+      "epoch": 9.89,
+      "learning_rate": 1.4626341074424065e-07,
+      "loss": 0.2599,
+      "step": 33500
+    },
+    {
+      "epoch": 10.0,
+      "eval_loss": 0.761792778968811,
+      "eval_runtime": 133.1533,
+      "eval_samples_per_second": 11.31,
+      "eval_steps_per_second": 1.419,
+      "step": 33860
     }
   ],
   "max_steps": 33860,
   "num_train_epochs": 10,
-  "total_flos": 6.037383614590157e+17,
+  "total_flos": 1.508264150738473e+18,
   "trial_name": null,
   "trial_params": null
 }

runs/Dec01_03-27-22_baa8ebef2c59/events.out.tfevents.1701401247.baa8ebef2c59.118280.1

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:338d509cd5e48963bb4f227a760dc452d6b5d378bb655a26db46c298d3070f7e
-size 16732
+oid sha256:752c4fdfaeb486837f616afa0776168c637f78e369a638462e518adf13065ea7
+size 18128