Training in progress, epoch 8

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3f750124e4d89a3026dcc378ca2ca45664b8ea7a69439f4d5bb6a5c9be6e7cd
+oid sha256:94e237c151f088f5ef335bdea20a800ef4fa3ba97cc2103e5c90fb26c58b560a
 size 26235704

checkpoint-27092/README.md

@@ -4,6 +4,17 @@ library_name: peft
 ## Training procedure
 
 
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - load_in_8bit: False
 - load_in_4bit: True
@@ -16,5 +27,6 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_compute_dtype: bfloat16
 ### Framework versions
 
+- PEFT 0.4.0
 
 - PEFT 0.4.0

{checkpoint-10159 → checkpoint-27092}/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17c61fd486c7c250e6d95b7c933d9743466ee75f384c49a72cb49ec6cba50191
+oid sha256:2fae3642ea775d16cfc6e7405d9b0d33fe717646f50affd3464e98b0f78a5a35
 size 26271757

{checkpoint-10159 → checkpoint-30478}/README.md

@@ -4,17 +4,6 @@ library_name: peft
 ## Training procedure
 
 
-The following `bitsandbytes` quantization config was used during training:
-- load_in_8bit: False
-- load_in_4bit: True
-- llm_int8_threshold: 6.0
-- llm_int8_skip_modules: None
-- llm_int8_enable_fp32_cpu_offload: False
-- llm_int8_has_fp16_weight: False
-- bnb_4bit_quant_type: nf4
-- bnb_4bit_use_double_quant: True
-- bnb_4bit_compute_dtype: bfloat16
-
 The following `bitsandbytes` quantization config was used during training:
 - load_in_8bit: False
 - load_in_4bit: True
@@ -27,6 +16,5 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_compute_dtype: bfloat16
 ### Framework versions
 
-- PEFT 0.4.0
 
 - PEFT 0.4.0

{checkpoint-10159 → checkpoint-30478}/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d0fdd0246e28dcc288595f1c8554ffca9e7cd45cca9a505297f6741f234581d
+oid sha256:94e237c151f088f5ef335bdea20a800ef4fa3ba97cc2103e5c90fb26c58b560a
 size 26235704

{checkpoint-10159 → checkpoint-30478}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2448140ca33c8b053fc4b18e424248206c18141e5b2d47ecd5a41ec9341ff24b
+oid sha256:bec8d66e5cfc64876022d3b37f12906f981209b574b2918af8df0587cf390039
 size 52562821

{checkpoint-10159 → checkpoint-30478}/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43dad6d0c9937215540f067b77f6347c85c47db32863a5edf2c67432c21e76ea
+oid sha256:5a84355efbed41ff0143d74388bd66566346b23439d387a38bcd095979522ce4
 size 14575

{checkpoint-10159 → checkpoint-30478}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:561b4f7d15722536516eb958de38a78ef8c31d0dd4875a02fdf194e8fa92d380
+oid sha256:abf5d65427578588c7df7f882cffe0e586dd3c598f6974c8e217f65b280de5a5
 size 627

{checkpoint-10159 → checkpoint-30478}/trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 0.7562845945358276,
-  "best_model_checkpoint": "./Zeroshot/01-12-23-NousResearch-Nous-Hermes-Llama2-13b_multilang-dataset-3.0.3-portuguese-2_epochs-10_batch_2/checkpoints/checkpoint-10159",
-  "epoch": 2.999852354938727,
-  "global_step": 10159,
+  "best_metric": 0.7249947190284729,
+  "best_model_checkpoint": "./Zeroshot/01-12-23-NousResearch-Nous-Hermes-Llama2-13b_multilang-dataset-3.0.3-portuguese-2_epochs-10_batch_2/checkpoints/checkpoint-20319",
+  "epoch": 8.999852354938728,
+  "global_step": 30478,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -150,11 +150,299 @@
       "eval_samples_per_second": 11.347,
       "eval_steps_per_second": 1.424,
       "step": 10159
+    },
+    {
+      "epoch": 3.1,
+      "learning_rate": 0.0003486234464660031,
+      "loss": 0.6199,
+      "step": 10500
+    },
+    {
+      "epoch": 3.25,
+      "learning_rate": 0.0003415452256149365,
+      "loss": 0.595,
+      "step": 11000
+    },
+    {
+      "epoch": 3.4,
+      "learning_rate": 0.0003340772155486902,
+      "loss": 0.5962,
+      "step": 11500
+    },
+    {
+      "epoch": 3.54,
+      "learning_rate": 0.0003262690400990177,
+      "loss": 0.6019,
+      "step": 12000
+    },
+    {
+      "epoch": 3.69,
+      "learning_rate": 0.00031811015213363493,
+      "loss": 0.6043,
+      "step": 12500
+    },
+    {
+      "epoch": 3.84,
+      "learning_rate": 0.0003096375227201415,
+      "loss": 0.5984,
+      "step": 13000
+    },
+    {
+      "epoch": 3.99,
+      "learning_rate": 0.0003008736580950757,
+      "loss": 0.6011,
+      "step": 13500
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 0.738968551158905,
+      "eval_runtime": 132.6823,
+      "eval_samples_per_second": 11.35,
+      "eval_steps_per_second": 1.424,
+      "step": 13546
+    },
+    {
+      "epoch": 4.13,
+      "learning_rate": 0.00029184183811650945,
+      "loss": 0.5125,
+      "step": 14000
+    },
+    {
+      "epoch": 4.28,
+      "learning_rate": 0.00028256605442468665,
+      "loss": 0.5143,
+      "step": 14500
+    },
+    {
+      "epoch": 4.43,
+      "learning_rate": 0.00027307094671192217,
+      "loss": 0.5171,
+      "step": 15000
+    },
+    {
+      "epoch": 4.58,
+      "learning_rate": 0.0002633817372710499,
+      "loss": 0.5133,
+      "step": 15500
+    },
+    {
+      "epoch": 4.72,
+      "learning_rate": 0.00025352416399628225,
+      "loss": 0.5155,
+      "step": 16000
+    },
+    {
+      "epoch": 4.87,
+      "learning_rate": 0.0002435244120144539,
+      "loss": 0.5217,
+      "step": 16500
+    },
+    {
+      "epoch": 5.0,
+      "eval_loss": 0.7306132912635803,
+      "eval_runtime": 132.674,
+      "eval_samples_per_second": 11.351,
+      "eval_steps_per_second": 1.425,
+      "step": 16932
+    },
+    {
+      "epoch": 5.02,
+      "learning_rate": 0.00023342937243316328,
+      "loss": 0.5015,
+      "step": 17000
+    },
+    {
+      "epoch": 5.17,
+      "learning_rate": 0.00022322540906831537,
+      "loss": 0.4297,
+      "step": 17500
+    },
+    {
+      "epoch": 5.32,
+      "learning_rate": 0.00021295975097746452,
+      "loss": 0.4336,
+      "step": 18000
+    },
+    {
+      "epoch": 5.46,
+      "learning_rate": 0.00020265966730197618,
+      "loss": 0.4382,
+      "step": 18500
+    },
+    {
+      "epoch": 5.61,
+      "learning_rate": 0.00019235251862948508,
+      "loss": 0.4357,
+      "step": 19000
+    },
+    {
+      "epoch": 5.76,
+      "learning_rate": 0.0001820862195310506,
+      "loss": 0.4411,
+      "step": 19500
+    },
+    {
+      "epoch": 5.91,
+      "learning_rate": 0.0001718469024885363,
+      "loss": 0.4411,
+      "step": 20000
+    },
+    {
+      "epoch": 6.0,
+      "eval_loss": 0.7249947190284729,
+      "eval_runtime": 132.741,
+      "eval_samples_per_second": 11.345,
+      "eval_steps_per_second": 1.424,
+      "step": 20319
+    },
+    {
+      "epoch": 6.05,
+      "learning_rate": 0.00016168236981825817,
+      "loss": 0.4086,
+      "step": 20500
+    },
+    {
+      "epoch": 6.2,
+      "learning_rate": 0.00015161962203746017,
+      "loss": 0.3539,
+      "step": 21000
+    },
+    {
+      "epoch": 6.35,
+      "learning_rate": 0.00014170511189134913,
+      "loss": 0.3628,
+      "step": 21500
+    },
+    {
+      "epoch": 6.5,
+      "learning_rate": 0.00013192544705149724,
+      "loss": 0.3737,
+      "step": 22000
+    },
+    {
+      "epoch": 6.64,
+      "learning_rate": 0.00012232661178840682,
+      "loss": 0.3739,
+      "step": 22500
+    },
+    {
+      "epoch": 6.79,
+      "learning_rate": 0.00011293410393119082,
+      "loss": 0.3781,
+      "step": 23000
+    },
+    {
+      "epoch": 6.94,
+      "learning_rate": 0.00010377287323195157,
+      "loss": 0.3773,
+      "step": 23500
+    },
+    {
+      "epoch": 7.0,
+      "eval_loss": 0.7291969656944275,
+      "eval_runtime": 132.6924,
+      "eval_samples_per_second": 11.35,
+      "eval_steps_per_second": 1.424,
+      "step": 23705
+    },
+    {
+      "epoch": 7.09,
+      "learning_rate": 9.486725509060299e-05,
+      "loss": 0.3353,
+      "step": 24000
+    },
+    {
+      "epoch": 7.23,
+      "learning_rate": 8.624090591162438e-05,
+      "loss": 0.3143,
+      "step": 24500
+    },
+    {
+      "epoch": 7.38,
+      "learning_rate": 7.79167402644616e-05,
+      "loss": 0.3206,
+      "step": 25000
+    },
+    {
+      "epoch": 7.53,
+      "learning_rate": 6.991687001449847e-05,
+      "loss": 0.3215,
+      "step": 25500
+    },
+    {
+      "epoch": 7.68,
+      "learning_rate": 6.226254558628854e-05,
+      "loss": 0.317,
+      "step": 26000
+    },
+    {
+      "epoch": 7.83,
+      "learning_rate": 5.497409951507149e-05,
+      "loss": 0.3158,
+      "step": 26500
+    },
+    {
+      "epoch": 7.97,
+      "learning_rate": 4.807089243652132e-05,
+      "loss": 0.3103,
+      "step": 27000
+    },
+    {
+      "epoch": 8.0,
+      "eval_loss": 0.7368675470352173,
+      "eval_runtime": 132.8691,
+      "eval_samples_per_second": 11.334,
+      "eval_steps_per_second": 1.422,
+      "step": 27092
+    },
+    {
+      "epoch": 8.12,
+      "learning_rate": 4.158384649176379e-05,
+      "loss": 0.28,
+      "step": 27500
+    },
+    {
+      "epoch": 8.27,
+      "learning_rate": 3.551592750827968e-05,
+      "loss": 0.2736,
+      "step": 28000
+    },
+    {
+      "epoch": 8.42,
+      "learning_rate": 2.9872348640621118e-05,
+      "loss": 0.2826,
+      "step": 28500
+    },
+    {
+      "epoch": 8.56,
+      "learning_rate": 2.468068770496861e-05,
+      "loss": 0.2811,
+      "step": 29000
+    },
+    {
+      "epoch": 8.71,
+      "learning_rate": 1.9954735549870106e-05,
+      "loss": 0.284,
+      "step": 29500
+    },
+    {
+      "epoch": 8.86,
+      "learning_rate": 1.5707045940146493e-05,
+      "loss": 0.273,
+      "step": 30000
+    },
+    {
+      "epoch": 9.0,
+      "eval_loss": 0.7521340250968933,
+      "eval_runtime": 133.117,
+      "eval_samples_per_second": 11.313,
+      "eval_steps_per_second": 1.42,
+      "step": 30478
     }
   ],
   "max_steps": 33860,
   "num_train_epochs": 10,
-  "total_flos": 4.52869206317568e+17,
+  "total_flos": 1.3575338402285568e+18,
   "trial_name": null,
   "trial_params": null
 }

runs/Dec01_03-27-22_baa8ebef2c59/events.out.tfevents.1701401247.baa8ebef2c59.118280.1

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8771a09a819a481eebc55187f5f267890f66f8f6630892f21cd4dcdf6cfed59
-size 15496
+oid sha256:338d509cd5e48963bb4f227a760dc452d6b5d378bb655a26db46c298d3070f7e
+size 16732