Training in progress, step 50, checkpoint

checkpoint-50/README.md

@@ -216,4 +216,23 @@ The following `bitsandbytes` quantization config was used during training:
 ### Framework versions
 
 
+- PEFT 0.6.0
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+
+### Framework versions
+
+
 - PEFT 0.6.0

checkpoint-50/adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "/kaggle/input/vietcuna-for-binary-classification/kalapa-vietcuna-3b",
+  "base_model_name_or_path": "vietcuna-3b-v2/kalapa-vietcuna-3b/",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -9,10 +9,10 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "lora_alpha": 32,
-  "lora_dropout": 0.1,
+  "lora_dropout": 0.05,
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 8,
+  "r": 16,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [

checkpoint-50/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0e2c08e5163bb3f67a5ce7acd4dd75e8daa2a7869bb3a6fd5bbe81296f5a646
-size 9859800
+oid sha256:4166664b368ce4cd6e33f9c9e5988f4cd0f67acb476bfaf70382c8384056788a
+size 19690328

checkpoint-50/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:649f6f10acc825530de20a2818f6a35ceaec27f008838bbd89d1eafa771845a3
-size 42724
+oid sha256:b45841b33a84da2a1b486d9551765151b191da5b910c16259aacd2114b08f224
+size 38087098

checkpoint-50/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be8ed0ba649837443f79e95d66f2cdebd362fce4049ee79d5b9831b2c22ccf76
+oid sha256:f197d949afeca5f73d2f9c4e8008213842dfb0a96b35f9276fd815dc3b955de7
 size 14244

checkpoint-50/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6a36ba0ea017dad0467a59150d5a3a03523965e4b322c77d048735654bcbefb
+oid sha256:609839f04f1b0fbf1991d0fad32a3ffd669d0403ec0a02950e111710295f6725
 size 1064

checkpoint-50/special_tokens_map.json

@@ -2,6 +2,6 @@
   "bos_token": "<s>",
   "eos_token": "</s>",
   "pad_token": "<pad>",
-  "sep_token": "<s>",
+  "sep_token": "\n\n",
   "unk_token": "<unk>"
 }

checkpoint-50/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

checkpoint-50/tokenizer_config.json

@@ -39,7 +39,7 @@
   "eos_token": "</s>",
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<pad>",
-  "sep_token": "<s>",
+  "sep_token": "\n\n",
   "tokenizer_class": "BloomTokenizer",
   "unk_token": "<unk>"
 }

checkpoint-50/trainer_state.json

@@ -1,7 +1,7 @@
 {
-  "best_metric": 0.522564172744751,
+  "best_metric": 6.581231117248535,
   "best_model_checkpoint": "output/checkpoint-50",
-  "epoch": 0.34904013961605584,
+  "epoch": 0.2552648372686662,
   "eval_steps": 50,
   "global_step": 50,
   "is_hyper_param_search": false,
@@ -10,31 +10,31 @@
   "log_history": [
     {
       "epoch": 0.01,
-      "learning_rate": 1.0000000000000001e-07,
-      "loss": 0.4196,
+      "learning_rate": 2.777777777777778e-07,
+      "loss": 9.5569,
       "step": 1
     },
     {
-      "epoch": 0.35,
-      "learning_rate": 4.6e-06,
-      "loss": 0.4279,
+      "epoch": 0.26,
+      "learning_rate": 1.25e-05,
+      "loss": 6.958,
       "step": 50
     },
     {
-      "epoch": 0.35,
-      "eval_accuracy": 0.7657142857142857,
-      "eval_loss": 0.522564172744751,
-      "eval_runtime": 29.0326,
-      "eval_samples_per_second": 6.028,
-      "eval_steps_per_second": 1.516,
+      "epoch": 0.26,
+      "eval_accuracy": 0.42857142857142855,
+      "eval_loss": 6.581231117248535,
+      "eval_runtime": 28.2966,
+      "eval_samples_per_second": 6.184,
+      "eval_steps_per_second": 1.555,
       "step": 50
     }
   ],
   "logging_steps": 50,
-  "max_steps": 500,
-  "num_train_epochs": 4,
+  "max_steps": 1000,
+  "num_train_epochs": 6,
   "save_steps": 50,
-  "total_flos": 1.891662110539776e+16,
+  "total_flos": 1.854149272129536e+16,
   "trial_name": null,
   "trial_params": null
 }

checkpoint-50/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1b0a94354c0f3dfee9c93e225dbd48ad6f856f98977ef38ff3415910328e64a
+oid sha256:5a0ef36d7c31265f68a3edc8ae53ce9272dcc90a79f261e29a2a46e040c77a33
 size 4600