Redo chat training

results_phi-2-super/adapter_config.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:371957d56469d7d330f8a67cc355891cc7e0d7536a787e3c225b46d00cbeeb96
-size 693
+oid sha256:13eef94a821e1664305a246a380cf594f1c9e3dd4759e01b41a1f614201f072f
+size 690

results_phi-2-super/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f40a24e7e4b9e547e0396f353ea7f48a5fbf3713cde5bfabb7f9d24a37c3b651
-size 188794992
+oid sha256:6231c5a9050bbfb4f34b9ae2190441edc2d4aebdc10f05cfedcb5fe287a6c5b5
+size 377538512

results_phi-2-super/checkpoint-100/adapter_config.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:371957d56469d7d330f8a67cc355891cc7e0d7536a787e3c225b46d00cbeeb96
-size 693
+oid sha256:13eef94a821e1664305a246a380cf594f1c9e3dd4759e01b41a1f614201f072f
+size 690

results_phi-2-super/checkpoint-100/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f40a24e7e4b9e547e0396f353ea7f48a5fbf3713cde5bfabb7f9d24a37c3b651
-size 188794992
+oid sha256:920ad6c5c63efda14d9475569cb990363c103e3992e36ddb7eacccd7c6616330
+size 377538512

results_phi-2-super/checkpoint-100/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ddf015e87a1893b2a9d67df5abb390fc221ec752fef5c20a56b42449c07e788
+oid sha256:71e207546bc91c8d641672563ea18c2da86c1a1fb15f8f76ae17a73e71e9322d
 size 189572052

results_phi-2-super/checkpoint-100/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:804a017089d7db98f6fc8bd0aa7c5e3f00cf43462d4247447772b85e0fb216f6
+oid sha256:2a91f983c9a6853f984e81a0938214444ef6e4cd928fa241e1da3c50ab92c316
 size 14244

results_phi-2-super/checkpoint-100/tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:337da36be7a71a6e88aa9148967a7bc8736f4b47c7de8e19ba92b89e80734cfc
-size 2114924
+oid sha256:972811be42a0dd9f0ffcdd9563d2234c47fda2bb248d1e2934f3c20ceed07cb6
+size 2114952

results_phi-2-super/checkpoint-100/tokenizer_config.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:712d74738df428d7648e65ebdc6058f49564f206ea86286764f3b0714840e60a
-size 7871
+oid sha256:06369f8433146a04712326910a5c8a8866aae246601dbe5c836b1d5d96ee080f
+size 7905

results_phi-2-super/checkpoint-100/trainer_state.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a33e105aa4e5c96151c2aec1b93595d1db028114daf7a39794e965fbdb2b66c
-size 25215
+oid sha256:411447059825b1e539c8cc827f50228b166c55c5aecd9a7d63d8ea48367c8569
+size 25920

results_phi-2-super/checkpoint-100/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c220f4ec7f147b97dbda215ca2eba37034c573d0f8560d9af60f478120faaf3
-size 4856
+oid sha256:a710d3d12fbd08f305d743e550415b25672249d2c48fd4005249a2ff55e65a91
+size 4920

results_phi-2-super/checkpoint-99/adapter_config.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:371957d56469d7d330f8a67cc355891cc7e0d7536a787e3c225b46d00cbeeb96
-size 693
+oid sha256:13eef94a821e1664305a246a380cf594f1c9e3dd4759e01b41a1f614201f072f
+size 690

results_phi-2-super/checkpoint-99/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb6ce6fc8e79e0de6c5a7ec1ff7cd88b5f4c65d329b2413384eaafb7d7c904c7
-size 188794992
+oid sha256:6231c5a9050bbfb4f34b9ae2190441edc2d4aebdc10f05cfedcb5fe287a6c5b5
+size 377538512

results_phi-2-super/checkpoint-99/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcd922cfa715083b6091fe4243953eb5a7f89684f0c025ae8a3bf096aace7f31
+oid sha256:cd0ddabda4b864a08403a4d2bed400745bb845376cd8fa702e3d9d4514ac1777
 size 189572052

results_phi-2-super/checkpoint-99/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1391fd71b0e652f5957583be39d12a5dbb3b482d730ddb77483ca72055f48d1
+oid sha256:a5379830b2ed121780d736ed09a12ec61c30e2d65980f203ccd2003bdd8823a3
 size 14244

results_phi-2-super/checkpoint-99/tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:337da36be7a71a6e88aa9148967a7bc8736f4b47c7de8e19ba92b89e80734cfc
-size 2114924
+oid sha256:972811be42a0dd9f0ffcdd9563d2234c47fda2bb248d1e2934f3c20ceed07cb6
+size 2114952

results_phi-2-super/checkpoint-99/tokenizer_config.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:712d74738df428d7648e65ebdc6058f49564f206ea86286764f3b0714840e60a
-size 7871
+oid sha256:06369f8433146a04712326910a5c8a8866aae246601dbe5c836b1d5d96ee080f
+size 7905

results_phi-2-super/checkpoint-99/trainer_state.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0dd26b5d090209bc6311666c46769cbe4efa17e2574027ee2b19e75ce0cb192
-size 24867
+oid sha256:986a54386f80b5d1477999e23379aed157c8ddafdfe12ec362a065d73f1ae50b
+size 25540

results_phi-2-super/checkpoint-99/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c220f4ec7f147b97dbda215ca2eba37034c573d0f8560d9af60f478120faaf3
-size 4856
+oid sha256:a710d3d12fbd08f305d743e550415b25672249d2c48fd4005249a2ff55e65a91
+size 4920

results_phi-2-super/tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:337da36be7a71a6e88aa9148967a7bc8736f4b47c7de8e19ba92b89e80734cfc
-size 2114924
+oid sha256:972811be42a0dd9f0ffcdd9563d2234c47fda2bb248d1e2934f3c20ceed07cb6
+size 2114952

results_phi-2-super/tokenizer_config.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:712d74738df428d7648e65ebdc6058f49564f206ea86286764f3b0714840e60a
-size 7871
+oid sha256:06369f8433146a04712326910a5c8a8866aae246601dbe5c836b1d5d96ee080f
+size 7905

results_phi-2-super/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c220f4ec7f147b97dbda215ca2eba37034c573d0f8560d9af60f478120faaf3
-size 4856
+oid sha256:a710d3d12fbd08f305d743e550415b25672249d2c48fd4005249a2ff55e65a91
+size 4920

train_csv_dataset_phi-2-super.py

@@ -1,9 +1,9 @@
 import time
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, TrainingArguments, DataCollatorForLanguageModeling
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, TrainingArguments
 from datasets import load_dataset
 from trl import SFTTrainer
-from peft import LoraConfig, prepare_model_for_kbit_training
+from peft import LoraConfig, prepare_model_for_kbit_training, PeftModel, PeftConfig
 
 dataset = load_dataset("csv", data_files="nowhere_training_input.csv", delimiter=";", split="train")
 
@@ -11,6 +11,7 @@ if torch.cuda.is_available():
     print("Cuda is available")
 
 base_model_id = "abacaj/phi-2-super"
+base_peft_id = "./results"
 output_dir = "./results_phi-2-super"
 
 tokenizer = AutoTokenizer.from_pretrained(base_model_id)
@@ -25,23 +26,15 @@ bnb_config = BitsAndBytesConfig(load_in_4bit=True,
                                 bnb_4bit_compute_dtype=torch.bfloat16,
                                 bnb_4bit_use_double_quant=False)
 
-model = AutoModelForCausalLM.from_pretrained(base_model_id, attn_implementation="flash_attention_2", quantization_config=bnb_config, torch_dtype="auto")
+base_model = AutoModelForCausalLM.from_pretrained(base_model_id, attn_implementation="flash_attention_2", quantization_config=bnb_config, torch_dtype="auto")
+model = PeftModel.from_pretrained(base_model, base_peft_id, is_trainable=True)
 print(model)
 
 # Gradient checkpointing to save memory
-model.gradient_checkpointing_enable()
+# model.gradient_checkpointing_enable()
 
 # Freeze base model layers and cast layernorm in fp32
-model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=True)
-
-peft_config = LoraConfig(
-    r=64,
-    lora_alpha=64,
-    target_modules= ["q_proj","k_proj","v_proj","dense","fc2","fc1"],
-    bias="none",
-    lora_dropout=0.05,
-    task_type="CAUSAL_LM",
-)
+# model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=True)
 
 training_args = TrainingArguments(
     output_dir=output_dir,  # Output directory for checkpoints and predictions
@@ -79,7 +72,7 @@ def formatting_func(data):
     ]
 
     text = tokenizer.apply_chat_template(chat, tokenize=False)
-    print(text)
+    # print(text)
     data['text'] = text
     
     return data
@@ -90,11 +83,11 @@ trainer = SFTTrainer(
     model=model,
     train_dataset=dataset,
     eval_dataset=dataset,
-    peft_config=peft_config,
     args=training_args,
     max_seq_length=1024,
     packing=True,
-    dataset_text_field="text"
+    dataset_text_field="text",
+    neftune_noise_alpha=5,
 )
 
 model.config.use_cache = False  # silence the warnings. Please re-enable for inference!