Update LLAMA_Fine-Tuning.py

LLAMA_Fine-Tuning.py

@@ -6,7 +6,7 @@ Automatically generated by Colab.
 Original file is located at
     https://colab.research.google.com/drive/1C-kNPOgPiCC9ybxVKhOkWB9ts53APbOb
 
-# Fine-tune Llama 2 in Google Colab
+# Fine-tune Llama 3 in Google Colab
 """
 
 !pip install -q accelerate==0.21.0 peft==0.4.0 bitsandbytes==0.40.2 transformers==4.31.0 trl==0.4.7
@@ -38,7 +38,7 @@ token_name = "XXXX"
 
 
 # Fine-tuned model name
-new_model = "llama-2-7b-minipython"
+new_model = "llama3_python_TFG"
 
 ################################################################################
 # QLoRA parameters
@@ -194,6 +194,64 @@ trainer.train()
 
 model.save_pretrained(new_model)
 
+from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, DataCollatorWithPadding
+from datasets import Dataset
+
+def load_text_file(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        return [line.strip() for line in f if line.strip()]
+
+train_texts = load_text_file('LLAMA_DatosEntrenamiento.txt')
+val_texts = load_text_file('LLAMA_DatosValidacion.txt')
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, token=token_name)
+
+def tokenize_and_encode(texts):
+    encodings = tokenizer(texts, truncation=True, padding="longest", max_length=512, return_tensors="pt")
+    encodings['labels'] = encodings['input_ids'].clone()  # Duplicar input_ids para usar como labels
+    return encodings
+
+train_encodings = tokenize_and_encode(train_texts)
+val_encodings = tokenize_and_encode(val_texts)
+
+train_dataset = Dataset.from_dict({key: val.numpy() for key, val in train_encodings.items()})
+val_dataset = Dataset.from_dict({key: val.numpy() for key, val in val_encodings.items()})
+
+training_arguments = TrainingArguments(
+    output_dir=output_dir,
+    evaluation_strategy="steps",  # Evaluar basado en el número de pasos
+    eval_steps=500,  # Evaluar cada 500 pasos
+    num_train_epochs=1,
+    per_device_train_batch_size=2,
+    logging_steps=logging_steps,
+    save_steps=1000,  # Guardar el modelo cada 1000 pasos para reducir la frecuencia de escritura en disco
+    learning_rate=2e-4,
+    weight_decay=0.001,
+    lr_scheduler_type="cosine",
+    warmup_ratio=0.03,
+    report_to="tensorboard",
+    fp16=False  # Desactivar la precisión mixta para simplificar el entrenamiento
+)
+
+model = AutoModelForCausalLM.from_pretrained(model_name, token=token_name)
+
+data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
+
+trainer = Trainer(
+    model=model,
+    args=training_arguments,
+    train_dataset=train_dataset,
+    eval_dataset=val_dataset,
+    data_collator=data_collator
+)
+
+trainer.train()
+
+model.save_pretrained(new_model)
+
+model.push_to_hub("eibeel/llama3-python-TFG")
+
+
 # Commented out IPython magic to ensure Python compatibility.
 #  %load_ext tensorboard
 #  %tensorboard --logdir results/runs