Update LLAMA_Fine-Tuning.py

LLAMA_Fine-Tuning.py

@@ -90,7 +90,7 @@ per_device_train_batch_size = 4
 per_device_eval_batch_size = 4
 
 # Number of update steps to accumulate the gradients for
-gradient_accumulation_steps = 1
+gradient_accumulation_steps = 2
 
 # Enable gradient checkpointing
 gradient_checkpointing = True
@@ -167,62 +167,7 @@ training_arguments = TrainingArguments(
     evaluation_strategy="steps",  # Evaluar basado en el número de pasos
     eval_steps=500,  # Evaluar cada 500 pasos
     num_train_epochs=1,
-    per_device_train_batch_size=2,
-    logging_steps=logging_steps,
-    save_steps=1000,  # Guardar el modelo cada 1000 pasos para reducir la frecuencia de escritura en disco
-    learning_rate=2e-4,
-    weight_decay=0.001,
-    lr_scheduler_type="cosine",
-    warmup_ratio=0.03,
-    report_to="tensorboard",
-    fp16=False  # Desactivar la precisión mixta para simplificar el entrenamiento
-)
-
-model = AutoModelForCausalLM.from_pretrained(model_name, token=token_name)
-
-data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
-
-trainer = Trainer(
-    model=model,
-    args=training_arguments,
-    train_dataset=train_dataset,
-    eval_dataset=val_dataset,
-    data_collator=data_collator
-)
-
-trainer.train()
-
-model.save_pretrained(new_model)
-
-from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, DataCollatorWithPadding
-from datasets import Dataset
-
-def load_text_file(file_path):
-    with open(file_path, 'r', encoding='utf-8') as f:
-        return [line.strip() for line in f if line.strip()]
-
-train_texts = load_text_file('LLAMA_DatosEntrenamiento.txt')
-val_texts = load_text_file('LLAMA_DatosValidacion.txt')
-
-tokenizer = AutoTokenizer.from_pretrained(model_name, token=token_name)
-
-def tokenize_and_encode(texts):
-    encodings = tokenizer(texts, truncation=True, padding="longest", max_length=512, return_tensors="pt")
-    encodings['labels'] = encodings['input_ids'].clone()  # Duplicar input_ids para usar como labels
-    return encodings
-
-train_encodings = tokenize_and_encode(train_texts)
-val_encodings = tokenize_and_encode(val_texts)
-
-train_dataset = Dataset.from_dict({key: val.numpy() for key, val in train_encodings.items()})
-val_dataset = Dataset.from_dict({key: val.numpy() for key, val in val_encodings.items()})
-
-training_arguments = TrainingArguments(
-    output_dir=output_dir,
-    evaluation_strategy="steps",  # Evaluar basado en el número de pasos
-    eval_steps=500,  # Evaluar cada 500 pasos
-    num_train_epochs=1,
-    per_device_train_batch_size=2,
+    per_device_train_batch_size=4,
     logging_steps=logging_steps,
     save_steps=1000,  # Guardar el modelo cada 1000 pasos para reducir la frecuencia de escritura en disco
     learning_rate=2e-4,
@@ -254,39 +199,4 @@ model.push_to_hub("eibeel/llama3-python-TFG")
 
 # Commented out IPython magic to ensure Python compatibility.
 #  %load_ext tensorboard
-#  %tensorboard --logdir results/runs
-
-train_texts[2]
-
-# Ignore warnings
-logging.set_verbosity(logging.CRITICAL)
-
-# Run text generation pipeline with our next model
-prompt = "Como puedo encontrar trabajo de ingeniero?"
-pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=200)
-result = pipe(f"<s>[INST] {prompt} [/INST]")
-print(result[0]['generated_text'])
-
-# Empty VRAM
-del model
-del pipe
-del trainer
-import gc
-gc.collect()
-gc.collect()
-
-# Reload model in FP16 and merge it with LoRA weights
-base_model = AutoModelForCausalLM.from_pretrained(
-    model_name,
-    low_cpu_mem_usage=True,
-    return_dict=True,
-    torch_dtype=torch.float16,
-    device_map=device_map,
-)
-model = PeftModel.from_pretrained(base_model, new_model)
-model = model.merge_and_unload()
-
-# Reload tokenizer to save it
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-tokenizer.pad_token = tokenizer.eos_token
-tokenizer.padding_side = "right"
+#  %tensorboard --logdir results/runs