app.py

import os
import gc
import torch

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, BitsAndBytesConfig
from datasets import load_dataset
from peft import LoraConfig, PeftModel, get_peft_model, prepare_model_for_kbit_training
from trl import DPOTrainer
import bitsandbytes as bnb
import wandb

# Defined in the secrets tab in Google Colab
# wb_token = "2eae619e4d6f0caef6408a6dc869dd0bfa6595f6"
hf_token = os.getenv("hf_token")
wb_token = os.getenv("wb_token")
wandb.login(key=wb_token)



# Fine-tune model with DPO


import gradio as gr


def greet(traindata_,output_repo):
    model_name = "HuggingFaceH4/zephyr-7b-gemma-v0.1"
    # new_model = "Gopal2002/zehpyr-gemma-dpo-finetune"
    new_model = output_repo

    try:
        tokenizer = AutoTokenizer.from_pretrained(model_name)
        tokenizer.pad_token = tokenizer.eos_token
        tokenizer.padding_side = "left"
    
    
        model = AutoModelForCausalLM.from_pretrained(
            model_name,
            torch_dtype=torch.float16,
            load_in_4bit=True
        )
        model.config.use_cache = False
    
        # Reference model
        ref_model = AutoModelForCausalLM.from_pretrained(
            model_name,
            torch_dtype=torch.float16,
            load_in_4bit=True
        )
    
        # specify how to quantize the model
        quantization_config = BitsAndBytesConfig(
                    load_in_4bit=True,
                    bnb_4bit_quant_type="nf4",
                    bnb_4bit_compute_dtype=torch.bfloat16,
        )
        device_map = {"": torch.cuda.current_device()} if torch.cuda.is_available() else None
    
        # Step 1: load the base model (Mistral-7B in our case) in 4-bit
        model_kwargs = dict(
            # attn_implementation="flash_attention_2", # set this to True if your GPU supports it (Flash Attention drastically speeds up model computations)
            torch_dtype="auto",
            use_cache=False,  # set to False as we're going to use gradient checkpointing
            device_map=device_map,
            quantization_config=quantization_config,
        )
        model = AutoModelForCausalLM.from_pretrained(model_name, **model_kwargs)
    
    # Training arguments
        peft_config = LoraConfig(
            r=16,
            lora_alpha=16,
            lora_dropout=0.05,
            bias="none",
            task_type="CAUSAL_LM",
            target_modules=['k_proj', 'gate_proj', 'v_proj', 'up_proj', 'q_proj', 'o_proj', 'down_proj']
        )
        training_args = TrainingArguments(
            per_device_train_batch_size=4,
            gradient_accumulation_steps=4,
            gradient_checkpointing=True,
            learning_rate=5e-5,
            lr_scheduler_type="cosine",
            max_steps=200,
            save_strategy="no",
            logging_steps=1,
            output_dir=new_model,
            optim="paged_adamw_32bit",
            warmup_steps=100,
            bf16=True,
            report_to="wandb",
        )
    
    #load the dataset
        dataset = load_dataset(traindata_, split='train')
        
        # dataset = load_dataset('Gopal2002/zephyr-gemma-finetune-dpo', split='train')
    
    # Create DPO trainer
        dpo_trainer = DPOTrainer(
            model,
            ref_model=None,
            args=training_args,
            train_dataset=dataset,
            tokenizer=tokenizer,
            peft_config=peft_config,
            beta=0.1,
            max_prompt_length=2048,
            max_length=1536,
        )
        dpo_trainer.train()
        return "Training Done"
    except Exception as e:
        return str(e)


with gr.Blocks() as demo:
    traindata_ = gr.Textbox(label="Enter training data repo")
    output_repo = gr.Textbox(label="Enter output model path")

    output = gr.Textbox(label="Output Box")
    greet_btn = gr.Button("TRAIN")
    greet_btn.click(fn=greet, inputs=[traindata_,output_repo], outputs=output, api_name="greet")

demo.launch()