Update README.md

README.md

@@ -67,20 +67,35 @@ This evaluation provides a solid foundation for future refinements and improved
 ## How to Use
 
 ```python
-!pip install bitsandbytes
+!pip install bitsandbytes peft 
+
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
 
-# **Model Name on Hugging Face**
+# ✅ Model Name (Your Uploaded Hugging Face Model)
 MODEL_NAME = "Vijayendra/DeepSeek-Qwen2.5-14B-DeepThinker-v2"
 
-# 🛠 **Load Model & Tokenizer from Hugging Face**
+# ✅ 4-bit Quantization Config
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",  # You can also try "fp4"
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_compute_dtype=torch.float32
+)
+
+# ✅ Load Tokenizer
 tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+# ✅ Load Quantized Model
 model = AutoModelForCausalLM.from_pretrained(
-    MODEL_NAME, 
-    device_map="auto",  # Automatically assigns model layers to available GPUs/CPUs
-    torch_dtype=torch.float16  # Use 16-bit precision for memory efficiency
-).to("cuda" if torch.cuda.is_available() else "cpu")  # Send model to GPU if available
+    MODEL_NAME,
+    device_map="auto",
+    quantization_config=bnb_config,
+    torch_dtype=torch.float16  # Optional but helps with memory
+)
+
+print("\n🚀 Model successfully loaded across GPUs! 🚀")
+print(model.hf_device_map)
 
 # 🛠 **Define Inference Function**
 def generate_response(model, tokenizer, prompt, max_new_tokens=4096, temperature=0.7):
@@ -131,6 +146,7 @@ for i, question in enumerate(questions, 1):
     response = generate_response(model, tokenizer, question)
     print(f"\n🟢 Question {i}: {question}")
     print(f"🔵 Response: {response}")
+
 ```