Update app.py

app.py

@@ -4,28 +4,23 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
 import gradio as gr
 import spaces
 
-# Load the model and tokenizer
-peft_model_id = "rootxhacker/CodeAstra-7B"
-config = PeftConfig.from_pretrained(peft_model_id)
 
-# Determine the device
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-# Load the model on the appropriate device
-model = AutoModelForCausalLM.from_pretrained(
+@spaces.GPU(duration=200)
+def get_completion(query, model, tokenizer):
+    peft_model_id = "rootxhacker/CodeAstra-7B"
+    config = PeftConfig.from_pretrained(peft_model_id)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = AutoModelForCausalLM.from_pretrained(
     config.base_model_name_or_path,
     return_dict=True,
     load_in_4bit=True,
     device_map="auto"  # This will automatically handle device placement
-)
+    )
 
-tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
+    tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
 
-# Load the Lora model
-model = PeftModel.from_pretrained(model, peft_model_id)
-
-@spaces.GPU(duration=200)
-def get_completion(query, model, tokenizer):
+    model = PeftModel.from_pretrained(model, peft_model_id)
     inputs = tokenizer(query, return_tensors="pt").to(device)  # Move inputs to the same device as the model
     outputs = model.generate(**inputs, max_new_tokens=512, do_sample=True, temperature=0.7)
     return tokenizer.decode(outputs[0], skip_special_tokens=True)