Update app.py

app.py

@@ -1,10 +1,19 @@
+import os
 import gradio as gr
 from transformers import AutoModelForCausalLM, AutoTokenizer
 import torch
 
+# Redirect HF cache to /tmp (ephemeral, unlimited)
+os.environ['HF_HOME'] = '/tmp/hf_home'
+os.environ['TRANSFORMERS_CACHE'] = '/tmp/hf_cache'
+
 model_name = "Qwen/Qwen2.5-Coder-14B-Instruct"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name, load_in_4bit=True, device_map="auto")
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    load_in_4bit=True,  # Quantization for T4 GPU (~9-10GB VRAM)
+    device_map="auto"
+)
 
 def chat(message, history):
     messages = [{"role": "user", "content": message}]
@@ -17,4 +26,5 @@ def chat(message, history):
     return history, ""
 
 demo = gr.ChatInterface(chat)
-demo.launch()
+if __name__ == "__main__":
+    demo.launch()