Update app.py

app.py

@@ -4,12 +4,13 @@ import os
 import gradio as gr
 from dotenv import load_dotenv
 from fastapi import FastAPI, Request
-from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi.responses import StreamingResponse
 import spaces
 import asyncio
 import random
 from io import BytesIO
 import requests
+import tempfile
 
 app = FastAPI()
 load_dotenv()
@@ -27,28 +28,26 @@ class ModelManager:
             "seed": -1,
             "stop": ["</s>"],
             "tokens": [],
-            "eos_token": None,
-            "pad_token": None,
         }
         self.unified_model = self.load_unified_model()
 
     def load_unified_model(self):
         model_configs = [
-            {"repo_id": "unsloth/Llama-3.2-3B-Instruct-GGUF", "filename": "Llama-3.2-3B-Instruct-Q4_K_M.gguf", "name": "Llama-3.2-3B-Instruct-GGUF"}, 
+            {"repo_id": "unsloth/Llama-3.2-3B-Instruct-GGUF", "filename": "Llama-3.2-3B-Instruct-Q4_K_M.gguf", "name": "Llama-3.2-3B-Instruct-GGUF"},
         ]
 
         models = []
         for config in model_configs:
-            model_data = BytesIO(requests.get(f"https://huggingface.co/{config['repo_id']}/resolve/main/{config['filename']}", headers={"Authorization": f"Bearer {HUGGINGFACE_TOKEN}"}).content)
-            model = Llama.from_pretrained(model=model_data, repo_id=config['repo_id'], filename=config['filename'], **self.params)
-            models.append(model)
+            with tempfile.TemporaryDirectory() as tmpdir:
+                model_path = os.path.join(tmpdir, config['filename'])
+                response = requests.get(f"https://huggingface.co/{config['repo_id']}/resolve/main/{config['filename']}", headers={"Authorization": f"Bearer {HUGGINGFACE_TOKEN}"})
+                with open(model_path, 'wb') as f:
+                    f.write(response.content)
+                model = Llama.from_pretrained(model_path, repo_id=config['repo_id'], filename=config['filename'], **self.params)
+                models.append(model)
 
         self.params["tokens"] = models[0].tokenize(b"Hello")
-        self.params["eos_token"] = models[0].eos_token
-        self.params["pad_token"] = models[0].pad_token
-
-        # Placeholder: Replace with your model combination logic
-        self.unified_model = models[0] 
+        self.unified_model = models[0]
         return self.unified_model
 
 model_manager = ModelManager()
@@ -95,4 +94,4 @@ iface = gr.Interface(
 )
 
 if __name__ == "__main__":
-    iface.launch()
+    iface.launch()