Create app.py

app.py

@@ -0,0 +1,117 @@
+from pydantic import BaseModel
+from llama_cpp import Llama
+import os
+import gradio as gr
+from dotenv import load_dotenv
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse, StreamingResponse
+import spaces
+import asyncio
+import random
+from transformers import pipeline
+from io import BytesIO
+import requests
+
+app = FastAPI()
+load_dotenv()
+
+HUGGINGFACE_TOKEN = os.getenv("HUGGINGFACE_TOKEN")
+
+summarizer = pipeline("summarization")
+
+class ModelManager:
+    def __init__(self):
+        self.unified_model = self.load_unified_model()
+        self.params = {
+            "n_ctx": 2048,
+            "n_batch": 512,
+            "n_predict": 512,
+            "repeat_penalty": 1.1,
+            "n_threads": int(os.cpu_count() * 0.75),
+            "seed": -1,
+            "stop": ["</s>"],
+            "tokens": [],
+            "eos_token_id": None,
+            "pad_token_id": None,
+        }
+
+    def load_unified_model(self):
+        model_configs = [
+            {"repo_id": "Ffftdtd5dtft/gpt2-xl-Q2_K-GGUF", "filename": "gpt2-xl-q2_k.gguf", "name": "GPT-2 XL"},
+            {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-8B-Instruct-Q2_K-GGUF", "filename": "meta-llama-3.1-8b-instruct-q2_k.gguf", "name": "Meta Llama 3.1-8B Instruct"},
+            {"repo_id": "Ffftdtd5dtft/gemma-2-9b-it-Q2_K-GGUF", "filename": "gemma-2-9b-it-q2_k.gguf", "name": "Gemma 2-9B IT"},
+            {"repo_id": "Ffftdtd5dtft/gemma-2-27b-Q2_K-GGUF", "filename": "gemma-2-27b-q2_k.gguf", "name": "Gemma 2-27B"},
+            {"repo_id": "Ffftdtd5dtft/Phi-3-mini-128k-instruct-Q2_K-GGUF", "filename": "phi-3-mini-128k-instruct-q2_k.gguf", "name": "Phi-3 Mini 128K Instruct"},
+            {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-8B-Q2_K-GGUF", "filename": "meta-llama-3.1-8b-q2_k.gguf", "name": "Meta Llama 3.1-8B"},
+            {"repo_id": "Ffftdtd5dtft/Qwen2-7B-Instruct-Q2_K-GGUF", "filename": "qwen2-7b-instruct-q2_k.gguf", "name": "Qwen2 7B Instruct"},
+            {"repo_id": "Ffftdtd5dtft/starcoder2-3b-Q2_K-GGUF", "filename": "starcoder2-3b-q2_k.gguf", "name": "Starcoder2 3B"},
+            {"repo_id": "Ffftdtd5dtft/Qwen2-1.5B-Instruct-Q2_K-GGUF", "filename": "qwen2-1.5b-instruct-q2_k.gguf", "name": "Qwen2 1.5B Instruct"},
+            {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-70B-Q2_K-GGUF", "filename": "meta-llama-3.1-70b-q2_k.gguf", "name": "Meta Llama 3.1-70B"},
+            {"repo_id": "Ffftdtd5dtft/Mistral-Nemo-Instruct-2407-Q2_K-GGUF", "filename": "mistral-nemo-instruct-2407-q2_k.gguf", "name": "Mistral Nemo Instruct 2407"},
+            {"repo_id": "Ffftdtd5dtft/Hermes-3-Llama-3.1-8B-IQ1_S-GGUF", "filename": "hermes-3-llama-3.1-8b-iq1_s-imat.gguf", "name": "Hermes 3 Llama 3.1-8B"},
+            {"repo_id": "Ffftdtd5dtft/Phi-3.5-mini-instruct-Q2_K-GGUF", "filename": "phi-3.5-mini-instruct-q2_k.gguf", "name": "Phi 3.5 Mini Instruct"},
+            {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-70B-Instruct-Q2_K-GGUF", "filename": "meta-llama-3.1-70b-instruct-q2_k.gguf", "name": "Meta Llama 3.1-70B Instruct"},
+            {"repo_id": "Ffftdtd5dtft/codegemma-2b-IQ1_S-GGUF", "filename": "codegemma-2b-iq1_s-imat.gguf", "name": "Codegemma 2B"},
+            {"repo_id": "Ffftdtd5dtft/Phi-3-mini-128k-instruct-IQ2_XXS-GGUF", "filename": "phi-3-mini-128k-instruct-iq2_xxs-imat.gguf", "name": "Phi 3 Mini 128K Instruct XXS"},
+            {"repo_id": "Ffftdtd5dtft/TinyLlama-1.1B-Chat-v1.0-IQ1_S-GGUF", "filename": "tinyllama-1.1b-chat-v1.0-iq1_s-imat.gguf", "name": "TinyLlama 1.1B Chat"},
+            {"repo_id": "Ffftdtd5dtft/Mistral-NeMo-Minitron-8B-Base-IQ1_S-GGUF", "filename": "mistral-nemo-minitron-8b-base-iq1_s-imat.gguf", "name": "Mistral NeMo Minitron 8B Base"},
+            {"repo_id": "eren23/DistiLabelOrca-TinyLLama-1.1B", "filename": "distilabelorca-tinyllama-1.1b.gguf", "name": "DistiLabelOrca-TinyLLama-1.1B"},
+            {"repo_id": "DevQuasar/Llama-3.2-1B-GGUF", "filename": "Llama-3.2-1B.Q2_K.gguf", "name": "Llama-3.2-1B-GGUF"},
+        ]
+
+        models = []
+        for config in model_configs:
+            model_data = BytesIO(requests.get(f"https://huggingface.co/{config['repo_id']}/resolve/main/{config['filename']}", headers={"Authorization": f"Bearer {HUGGINGFACE_TOKEN}"}).content)
+            model = Llama.from_pretrained(model=model_data, **self.params)
+            models.append(model)
+
+        self.params["tokens"] = models[0].tokenize(b"Hello")
+        self.params["eos_token_id"] = models[0].eos_token_id
+        self.params["pad_token_id"] = models[0].pad_token_id
+        return models[0]
+
+model_manager = ModelManager()
+
+class ChatRequest(BaseModel):
+    message: str
+
+@spaces.GPU()
+async def generate_streaming_response(inputs):
+    top_p = round(random.uniform(0.01, 1.00), 2)
+    top_k = random.randint(1, 100)
+    temperature = round(random.uniform(0.01, 2.00), 2)
+    max_tokens = model_manager.params["n_ctx"] - len(model_manager.unified_model.tokenize(inputs))
+
+    async def stream_response():
+        response = await asyncio.to_thread(model_manager.unified_model, inputs, top_p=top_p, top_k=top_k, temperature=temperature, max_tokens=max_tokens, **model_manager.params)
+        full_text = response['choices'][0]['text']
+
+        if len(full_text) > max_tokens:
+            chunks = [full_text[i:i + max_tokens] for i in range(0, len(full_text), max_tokens)]
+            for chunk in chunks:
+                yield chunk
+        else:
+            yield full_text
+
+    return StreamingResponse(stream_response())
+
+async def process_message(message):
+    inputs = message.strip()
+    return await generate_streaming_response(inputs)
+
+@app.post("/generate_multimodel")
+async def api_generate_multimodel(request: Request):
+    data = await request.json()
+    message = data["message"]
+    return await process_message(message)
+
+iface = gr.Interface(
+    fn=process_message,
+    inputs=gr.Textbox(lines=2, placeholder="Enter your message here..."),
+    outputs=gr.Markdown(stream=True),
+    title="Unified Multi-Model API",
+    description="Enter a message to get responses from a unified model."
+)
+
+if __name__ == "__main__":
+    iface.launch()