Spaces:
Sleeping
Sleeping
| from ctransformers import AutoModelForCausalLM | |
| from fastapi import FastAPI | |
| from pydantic import BaseModel | |
| llm = AutoModelForCausalLM.from_pretrained("zephyr-7b-beta.Q4_K_S.gguf", | |
| model_type='mistral', | |
| max_new_tokens=1096, | |
| threads=3) | |
| # Pydantic object | |
| class Validation(BaseModel): | |
| prompt: str | |
| # Fast API | |
| app = FastAPI() | |
| async def stream(item: Validation): | |
| system_prompt = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' | |
| E_INST = "</s>" | |
| user, assistant = "", "" | |
| prompt = f"{system_prompt}{E_INST}\n{user}\n{item.prompt}{E_INST}\n{assistant}\n" | |
| return await llm(prompt) # Making the call asynchronous | |