Spaces:
Sleeping
Sleeping
File size: 1,971 Bytes
d20d20b d899b2a 6f13f13 84a1e58 d899b2a d20d20b b7ebcbb d20d20b d899b2a d20d20b d899b2a d20d20b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
from fastapi import FastAPI
from pydantic import BaseModel
import requests
from ctransformers import AutoModelForCausalLM
llms = {
"tinyllama":{"name": "TheBloke/TinyLlama-1.1B-1T-OpenOrca-GGUF", "file":"tinyllama-1.1b-1t-openorca.Q4_K_M.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant <|im_end|><|im_start|>user"},
"tinyllama2":{"name": "TheBloke/TinyLlama-1.1B-1T-OpenOrca-GGUF", "file":"tinyllama-1.1b-1t-openorca.Q3_K_M.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant <|im_end|><|im_start|>user"},
"tinyllama3":{"name": "TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF", "file":"tinyllama-1.1b-chat-v0.3.Q4_K_M.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant <|im_end|><|im_start|>user"},
"tinyllama4":{"name": "TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF", "file":"tinyllama-1.1b-chat-v0.3.Q2_K.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant <|im_end|><|im_start|>user"},
"solar":{"name": "TheBloke/SOLAR-10.7B-Instruct-v1.0-GGUF", "file":"solar-10.7b-instruct-v1.0.Q4_K_M.gguf", "suffix":"\n### Assistant:\n", "prefix":"### User:\n"},
"phi2":{"name": "TheBloke/phi-2-GGUF", "file":"phi-2.Q4_K_M.gguf", "suffix":"\nOutput:\n", "prefix":"Instruct:\n"}
}
for k in llms.keys():
AutoModelForCausalLM.from_pretrained(llms[k]['name'], model_file=llms[k]['file'])
#Pydantic object
class validation(BaseModel):
prompt: str
llm: str
#Fast API
app = FastAPI()
@app.post("/llm_on_cpu")
async def stream(item: validation):
prefix=llms[item.llm]['prefix']
suffix=llms[item.llm]['suffix']
user="""
{prompt}"""
llm = AutoModelForCausalLM.from_pretrained(llms[item.llm]['name'], model_file=llms[item.llm]['file'])
prompt = f"{prefix}{user.replace('{prompt}', item.prompt)}{suffix}"
return llm(prompt) |