Spaces:
Runtime error
Runtime error
File size: 638 Bytes
7efc171 83982dc 7efc171 278e439 7efc171 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 |
from transformers import AutoTokenizer, LlamaForCausalLM
from fastapi import FastAPI
from pydantic import BaseModel
model_path = 'EleutherAI/llemma_7b'
model = LlamaForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)
class validation(BaseModel):
prompt: str
app = FastAPI()
@app.post("/prompt")
async def stream(item: validation):
inputs = tokenizer(item.prompt, return_tensors="pt")
generate_ids = model.generate(inputs.input_ids, max_length=30)
var = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
return var
|