Spaces:
Runtime error
Runtime error
File size: 1,742 Bytes
84ca46a 444b5a5 bb10374 84ca46a 5189bbe 84ca46a f41aa83 84ca46a ab597f1 84ca46a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 |
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from pydantic import BaseModel
from fastapi import HTTPException
class UserQuery(BaseModel):
user_query: str
class ChatBot:
def __init__(self):
self.tokenizer = None
self.model = None
def load_from_hub(self,model_id: str):
self.tokenizer = AutoTokenizer.from_pretrained(model_id,)
self.model = AutoModelForCausalLM.from_pretrained(model_id,ignore_mismatched_sizes=True)
def get_response(self,text: UserQuery) -> str:
if not self.model or not self.tokenizer:
raise HTTPException(status_code=400, detail="Model is not loaded")
inputs = self.tokenizer(text,return_tensors='pt')
outputs = self.model.generate(**inputs,
max_new_tokens = 100,
# add extra parameters if models runs successfully
)
response = self.tokenizer.decode(outputs[0],skip_special_tokens=True)
# response = self.get_clean_response(response)
return response
def get_clean_response(self,response):
if type(response) == list:
response = response[0].split("\n")
else:
response = response.split("\n")
ans = ''
cnt = 0 # to verify if we have seen Human before
for answer in response:
if answer.startswith("[|Human|]"): cnt += 1
elif answer.startswith('[|AI|]'):
answer = answer.split(' ')
ans += ' '.join(char for char in answer[1:])
ans += '\n'
elif cnt:
ans += answer + '\n'
return ans |