manish commited on
Commit
a6c74eb
1 Parent(s): 7222796

add custom handler and gitignore

Browse files
Files changed (1) hide show
  1. handler.py +26 -0
handler.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Dict, List, Any
2
+ from transformers import AutoTokenizer, AutoModelForCausalLM
3
+
4
+ class EndpointHandler():
5
+ def __init__(self, path=""):
6
+ # init
7
+ self.tokenizer = AutoTokenizer.from_pretrained("verseAI/vai-GPT-NeoXT-Chat-Base-20B")
8
+ self.model = AutoModelForCausalLM.from_pretrained("verseAI/vai-GPT-NeoXT-Chat-Base-20B", device_map="auto", load_in_8bit=True)
9
+
10
+ def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
11
+ """
12
+ data args:
13
+ inputs (:obj: `str`)
14
+ date (:obj: `str`)
15
+ Return:
16
+ A :obj:`list` | `dict`: will be serialized and returned
17
+
18
+ from transformers import AutoTokenizer, AutoModelForCausalLM
19
+ """
20
+ # infer
21
+ inputs = self.tokenizer("<human>: Hello!\n<bot>:", return_tensors='pt').to(self.model.device)
22
+ outputs = self.model.generate(**inputs, max_new_tokens=10, do_sample=True, temperature=0.8)
23
+ output_str = self.tokenizer.decode(outputs[0])
24
+ print(output_str)
25
+ return output_str
26
+