Spaces:
Build error
Build error
rishabhjain16
commited on
Commit
•
ac5e76e
1
Parent(s):
e5e345f
create app.py
Browse files
app.py
ADDED
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
import torch
|
3 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
4 |
+
from transformers import StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer
|
5 |
+
from threading import Thread
|
6 |
+
|
7 |
+
# Loading the tokenizer and model from Hugging Face's model hub.
|
8 |
+
tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-v01")
|
9 |
+
model = AutoModelForCausalLM.from_pretrained("CohereForAI/c4ai-command-r-v01")
|
10 |
+
|
11 |
+
# using CUDA for an optimal experience
|
12 |
+
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
13 |
+
model = model.to(device)
|
14 |
+
|
15 |
+
|
16 |
+
# Defining a custom stopping criteria class for the model's text generation.
|
17 |
+
class StopOnTokens(StoppingCriteria):
|
18 |
+
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
|
19 |
+
stop_ids = [2] # IDs of tokens where the generation should stop.
|
20 |
+
for stop_id in stop_ids:
|
21 |
+
if input_ids[0][-1] == stop_id: # Checking if the last generated token is a stop token.
|
22 |
+
return True
|
23 |
+
return False
|
24 |
+
|
25 |
+
|
26 |
+
# Function to generate model predictions.
|
27 |
+
def predict(message, history):
|
28 |
+
history_transformer_format = history + [[message, ""]]
|
29 |
+
stop = StopOnTokens()
|
30 |
+
|
31 |
+
# Formatting the input for the model.
|
32 |
+
messages = "</s>".join(["</s>".join(["\n<|user|>:" + item[0], "\n<|assistant|>:" + item[1]])
|
33 |
+
for item in history_transformer_format])
|
34 |
+
model_inputs = tokenizer([messages], return_tensors="pt").to(device)
|
35 |
+
streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
|
36 |
+
generate_kwargs = dict(
|
37 |
+
model_inputs,
|
38 |
+
streamer=streamer,
|
39 |
+
max_new_tokens=1024,
|
40 |
+
do_sample=True,
|
41 |
+
top_p=0.95,
|
42 |
+
top_k=50,
|
43 |
+
temperature=0.7,
|
44 |
+
num_beams=1,
|
45 |
+
stopping_criteria=StoppingCriteriaList([stop])
|
46 |
+
)
|
47 |
+
t = Thread(target=model.generate, kwargs=generate_kwargs)
|
48 |
+
t.start() # Starting the generation in a separate thread.
|
49 |
+
partial_message = ""
|
50 |
+
for new_token in streamer:
|
51 |
+
partial_message += new_token
|
52 |
+
if '</s>' in partial_message: # Breaking the loop if the stop token is generated.
|
53 |
+
break
|
54 |
+
yield partial_message
|
55 |
+
|
56 |
+
|
57 |
+
# Setting up the Gradio chat interface.
|
58 |
+
gr.ChatInterface(predict,
|
59 |
+
title="Command-R test",
|
60 |
+
description="Ask Comman-R any questions",
|
61 |
+
examples=['How is bill gates?', 'Who is the president of US now?']
|
62 |
+
).launch() # Launching the web interface.
|