awacke1 commited on
Commit
1a41583
1 Parent(s): 0163a8b

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +52 -0
app.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from huggingface_hub import InferenceClient
2
+ import gradio as gr
3
+
4
+ client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
5
+
6
+ def format_prompt(message, history):
7
+ prompt = "<s>"
8
+ for user_prompt, bot_response in history:
9
+ prompt += f"[INST] {user_prompt} [/INST]"
10
+ prompt += f" {bot_response}</s> "
11
+ prompt += f"[INST] {message} [/INST]"
12
+ return prompt
13
+
14
+ def generate(
15
+ prompt, history, temperature=0.2, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0,
16
+ ):
17
+ temperature = float(temperature)
18
+ if temperature < 1e-2:
19
+ temperature = 1e-2
20
+ top_p = float(top_p)
21
+
22
+ generate_kwargs = dict(
23
+ temperature=temperature,
24
+ max_new_tokens=max_new_tokens,
25
+ top_p=top_p,
26
+ repetition_penalty=repetition_penalty,
27
+ do_sample=True,
28
+ seed=42,
29
+ )
30
+
31
+ formatted_prompt = format_prompt(prompt, history)
32
+
33
+ stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
34
+ output = ""
35
+
36
+ for response in stream:
37
+ output += response.token.text
38
+ yield output
39
+ return output
40
+
41
+
42
+ mychatbot = gr.Chatbot(
43
+ avatar_images=["./user.png", "./botm.png"], bubble_full_width=False, show_label=False, show_copy_button=True, likeable=True,)
44
+
45
+ demo = gr.ChatInterface(fn=generate,
46
+ chatbot=mychatbot,
47
+ title="Tomoniai's Mixtral 8x7b Chat",
48
+ retry_btn=None,
49
+ undo_btn=None
50
+ )
51
+
52
+ demo.queue().launch(show_api=False)