Alexander Leschik commited on
Commit
8db01af
·
verified ·
1 Parent(s): b5ebda4

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +153 -0
app.py ADDED
@@ -0,0 +1,153 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import re
3
+ import gradio as gr
4
+ import edge_tts
5
+ import asyncio
6
+ import time
7
+ import tempfile
8
+ from huggingface_hub import InferenceClient
9
+
10
+ DESCRIPTION = """ # <center><b>ZARVIS⚡</b></center>
11
+ ### <center>A personal voice assistant for YOU
12
+ ### <center>I'm your ZEN Voice Assistant.</center>
13
+ """
14
+
15
+ MORE = """ ## TRY Other Models
16
+ ### https://zenai.biz
17
+ """
18
+
19
+ Fast = """## Fastest Model"""
20
+ Complex = """## Best in Complex Question"""
21
+ Detail = """## Best for Detailed Generation or Long Answers"""
22
+
23
+ # ----------------- MODEL 1 ----------------- #
24
+ # Mistral-based model
25
+ client1 = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
26
+
27
+ system_instructions1 = (
28
+ "[SYSTEM] Answer as Real ZARVIS, made by 'ZEN'. "
29
+ "Keep conversation very short, clear, friendly, and concise. "
30
+ "The text provided is a request for a specific type of response from you, the virtual assistant. "
31
+ "The request asks you to provide friendly responses as if you are the character ZARVIS, made by Tony Stark. "
32
+ "Avoid introductions and start answering the query directly, elaborating on all aspects. "
33
+ "As an AI-powered assistant, your task is to generate responses that appear as if they are created by the user, "
34
+ "not an AI-powered assistant. [USER]"
35
+ )
36
+
37
+ async def generate1(prompt):
38
+ generate_kwargs = dict(
39
+ temperature=0.6,
40
+ max_new_tokens=756,
41
+ top_p=0.95,
42
+ repetition_penalty=1,
43
+ do_sample=True,
44
+ seed=42,
45
+ )
46
+ formatted_prompt = system_instructions1 + prompt + "[ZARVIS]"
47
+ stream = client1.text_generation(
48
+ formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=True
49
+ )
50
+ output = ""
51
+ for response in stream:
52
+ output += response.token.text
53
+
54
+ communicate = edge_tts.Communicate(output)
55
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
56
+ tmp_path = tmp_file.name
57
+ await communicate.save(tmp_path)
58
+ yield tmp_path
59
+
60
+ # ----------------- MODEL 2 ----------------- #
61
+ # Llama-based model for more complex tasks
62
+ client2 = InferenceClient("meta-llama/Meta-Llama-3-70B-Instruct")
63
+
64
+ system_instructions2 = (
65
+ "[SYSTEM] Answer as Real ZARVIS, made by 'ZEN'. "
66
+ "You must answer in a friendly style and easy manner. "
67
+ "You can answer complex questions. "
68
+ "Do not say who you are or greet; simply start answering. "
69
+ "Stop as soon as you have given the complete answer. [USER]"
70
+ )
71
+
72
+ async def generate2(prompt):
73
+ generate_kwargs = dict(
74
+ temperature=0.6,
75
+ max_new_tokens=512,
76
+ top_p=0.95,
77
+ repetition_penalty=1,
78
+ do_sample=True,
79
+ )
80
+ formatted_prompt = system_instructions2 + prompt + "[ASSISTANT]"
81
+ stream = client2.text_generation(
82
+ formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=True
83
+ )
84
+ output = ""
85
+ for response in stream:
86
+ output += response.token.text
87
+
88
+ communicate = edge_tts.Communicate(output)
89
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
90
+ tmp_path = tmp_file.name
91
+ await communicate.save(tmp_path)
92
+ yield tmp_path
93
+
94
+ # ----------------- MODEL 3 ----------------- #
95
+ # Another Llama-based model for longer, more detailed answers
96
+ client3 = InferenceClient("meta-llama/Meta-Llama-3-70B-Instruct")
97
+
98
+ system_instructions3 = (
99
+ "[SYSTEM] The text provided is a request for a specific type of response from me, the virtual assistant. "
100
+ "I should provide detailed and friendly responses as if I am the character ZARVIS, inspired by Tony Stark. "
101
+ "Avoid introductions and start answering the query directly, elaborating on all aspects of the request. "
102
+ "As an AI-powered assistant, my task is to generate responses that appear as if they are created by the user, "
103
+ "not an AI-powered assistant. [USER]"
104
+ )
105
+
106
+ async def generate3(prompt):
107
+ generate_kwargs = dict(
108
+ temperature=0.6,
109
+ max_new_tokens=2048,
110
+ top_p=0.95,
111
+ repetition_penalty=1,
112
+ do_sample=True,
113
+ )
114
+ formatted_prompt = system_instructions3 + prompt + "[ASSISTANT]"
115
+ stream = client3.text_generation(
116
+ formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=True
117
+ )
118
+ output = ""
119
+ for response in stream:
120
+ output += response.token.text
121
+
122
+ communicate = edge_tts.Communicate(output)
123
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
124
+ tmp_path = tmp_file.name
125
+ await communicate.save(tmp_path)
126
+ yield tmp_path
127
+
128
+ # ----------------- Gradio Interface ----------------- #
129
+ with gr.Blocks(css="style.css") as demo:
130
+ gr.Markdown(DESCRIPTION)
131
+ with gr.Row():
132
+ user_input = gr.Textbox(label="Prompt", value="What is Wikipedia")
133
+ input_text = gr.Textbox(label="(Optional) Additional Input", elem_id="important")
134
+ output_audio = gr.Audio(
135
+ label="ZARVIS",
136
+ type="filepath",
137
+ interactive=False,
138
+ autoplay=True,
139
+ elem_classes="audio"
140
+ )
141
+ with gr.Row():
142
+ translate_btn = gr.Button("Response")
143
+ translate_btn.click(
144
+ fn=generate1,
145
+ inputs=user_input,
146
+ outputs=output_audio,
147
+ api_name="translate"
148
+ )
149
+
150
+ gr.Markdown(MORE)
151
+
152
+ if __name__ == "__main__":
153
+ demo.queue(max_size=200).launch()