Ganesh Karbhari
commited on
Commit
•
514c544
1
Parent(s):
c9a5e5e
Update app.py
Browse files
app.py
CHANGED
@@ -168,8 +168,65 @@
|
|
168 |
|
169 |
|
170 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
171 |
from huggingface_hub import InferenceClient
|
172 |
-
import gradio as gr
|
|
|
173 |
client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.2")
|
174 |
|
175 |
def format_prompt(message, history):
|
@@ -181,7 +238,7 @@ def format_prompt(message, history):
|
|
181 |
return prompt
|
182 |
|
183 |
def generate(
|
184 |
-
prompt, history, temperature=0.
|
185 |
):
|
186 |
temperature = float(temperature)
|
187 |
if temperature < 1e-2:
|
@@ -192,9 +249,10 @@ def generate(
|
|
192 |
temperature=temperature,
|
193 |
max_new_tokens=max_new_tokens,
|
194 |
top_p=top_p,
|
195 |
-
|
196 |
-
|
197 |
-
|
|
|
198 |
)
|
199 |
|
200 |
formatted_prompt = format_prompt(prompt, history)
|
@@ -207,15 +265,45 @@ def generate(
|
|
207 |
yield output
|
208 |
return output
|
209 |
|
210 |
-
|
211 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
212 |
avatar_images=["./user.png", "./bot.png"], bubble_full_width=False, show_label=False, show_copy_button=True, likeable=True,)
|
213 |
|
214 |
demo = gr.ChatInterface(fn=generate,
|
215 |
-
chatbot=
|
216 |
-
title="Mistral-Chat
|
217 |
-
|
218 |
-
undo_btn=None
|
219 |
)
|
220 |
|
221 |
demo.queue().launch(show_api=False)
|
@@ -226,74 +314,3 @@ demo.queue().launch(show_api=False)
|
|
226 |
|
227 |
|
228 |
|
229 |
-
|
230 |
-
|
231 |
-
|
232 |
-
|
233 |
-
|
234 |
-
# import gradio as gr
|
235 |
-
# import boto3
|
236 |
-
# import json
|
237 |
-
# from botocore.exceptions import ClientError
|
238 |
-
# import os
|
239 |
-
|
240 |
-
# access_key_id = os.environ['aws_access_key_id']
|
241 |
-
# secret_access_key = os.environ['aws_secret_access_key']
|
242 |
-
|
243 |
-
|
244 |
-
# bedrock = boto3.client(service_name='bedrock-runtime',region_name='us-east-1',aws_access_key_id=access_key_id,aws_secret_access_key=secret_access_key)
|
245 |
-
|
246 |
-
|
247 |
-
# def invoke_llama3_8b(user_message):
|
248 |
-
# try:
|
249 |
-
# # Set the model ID, e.g., Llama 3 8B Instruct.
|
250 |
-
# model_id = "meta.llama3-8b-instruct-v1:0"
|
251 |
-
|
252 |
-
|
253 |
-
# # Embed the message in Llama 3's prompt format.
|
254 |
-
# prompt = f"""
|
255 |
-
# <|begin_of_text|>
|
256 |
-
# <|start_header_id|>user<|end_header_id|>
|
257 |
-
# {user_message}
|
258 |
-
# <|eot_id|>
|
259 |
-
# <|start_header_id|>assistant<|end_header_id|>
|
260 |
-
# """
|
261 |
-
|
262 |
-
# # Format the request payload using the model's native structure.
|
263 |
-
# request = {
|
264 |
-
# "prompt": prompt,
|
265 |
-
# # Optional inference parameters:
|
266 |
-
# "max_gen_len": 1024,
|
267 |
-
# "temperature": 0.6,
|
268 |
-
# "top_p": 0.9,
|
269 |
-
# }
|
270 |
-
|
271 |
-
# # Encode and send the request.
|
272 |
-
# response = bedrock.invoke_model(body=json.dumps(request), modelId=model_id)
|
273 |
-
|
274 |
-
# # Decode the native response body.
|
275 |
-
# model_response = json.loads(response["body"].read())
|
276 |
-
|
277 |
-
# # Extract and print the generated text.
|
278 |
-
# response_text = model_response["generation"]
|
279 |
-
|
280 |
-
# return response_text
|
281 |
-
|
282 |
-
# except ClientError:
|
283 |
-
# print("Couldn't invoke llama3 8B")
|
284 |
-
# raise
|
285 |
-
|
286 |
-
|
287 |
-
|
288 |
-
# mychatbot = gr.Chatbot(
|
289 |
-
# avatar_images=["./user.png", "./bot.png"], bubble_full_width=False, show_label=False, show_copy_button=True, likeable=True,)
|
290 |
-
|
291 |
-
# demo = gr.ChatInterface(fn=invoke_llama3_8b,
|
292 |
-
# chatbot=mychatbot,
|
293 |
-
# title="llama3-Chat",
|
294 |
-
# retry_btn=None,
|
295 |
-
# undo_btn=None
|
296 |
-
# )
|
297 |
-
|
298 |
-
# demo.queue().launch(show_api=False)
|
299 |
-
|
|
|
168 |
|
169 |
|
170 |
|
171 |
+
# from huggingface_hub import InferenceClient
|
172 |
+
# import gradio as gr
|
173 |
+
# client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.2")
|
174 |
+
|
175 |
+
# def format_prompt(message, history):
|
176 |
+
# prompt = "<s>"
|
177 |
+
# for user_prompt, bot_response in history:
|
178 |
+
# prompt += f"[INST] {user_prompt} [/INST]"
|
179 |
+
# prompt += f" {bot_response}</s> "
|
180 |
+
# prompt += f"[INST] {message} [/INST]"
|
181 |
+
# return prompt
|
182 |
+
|
183 |
+
# def generate(
|
184 |
+
# prompt, history, temperature=0.2, max_new_tokens=3000, top_p=0.95, repetition_penalty=1.0,
|
185 |
+
# ):
|
186 |
+
# temperature = float(temperature)
|
187 |
+
# if temperature < 1e-2:
|
188 |
+
# temperature = 1e-2
|
189 |
+
# top_p = float(top_p)
|
190 |
+
|
191 |
+
# generate_kwargs = dict(
|
192 |
+
# temperature=temperature,
|
193 |
+
# max_new_tokens=max_new_tokens,
|
194 |
+
# top_p=top_p,
|
195 |
+
# repetition_penalty=repetition_penalty,
|
196 |
+
# do_sample=True,
|
197 |
+
# seed=42,
|
198 |
+
# )
|
199 |
+
|
200 |
+
# formatted_prompt = format_prompt(prompt, history)
|
201 |
+
|
202 |
+
# stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
|
203 |
+
# output = ""
|
204 |
+
|
205 |
+
# for response in stream:
|
206 |
+
# output += response.token.text
|
207 |
+
# yield output
|
208 |
+
# return output
|
209 |
+
|
210 |
+
|
211 |
+
# mychatbot = gr.Chatbot(
|
212 |
+
# avatar_images=["./user.png", "./bot.png"], bubble_full_width=False, show_label=False, show_copy_button=True, likeable=True,)
|
213 |
+
|
214 |
+
# demo = gr.ChatInterface(fn=generate,
|
215 |
+
# chatbot=mychatbot,
|
216 |
+
# title="Mistral-Chat",
|
217 |
+
# retry_btn=None,
|
218 |
+
# undo_btn=None
|
219 |
+
# )
|
220 |
+
|
221 |
+
# demo.queue().launch(show_api=False)
|
222 |
+
|
223 |
+
|
224 |
+
|
225 |
+
|
226 |
+
|
227 |
from huggingface_hub import InferenceClient
|
228 |
+
import gradio as gr
|
229 |
+
|
230 |
client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.2")
|
231 |
|
232 |
def format_prompt(message, history):
|
|
|
238 |
return prompt
|
239 |
|
240 |
def generate(
|
241 |
+
prompt, history, temperature=0.3, max_new_tokens=3000, top_p=0.90,top_k=50
|
242 |
):
|
243 |
temperature = float(temperature)
|
244 |
if temperature < 1e-2:
|
|
|
249 |
temperature=temperature,
|
250 |
max_new_tokens=max_new_tokens,
|
251 |
top_p=top_p,
|
252 |
+
top_k=top_k,
|
253 |
+
#repetition_penalty=repetition_penalty,
|
254 |
+
#do_sample=True,
|
255 |
+
#seed=42,
|
256 |
)
|
257 |
|
258 |
formatted_prompt = format_prompt(prompt, history)
|
|
|
265 |
yield output
|
266 |
return output
|
267 |
|
268 |
+
|
269 |
+
additional_inputs=[
|
270 |
+
gr.Slider(
|
271 |
+
label="Temperature",
|
272 |
+
value=0.3,
|
273 |
+
minimum=0.0,
|
274 |
+
maximum=1.0,
|
275 |
+
step=0.1,
|
276 |
+
interactive=True,
|
277 |
+
info="Higher values generate more diverse outputs",
|
278 |
+
)
|
279 |
+
gr.Slider(
|
280 |
+
label="top_p",
|
281 |
+
value=0.9,
|
282 |
+
minimum=0.0,
|
283 |
+
maximum=1.0,
|
284 |
+
step=0.1,
|
285 |
+
interactive=True,
|
286 |
+
info="Higher values generate more diverse outputs",
|
287 |
+
)
|
288 |
+
|
289 |
+
gr.Slider(
|
290 |
+
label="max_new_tokens",
|
291 |
+
value=3000,
|
292 |
+
minimum=512,
|
293 |
+
maximum=5000,
|
294 |
+
step=100,
|
295 |
+
interactive=True,
|
296 |
+
info="Output response limit in tokens",
|
297 |
+
)
|
298 |
+
]
|
299 |
+
|
300 |
+
bbchatbot = gr.Chatbot(
|
301 |
avatar_images=["./user.png", "./bot.png"], bubble_full_width=False, show_label=False, show_copy_button=True, likeable=True,)
|
302 |
|
303 |
demo = gr.ChatInterface(fn=generate,
|
304 |
+
chatbot=bbchatbot,
|
305 |
+
title="Mistral-Chat,
|
306 |
+
additional_inputs=additional_inputs
|
|
|
307 |
)
|
308 |
|
309 |
demo.queue().launch(show_api=False)
|
|
|
314 |
|
315 |
|
316 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|