Update app.py

app.py

@@ -168,112 +168,55 @@
 
 
 
-# from huggingface_hub import InferenceClient
-# import gradio as gr  
-# client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.2")
-
-# def format_prompt(message, history):
-#   prompt = "<s>"
-#   for user_prompt, bot_response in history:
-#     prompt += f"[INST] {user_prompt} [/INST]"
-#     prompt += f" {bot_response}</s> "
-#   prompt += f"[INST] {message} [/INST]"
-#   return prompt
-
-# def generate(
-#     prompt, history, temperature=0.2, max_new_tokens=3000, top_p=0.95, repetition_penalty=1.0,
-# ):
-#     temperature = float(temperature)
-#     if temperature < 1e-2:
-#         temperature = 1e-2
-#     top_p = float(top_p)
-
-#     generate_kwargs = dict(
-#         temperature=temperature,
-#         max_new_tokens=max_new_tokens,
-#         top_p=top_p,
-#         repetition_penalty=repetition_penalty,
-#         do_sample=True,
-#         seed=42,
-#     )
-
-#     formatted_prompt = format_prompt(prompt, history)
-
-#     stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
-#     output = ""
-
-#     for response in stream:
-#         output += response.token.text
-#         yield output
-#     return output
-
-    
-# mychatbot = gr.Chatbot(
-#     avatar_images=["./user.png", "./bot.png"], bubble_full_width=False, show_label=False, show_copy_button=True, likeable=True,)
-
-# demo = gr.ChatInterface(fn=generate, 
-#                         chatbot=mychatbot,
-#                         title="Mistral-Chat",
-#                         retry_btn=None,
-#                         undo_btn=None
-#                        )
-
-# demo.queue().launch(show_api=False)
-
-
-
-
-
-
+from huggingface_hub import InferenceClient
 import gradio as gr  
-import boto3
-import os
-from langchain.llms import Bedrock
-from langchain.chains import ConversationChain
-from langchain.memory import ConversationBufferMemory
-from langchain.prompts.prompt import PromptTemplate
-access_key_id = os.environ['aws_access_key_id']
-secret_access_key = os.environ['aws_secret_access_key']
-client = boto3.client(service_name='bedrock-runtime',region_name='us-east-1',aws_access_key_id=access_key_id,aws_secret_access_key=secret_access_key)
-
-template = """
-        <|begin_of_text|>
-        {history}
-        <|start_header_id|>user<|end_header_id|>
-        {input}
-        <|eot_id|>
-        <|start_header_id|>assistant<|end_header_id|>
-        """
-prompt_temp = PromptTemplate(input_variables=["history", "input"], template=template)
+client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.2")
+
+def format_prompt(message, history):
+  prompt = "<s>"
+  for user_prompt, bot_response in history:
+    prompt += f"[INST] {user_prompt} [/INST]"
+    prompt += f" {bot_response}</s> "
+  prompt += f"[INST] {message} [/INST]"
+  return prompt
 
 def generate(
-    prompt_temp, temperature=0.2, max_gen_len=1024, top_p=0.95,
+    prompt, history, temperature=0.2, max_new_tokens=3000, top_p=0.95, repetition_penalty=1.0,
 ):
+    temperature = float(temperature)
+    if temperature < 1e-2:
+        temperature = 1e-2
+    top_p = float(top_p)
 
     generate_kwargs = dict(
         temperature=temperature,
-        max_gen_len=max_gen_len,
-        top_p=top_p)
-    
-    llm = Bedrock(model_id="meta.llama3-8b-instruct-v1:0",model_kwargs=generate_kwargs,client=client)
-
-    conversation = ConversationChain(
-        prompt=prompt_temp,
-        llm=llm,
-        verbose=True,
-        memory= ConversationBufferMemory(ai_prefix="AI Assistant")
+        max_new_tokens=max_new_tokens,
+        top_p=top_p,
+        repetition_penalty=repetition_penalty,
+        do_sample=True,
+        seed=42,
     )
 
-    chat_history = []
+    formatted_prompt = format_prompt(prompt, history)
 
-    #result =conversation.predict(input="Hi there!")
-    
-    result = conversation({"input": message, "history":chat_history })
-    chat_history.append((message, result['response']))
-    return result['response']
+    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+    output = ""
+
+    for response in stream:
+        output += response.token.text
+        yield output
+    return output
 
-demo=gr.ChatInterface(generate)
+    
+mychatbot = gr.Chatbot(
+    avatar_images=["./user.png", "./bot.png"], bubble_full_width=False, show_label=False, show_copy_button=True, likeable=True,)
 
+demo = gr.ChatInterface(fn=generate, 
+                        chatbot=mychatbot,
+                        title="Mistral-Chat",
+                        retry_btn=None,
+                        undo_btn=None
+                       )
 
 demo.queue().launch(show_api=False)
 
@@ -285,6 +228,9 @@ demo.queue().launch(show_api=False)
 
 
 
+
+
+
 # import gradio as gr  
 # import boto3
 # import json