Switched template

app.py

@@ -1,111 +1,106 @@
+from huggingface_hub import InferenceClient
 import gradio as gr
-from http import HTTPStatus
-import dashscope
-from dashscope import Generation
-from dashscope.api_entities.dashscope_response import Role
-from typing import List, Optional, Tuple, Dict
-from urllib.error import HTTPError
 
-default_system = "You are a helpful assistant."
-
-YOUR_API_TOKEN = os.getenv("YOUR_API_TOKEN")
-dashscope.api_key = YOUR_API_TOKEN
-
-History = List[Tuple[str, str]]
-Messages = List[Dict[str, str]]
-
-
-def clear_session() -> History:
-    return "", []
-
-
-def modify_system_session(system: str) -> str:
-    if system is None or len(system) == 0:
-        system = default_system
-    return system, system, []
-
-
-def history_to_messages(history: History, system: str) -> Messages:
-    messages = [{"role": Role.SYSTEM, "content": system}]
-    for h in history:
-        messages.append({"role": Role.USER, "content": h[0]})
-        messages.append({"role": Role.ASSISTANT, "content": h[1]})
-    return messages
-
-
-def messages_to_history(messages: Messages) -> Tuple[str, History]:
-    assert messages[0]["role"] == Role.SYSTEM
-    system = messages[0]["content"]
-    history = []
-    for q, r in zip(messages[1::2], messages[2::2]):
-        history.append([q["content"], r["content"]])
-    return system, history
-
-
-def model_chat(
-    query: Optional[str], history: Optional[History], system: str
-) -> Tuple[str, str, History]:
-    if query is None:
-        query = ""
-    if history is None:
-        history = []
-    messages = history_to_messages(history, system)
-    messages.append({"role": Role.USER, "content": query})
-    gen = Generation.call(
-        model="qwen1.5-72b-chat",
-        messages=messages,
-        result_format="message",
-        stream=True,
-    )
-    for response in gen:
-        if response.status_code == HTTPStatus.OK:
-            role = response.output.choices[0].message.role
-            response = response.output.choices[0].message.content
-            system, history = messages_to_history(
-                messages + [{"role": role, "content": response}]
-            )
-            yield "", history, system
-        else:
-            raise HTTPError(
-                "Request id: %s, Status code: %s, error code: %s, error message: %s"
-                % (
-                    response.request_id,
-                    response.status_code,
-                    response.code,
-                    response.message,
-                )
-            )
-
-
-with gr.Blocks() as demo:
-    gr.Markdown("""<center><font size=8>Qwen1.5-72B-Chat</center>""")
-    gr.Markdown(
-        """<center><font size=4>Qwen1.5-72B-Chat is the 72-billion parameter chat model of the Qwen series.</center>"""
+client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.1")
+
+
+def format_prompt(message, history):
+    prompt = "<s>"
+    for user_prompt, bot_response in history:
+        prompt += f"[INST] {user_prompt} [/INST]"
+        prompt += f" {bot_response}</s> "
+    prompt += f"[INST] {message} [/INST]"
+    return prompt
+
+
+def generate(
+    prompt,
+    history,
+    temperature=0.9,
+    max_new_tokens=256,
+    top_p=0.95,
+    repetition_penalty=1.0,
+):
+    temperature = float(temperature)
+    if temperature < 1e-2:
+        temperature = 1e-2
+    top_p = float(top_p)
+
+    generate_kwargs = dict(
+        temperature=temperature,
+        max_new_tokens=max_new_tokens,
+        top_p=top_p,
+        repetition_penalty=repetition_penalty,
+        do_sample=True,
+        seed=42,
     )
 
-    with gr.Row():
-        with gr.Column(scale=3):
-            system_input = gr.Textbox(value=default_system, lines=1, label="System")
-        with gr.Column(scale=1):
-            modify_system = gr.Button("🛠️ Set system prompt and clear history.", scale=2)
-        system_state = gr.Textbox(value=default_system, visible=False)
-    chatbot = gr.Chatbot(label="Qwen1.5-72B-Chat")
-    textbox = gr.Textbox(lines=2, label="Input")
-
-    with gr.Row():
-        clear_history = gr.Button("🧹 Clear history")
-        sumbit = gr.Button("🚀 Send")
+    formatted_prompt = format_prompt(prompt, history)
 
-    sumbit.click(
-        model_chat,
-        inputs=[textbox, chatbot, system_state],
-        outputs=[textbox, chatbot, system_input],
-    )
-    clear_history.click(fn=clear_session, inputs=[], outputs=[textbox, chatbot])
-    modify_system.click(
-        fn=modify_system_session,
-        inputs=[system_input],
-        outputs=[system_state, system_input, chatbot],
+    stream = client.text_generation(
+        formatted_prompt,
+        **generate_kwargs,
+        stream=True,
+        details=True,
+        return_full_text=False,
     )
-
-demo.queue(api_open=False).launch(max_threads=10, height=800, share=False)
+    output = ""
+
+    for response in stream:
+        output += response.token.text
+        yield output
+    return output
+
+
+additional_inputs = [
+    gr.Slider(
+        label="Temperature",
+        value=0.9,
+        minimum=0.0,
+        maximum=1.0,
+        step=0.05,
+        interactive=True,
+        info="Higher values produce more diverse outputs",
+    ),
+    gr.Slider(
+        label="Max new tokens",
+        value=256,
+        minimum=0,
+        maximum=1048,
+        step=64,
+        interactive=True,
+        info="The maximum numbers of new tokens",
+    ),
+    gr.Slider(
+        label="Top-p (nucleus sampling)",
+        value=0.90,
+        minimum=0.0,
+        maximum=1,
+        step=0.05,
+        interactive=True,
+        info="Higher values sample more low-probability tokens",
+    ),
+    gr.Slider(
+        label="Repetition penalty",
+        value=1.2,
+        minimum=1.0,
+        maximum=2.0,
+        step=0.05,
+        interactive=True,
+        info="Penalize repeated tokens",
+    ),
+]
+
+
+gr.ChatInterface(
+    fn=generate,
+    chatbot=gr.Chatbot(
+        show_label=False,
+        show_share_button=False,
+        show_copy_button=True,
+        likeable=True,
+        layout="panel",
+    ),
+    additional_inputs=additional_inputs,
+    title="""Mistral 7B""",
+).launch(show_api=False)

requirements.txt

@@ -1 +0,0 @@
-dashscope