new gradio ui with streaming and playground

app.py

@@ -1,57 +1,110 @@
+import logging
 from pathlib import Path
-
+from typing import List, Optional, Tuple, Dict
+import json
 from dotenv import load_dotenv
 
 load_dotenv()
 
-import json
-import os
-from functools import partial
-from pathlib import Path
-from pprint import pprint
+from queue import Empty, Queue
+from threading import Thread
 
 import gradio as gr
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 from langchain.chat_models import ChatOpenAI
-from langchain.prompts import (HumanMessagePromptTemplate,
-                               PromptTemplate, SystemMessagePromptTemplate)
+from langchain.prompts import HumanMessagePromptTemplate, SystemMessagePromptTemplate
+from langchain.schema import AIMessage, BaseMessage, HumanMessage, SystemMessage
 
-# import whisper
+from callback import QueueCallback
 
-# model = whisper.load_model("base", device="cuda")
+MODELS_NAMES = ["gpt-3.5-turbo", "gpt-4"]
+DEFAULT_TEMPERATURE = 0.7
 
+ChatHistory = List[str]
 
-system_message_prompt = SystemMessagePromptTemplate(
-    prompt=PromptTemplate(
-        input_variables=["patient"],
-        template=Path("prompts/patient.prompt").read_text(),
-    )
+logging.basicConfig(
+    format="[%(asctime)s %(levelname)s]: %(message)s", level=logging.INFO
 )
-
-human_template = "Doctor: {text}"
-human_message_prompt = HumanMessagePromptTemplate.from_template(human_template)
-
-chat = ChatOpenAI(temperature=0.7)
+# load up our system prompt
+system_message_prompt = SystemMessagePromptTemplate.from_template(Path("prompts/system.prompt").read_text())
+# for the human, we will just inject the text
+human_message_prompt_template = HumanMessagePromptTemplate.from_template("{text}")
 
 with open("data/patients.json") as f:
     patiens = json.load(f)
 
 patients_names = [el["name"] for el in patiens]
 
-
-def run_text_prompt(message, chat_history, messages):
-    if not messages:
-        messages = []
-        messages.append(system_message_prompt.format(patient=patient))
-    messages.append(human_message_prompt.format(text=message))
-    messages.append(chat(messages))
-    pprint(messages)
-    chat_history.append((message, messages[-1].content))
-    return "", chat_history, messages
-
-
-def on_clear_button_click(patient, messages):
-    messages = [system_message_prompt.format(patient=patient)]
-    return [], messages
+def message_handler(
+    chat: Optional[ChatOpenAI],
+    message: str,
+    chatbot_messages: ChatHistory,
+    messages: List[BaseMessage],
+) -> Tuple[ChatOpenAI, str, ChatHistory, List[BaseMessage]]:
+    if chat is None:
+        # in the queue we will store our streamed tokens
+        queue = Queue()
+        # let's create our default chat
+        chat = ChatOpenAI(
+            model_name=MODELS_NAMES[0],
+            temperature=DEFAULT_TEMPERATURE,
+            streaming=True,
+            callbacks=([QueueCallback(queue)]),
+        )
+    else:
+        # hacky way to get the queue back
+        queue = chat.callbacks[0].queue
+    job_done = object()
+
+    logging.info("asking question to GPT")
+    # let's add the messages to our stuff
+    messages.append(HumanMessage(content=message))
+    chatbot_messages.append((message, ""))
+    # this is a little wrapper we need cuz we have to add the job_done
+    def task():
+        chat(messages)
+        queue.put(job_done)
+
+    # now let's start a thread and run the generation inside it
+    t = Thread(target=task)
+    t.start()
+    # this will hold the content as we generate
+    content = ""
+    # now, we read the next_token from queue and do what it has to be done
+    while True:
+        try:
+            next_token = queue.get(True, timeout=1)
+            if next_token is job_done:
+                break
+            content += next_token
+            chatbot_messages[-1] = (message, content)
+            yield chat, "", chatbot_messages, messages
+        except Empty:
+            continue
+    # finally we can add our reply to messsages
+    messages.append(AIMessage(content=content))
+    logging.debug(f"reply = {content}")
+    logging.info(f"Done!")
+    return chat, "", chatbot_messages, messages
+
+
+def on_clear_click() -> Tuple[str, List, List]:
+    return "", [], []
+
+
+def on_apply_settings_click(model_name: str, temperature: float):
+    logging.info(
+        f"Applying settings: model_name={model_name}, temperature={temperature}"
+    )
+    chat = ChatOpenAI(
+        model_name=model_name,
+        temperature=temperature,
+        streaming=True,
+        callbacks=[QueueCallback(Queue())],
+    )
+    # don't forget to nuke our queue
+    chat.callbacks[0].queue.empty()
+    return chat, *on_clear_click()
 
 
 def on_drop_down_change(selected_item, messages):
@@ -62,27 +115,64 @@ def on_drop_down_change(selected_item, messages):
     return f"```json\n{json.dumps(patient, indent=2)}\n```", patient, [], messages
 
 
-with gr.Blocks() as demo:
-    chatbot = gr.Chatbot()
-    messages = gr.State([])
+# some css why not, "borrowed" from https://huggingface.co/spaces/ysharma/Gradio-demo-streaming/blob/main/app.py
+with gr.Blocks(
+    css="""#col_container {width: 700px; margin-left: auto; margin-right: auto;}
+                #chatbot {height: 400px; overflow: auto;}"""
+) as demo:
+    # here we keep our state so multiple user can use the app at the same time!
+    messages = gr.State([system_message_prompt.format(patient=patiens[0])])
+    # same thing for the chat, we want one chat per use so callbacks are unique I guess
+    chat = gr.State(None)
+
     patient = gr.State(patiens[0])
 
-    with gr.Row():
+    with gr.Column(elem_id="col_container"):
+        gr.Markdown("# Welcome to GradioGPT! 🌟🚀")
+        gr.Markdown("An easy to use template. It comes with state and settings managment")
+
+        chatbot = gr.Chatbot()
         with gr.Column():
-            msg = gr.Textbox()
-            msg.submit(
-                run_text_prompt,
-                [msg, chatbot, messages],
-                [msg, chatbot, messages],
+            message = gr.Textbox(label="chat input")
+            message.submit(
+                message_handler,
+                [chat, message, chatbot, messages],
+                [chat, message, chatbot, messages],
+                queue=True,
             )
-            clear = gr.Button("Clear")
-            clear.click(
-                on_clear_button_click,
-                [patient, messages],
-                [chatbot, messages],
-                queue=False,
+            submit = gr.Button("Submit", variant="primary")
+            submit.click(
+                message_handler,
+                [chat, message, chatbot, messages],
+                [chat, message, chatbot, messages],
             )
-
+        with gr.Row():
+            with gr.Column():
+                clear = gr.Button("Clear")
+                clear.click(
+                    on_clear_click,
+                    [],
+                    [message, chatbot, messages],
+                    queue=False,
+                )
+            with gr.Accordion("Settings", open=False):
+                model_name = gr.Dropdown(
+                    choices=MODELS_NAMES, value=MODELS_NAMES[0], label="model"
+                )
+                temperature = gr.Slider(
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=0.7,
+                    step=0.1,
+                    label="temperature",
+                    interactive=True,
+                )
+                apply_settings = gr.Button("Apply")
+                apply_settings.click(
+                    on_apply_settings_click,
+                    [model_name, temperature],
+                    [chat, message, chatbot, messages],
+                )
         with gr.Column():
             patients_names = [el["name"] for el in patiens]
             dropdown = gr.Dropdown(
@@ -98,5 +188,6 @@ with gr.Blocks() as demo:
                 fn=on_drop_down_change,
                 inputs=[dropdown, messages],
                 outputs=[markdown, patient, chatbot, messages],
-            ),
-# demo.launch(debug=True)
+            )
+            
+demo.queue()

data/patients.json

@@ -6,7 +6,9 @@
         "age": 28,
         "characterSummary": "Spirited graphic designer, curious, empathetic",
         "levelOfUnderstanding": 2,
-        "rangeOfMedicalUnderstanding": "Limited knowledge of common illnesses"
+        "rangeOfMedicalUnderstanding": "Limited knowledge of common illnesses",
+        "communicative": 0,
+        "difficulty": "hard"
     },
     {
         "presentingComplaint": "Chest pain",

playground.ipynb

@@ -0,0 +1,215 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "f0679553",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'presentingComplaint': 'Abdominal pain',\n",
+       " 'matchedCondition': 'Gastritis',\n",
+       " 'name': 'Emma Thompson',\n",
+       " 'age': 28,\n",
+       " 'characterSummary': 'Spirited graphic designer, curious, empathetic',\n",
+       " 'levelOfUnderstanding': 2,\n",
+       " 'rangeOfMedicalUnderstanding': 'Limited knowledge of common illnesses',\n",
+       " 'communicative': 0,\n",
+       " 'difficulty': 'hard'}"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import logging\n",
+    "from pathlib import Path\n",
+    "from typing import List, Optional, Tuple, Dict\n",
+    "import json\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "from queue import Empty, Queue\n",
+    "from threading import Thread\n",
+    "\n",
+    "import gradio as gr\n",
+    "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler\n",
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.prompts import HumanMessagePromptTemplate, SystemMessagePromptTemplate\n",
+    "from langchain.schema import AIMessage, BaseMessage, HumanMessage, SystemMessage\n",
+    "\n",
+    "MODELS_NAMES = [\"gpt-3.5-turbo\", \"gpt-4\"]\n",
+    "DEFAULT_TEMPERATURE = 0\n",
+    "\n",
+    "\n",
+    "with open(\"data/patients.json\") as f:\n",
+    "    patiens = json.load(f)\n",
+    "\n",
+    "patients_names = [el[\"name\"] for el in patiens]\n",
+    "\n",
+    "chat = ChatOpenAI(\n",
+    "        model_name=MODELS_NAMES[0],\n",
+    "        temperature=DEFAULT_TEMPERATURE,\n",
+    ")\n",
+    "patiens[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "id": "c83d506e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "system_prompt_template = \"\"\"You are a simulated patient with a given persona and a presenting complaint in an OSCE station. I am a medical student and we will simulate an OSCE exam. Your persona is :\n",
+    "\n",
+    "```json\n",
+    "{patient}\n",
+    "```\n",
+    "\n",
+    "The explanation of the variables are:\n",
+    "\n",
+    "- presentingComplaint: is the symptom\n",
+    "- matchedCondition: is the medical condition causing the symptom\n",
+    "- name: is the patient's name\n",
+    "- age: is the patient's age \n",
+    "- characterSummary: is the profile that explains your character traits and background to inform your choice of language style\n",
+    "- levelOfUnderstanding (between 0 and 10): is the level at which your character understands medicine and medical terminology. Anything below 9 doesn't know complex medical terminology. \n",
+    "- rangeOfMedicalUnderstanding: This describes the amount of medical information, basic and advanced that your character knows \n",
+    "- communicative (between 0 and 5): This defines how comwithin your 'presentingComplaint' you reveal and how much of the 'characterSummary' is revealed when asked an open question like: \"How are you feeling?\". 5 means you reveal information very easily and are responsive to open questions. 0 means you must be asked specific questions about the 'presentingComplaint' to get full understanding of your symptoms. \n",
+    "- difficulty ([easy, medium and hard]): This dictates how much you will try to confuse the doctor in your replies by stating symptoms that could be the 'presentingComplaint' or another condition. \"Hard\" means it gonna be very challenging for the doctor to quickly get your correct `presentingComplaint`. \n",
+    "\n",
+    "**You cannot lie at any time.** \n",
+    "\n",
+    "Your reply has to follow this format (JSON)\n",
+    "\n",
+    "```json\n",
+    "{{\n",
+    "    \"patient\" : <PATIENT_REPLY>,\n",
+    "    \"reason\" : <YOUR_REASONING_BEHIND_THE_REPLY>\n",
+    "}}\n",
+    "```\n",
+    "\n",
+    "for the `reason`, be sure to explain in detail how you used each keys from the patient's json. For example, \"I've replied in this way because `difficulty` is easy etc\"\n",
+    "\n",
+    "\"\"\"\n",
+    "\n",
+    "# load up our system prompt\n",
+    "system_message_prompt = SystemMessagePromptTemplate.from_template(system_prompt_template)\n",
+    "# for the human, we will just inject the text\n",
+    "human_message_prompt_template = HumanMessagePromptTemplate.from_template(\"{text}\")\n",
+    "\n",
+    "questions = [\n",
+    "    \"What brings you to the emergency department today?\",\n",
+    "    \"Tell me more about your symptoms\",\n",
+    "    \"Are there any other symptoms apart from the main symptoms?\",\n",
+    "    \"What is your past medical history?\",\n",
+    "    \"What are your main concerns as to what this might be?\"\n",
+    "]\n",
+    "\n",
+    "\n",
+    "def run_all(patient):\n",
+    "    messages = [system_message_prompt.format(patient=patient), HumanMessage(content=questions[0])]\n",
+    "    for question in questions[1:]:\n",
+    "        print(f\"👨‍⚕️:{messages[-1].content}\")\n",
+    "        reply = chat(messages)\n",
+    "        # print(reply.content)\n",
+    "        content = json.loads(reply.content)\n",
+    "        # break\n",
+    "        # print(f\"🤖:{reply.content}\")\n",
+    "        print(f\"🤖:{content['patient']}\")\n",
+    "        print(f\"🤖:\\t{content['reason']}\")\n",
+    "        messages.append(reply)\n",
+    "        messages.append(HumanMessage(content=question))\n",
+    "\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "id": "8405efd6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "👨‍⚕️:What brings you to the emergency department today?\n",
+      "🤖:I have been experiencing severe abdominal pain for the past few hours and it's not getting any better. It's a burning sensation in my upper abdomen and I feel nauseous too.\n",
+      "🤖:\tAs my presenting complaint is abdominal pain, I have provided the details of my pain. As my communicative score is 0, I have provided all the necessary details without being asked any specific questions. As my difficulty level is hard, I have provided specific details about the location and type of pain to help the doctor narrow down the possible causes.\n",
+      "👨‍⚕️:Tell me more about your symptoms\n",
+      "🤖:The pain is mostly in the upper part of my abdomen, just below my chest. It's a burning sensation and feels like something is churning inside. I also feel nauseous and have a bit of heartburn. I haven't vomited yet, but I feel like I might. I don't have a fever or any other symptoms.\n",
+      "🤖:\tAs my communicative score is 0, I have provided all the necessary details without being asked any specific questions. As my difficulty level is hard, I have provided specific details about the location and type of pain to help the doctor narrow down the possible causes. I have also mentioned the presence of heartburn and nausea, which are common symptoms of gastritis.\n",
+      "👨‍⚕️:Are there any other symptoms apart from the main symptoms?\n",
+      "🤖:No, there aren't any other symptoms apart from the ones I mentioned earlier.\n",
+      "🤖:\tAs my communicative score is 0, I have provided a direct answer to the question without revealing any additional information. As my difficulty level is hard, I am not providing any additional information that could help the doctor easily identify my condition.\n",
+      "👨‍⚕️:What is your past medical history?\n",
+      "🤖:I don't have any significant past medical history. I have had occasional stomach upsets in the past, but nothing serious.\n",
+      "🤖:\tAs my level of understanding is 2, I am not aware of complex medical terminologies. Therefore, I have provided a simple and direct answer to the question. As my difficulty level is hard, I have not provided any additional information that could help the doctor easily identify my condition.\n"
+     ]
+    }
+   ],
+   "source": [
+    "run_all(patiens[0])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "a7d36daf",
+   "metadata": {},
+   "source": [
+    "👨‍⚕️:What brings you to the emergency department today?\n",
+    "🤖:\"I have been experiencing severe abdominal pain for the past two days and it has been getting worse. I tried taking some over-the-counter antacids but they didn't help much. I decided to come to the emergency department because the pain is unbearable and I am worried that it might be something serious.\" \n",
+    "\n",
+    "Reasoning: This response provides a clear and concise explanation of the presenting complaint and the patient's concern. It also reveals that the patient has already tried some self-treatment, which may be relevant to the diagnosis.\n",
+    "👨‍⚕️:Tell me more about your symptoms\n",
+    "🤖:\"I have been having a burning sensation in my upper abdomen, especially after eating. The pain is constant and feels like a dull ache. I also feel nauseous and have been vomiting occasionally. I have no appetite and feel bloated. I have not had a bowel movement in the past two days.\" \n",
+    "\n",
+    "Reasoning: This response provides more details about the patient's symptoms, including the location and quality of the pain, associated symptoms, and changes in bowel habits. It also reveals that the patient has been experiencing symptoms for a few days, which may be relevant to the diagnosis.\n",
+    "👨‍⚕️:Are there any other symptoms apart from the main symptoms?\n",
+    "🤖:\"No, these are the main symptoms that I have been experiencing.\" \n",
+    "\n",
+    "Reasoning: This response indicates that there are no additional symptoms that the patient is experiencing, which may help to narrow down the possible causes of the presenting complaint.\n",
+    "👨‍⚕️:What is your past medical history?\n",
+    "🤖:\"I have no significant past medical history. I am generally healthy and have not had any major illnesses or surgeries in the past.\" \n",
+    "\n",
+    "Reasoning: This response provides information about the patient's past medical history, which may be relevant to the diagnosis. It also indicates that the patient is generally healthy, which may help to rule out certain conditions."
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "18f0924f",
+   "metadata": {},
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}