finished answer evaluation

database/{mock_qna.db → mock_qna.sqlite}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c380902975056aca9cbc32ff2948725fc9901a59ae01e2cf1634f475e1c889f
-size 8192
+oid sha256:8949591dc84ad447843c2741803c39e545dc11c6e39cefca75ab1416a6140e3a
+size 20480

notebooks/003_create_mock_qna.ipynb

@@ -36,7 +36,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "db_path = \"../database/mock_qna.db\"\n",
+    "db_path = \"../database/mock_qna.sqlite\"\n",
     "nature_of_run = \"new\" if not os.path.exists(db_path) else \"existing\"\n",
     "\n",
     "qna_path = \"../database/mock_qna_source.csv\""
@@ -61,7 +61,10 @@
    "source": [
     "qna_data = pd.read_csv( qna_path )\n",
     "qna_cols = list(qna_data.columns)\n",
-    "qna_data.shape"
+    "print(\"qna_data.shape\", qna_data.shape)\n",
+    "\n",
+    "qna_data = qna_data[ qna_data[\"question\"].notnull() ].reset_index(drop=True)\n",
+    "print(\"qna_data.shape\", qna_data.shape)"
    ]
   },
   {
@@ -124,7 +127,26 @@
     "                    )\n",
     "                \"\"\")\n",
     "    print(\"created table `qna_tbl`\")\n",
-    "    print(f\"columns for `qna_tbl` are {qna_cols_str}\")"
+    "    print(f\"columns for `qna_tbl` are {qna_cols_str}\")\n",
+    "\n",
+    "    \n",
+    "    cur.execute(f\"\"\"CREATE TABLE answer_tbl (\n",
+    "                        id, correct_answer, user_answer\n",
+    "                    )\n",
+    "                \"\"\")\n",
+    "    print(\"created table `answer_tbl`\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "31b6fe49-f55d-44f0-90c1-248158eac96c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if False:\n",
+    "    cur.execute(\"DELETE FROM answer_tbl\")\n",
+    "    con.commit()"
    ]
   },
   {

notebooks/004_qna_prompting_with_function_calling.ipynb

@@ -37,7 +37,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "db_path = \"../database/mock_qna.db\""
+    "db_path = \"../database/mock_qna.sqlite\""
    ]
   },
   {

notebooks/005_qna_prompting_with_pydantic_embeddings.ipynb

@@ -45,7 +45,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "db_path = \"../database/mock_qna.db\"\n",
+    "db_path = \"../database/mock_qna.sqlite\"\n",
     "con = sqlite3.connect(db_path)\n",
     "cur = con.cursor()"
    ]

qna_prompting.py

@@ -1,13 +1,15 @@
 import sqlite3
+import streamlit as st
 from pydantic import BaseModel, Field
 from llama_index.core.tools import FunctionTool
 
+import time
 
-db_path = "./database/mock_qna.db"
-description = """
-      Use this tool to extract the chapter information from the body of the input text, 
-      when user wants to learn more about a particular chapter and requested to be asked 
-      with a question to test his/her understanding.
+db_path = "./database/mock_qna.sqlite"
+qna_question_description = """
+      Use this tool to extract the chapter number from the body of input text, 
+      thereafter, chapter number will be used as a filtering criteria for
+      extracting the right questions set from database.
       The format of the function argument looks as follow:
         It should be in the format with `Chapter_` as prefix.
         Example 1: `Chapter_1` for first chapter
@@ -15,36 +17,68 @@ description = """
         Example 3: `Chapter_5` for fifth chapter
         Thereafter, the chapter_n argument will be passed to the function for Q&A question retrieval.
 """
+qna_answer_description = """
+      Use this tool to trigger the evaluation of user's provided input with the 
+      correct answer of the Q&A question asked. When user provides answer to the
+      question asked, they can reply in natural language or giving the alphabet
+      symbol of which selected answer they think it's most reasonable.
+      The format of the function argument `user_selected_answer` looks as follow:
+        It should be in the format with character such as A, B, C and D.
+        Example 1: User's answer is `a`, it means choice `A`.
+        Example 2: User's answer is contextually closer to 3rd answer choice, it means `C`.
+        Example 3: User says last is the answer, it means `D`.
+        Thereafter, the `user_selected_answer` argument will be passed to the 
+        function for Q&A question evaluation.
+"""
 
-class QnA_Model(BaseModel):
-    chapter_n: str = Field(..., 
-                           pattern=r'^Chapter_\d*$',
-                           description=(
-                                "which chapter to extract, the format of this function argumet"
-                                "is with `Chapter_` as prefix concatenated with chapter number"
-                                "in integer. For example, `Chapter_2`, `Chapter_10`."
-                                "if no chapter number specified or user requested for random question"
-                                "or user has no preference over which chapter of textbook to be tested"
-                                "return `Chapter_0`"
-                                )
+class Question_Model(BaseModel):
+    chapter_n: str = \
+                Field(..., 
+                      pattern=r'^Chapter_\d*$',
+                      description=(
+                          "which chapter to extract, the format of this function argumet"
+                          "is with `Chapter_` as prefix concatenated with chapter number"
+                          "in integer. For example, `Chapter_2`, `Chapter_10`."
+                          "if no chapter number specified or user requested for random question"
+                          "or user has no preference over which chapter of textbook to be tested"
+                          "return `Chapter_0`"
                           )
+                    )
+class Answer_Model(BaseModel):
+    user_selected_answer: str = \
+                          Field(...,
+                                pattern=r'^[ABCD]$',
+                                description=(
+                                    "which answer choice `A`, `B`, `C`, `D`"
+                                    "user selected. The return format should be"
+                                    "in single character such as A, B, C and D."
+                                    "if user's answer is contextually closer to a "
+                                    "particular answer choice, return the corresponding"
+                                    "alphabet A, B, C or D for the answer "
+                                    "is closest."
+                                ))
 
 def get_qna_question(chapter_n: str) -> str:
     """
-      Use this tool to extract the chapter information from the body of the input text, 
-      the format looks as follow:
-        The output should be in the format with `Chapter_` as prefix.
+      Use this tool to extract the chapter number from the body of input text, 
+      thereafter, chapter number will be used as a filtering criteria for
+      extracting the right questions set from database.
+      The format of the function argument looks as follow:
+        It should be in the format with `Chapter_` as prefix.
         Example 1: `Chapter_1` for first chapter
         Example 2: For chapter 12 of the textbook, you should return `Chapter_12`
         Example 3: `Chapter_5` for fifth chapter
         Thereafter, the chapter_n argument will be passed to the function for Q&A question retrieval.
+        Once the question is retrieved from database, be reminded to ask user the question.
     """
     con = sqlite3.connect(db_path)
     cur = con.cursor()
     
-    filter_clause = "" if chapter_n == "Chapter_0" else f"WHERE chapter='{chapter_n}'"
-    sql_string = """SELECT id, question, option_1, option_2, option_3, option_4, correct_answer
-                    FROM qna_tbl
+
+    filter_clause = "WHERE a.id IS NULL" if chapter_n == "Chapter_0" else f"WHERE a.id IS NULL AND chapter='{chapter_n}'"
+    sql_string = """SELECT q.id, question, option_1, option_2, option_3, option_4, q.correct_answer
+                    FROM qna_tbl q LEFT JOIN answer_tbl a
+                                   ON q.id = a.id
                  """ + filter_clause
     
     res = cur.execute(sql_string)
@@ -66,13 +100,86 @@ def get_qna_question(chapter_n: str) -> str:
                "C) " + option_3 + "\n" + \
                "D) " + option_4
     
+    st.session_state.question_id = id
+    st.session_state.qna_answer = c_answer
+    
     con.close()
     
     return qna_str
 
+def evaluate_qna_answer(user_selected_answer: str) -> str:
+    """
+      Use this tool to trigger the evaluation of user's provided input with the 
+      correct answer of the Q&A question asked. When user provides answer to the
+      question asked, they can reply in natural language or giving the alphabet
+      symbol of which selected answer they think it's most reasonable.
+      The format of the function argument `user_selected_answer` looks as follow:
+        It should be in the format with character such as A, B, C and D.
+        Example 1: User's answer is `a`, it means choice `A`.
+        Example 2: User's answer is contextually closer to 3rd answer choice, it means `C`.
+        Example 3: User says last is the answer, it means `D`.
+        Thereafter, the `user_selected_answer` argument will be passed to the 
+        function for Q&A question evaluation.
+    """
+    answer_mapping = {
+        "A": 1,
+        "B": 2,
+        "C": 3,
+        "D": 4
+    }
+    num_mapping = dict((v,k) for k,v in answer_mapping.items())
+
+    user_answer_numeric = answer_mapping.get(user_selected_answer, None)
+    if user_answer_numeric is None:
+        raise Exception(f"User's answer can't be found: {user_selected_answer}") 
+    
+    question_id = st.session_state.question_id
+    qna_answer  = st.session_state.qna_answer
+    qna_answer_alphabet = num_mapping[qna_answer]
+
+    con = sqlite3.connect(db_path)
+    cur = con.cursor()
+    sql_string = f"""INSERT INTO answer_tbl 
+                     VALUES ({question_id}, {qna_answer}, {user_answer_numeric})
+    """
+    
+    res = cur.execute(sql_string)
+    con.commit()
+    con.close()
+
+    if qna_answer == user_answer_numeric:
+        st.toast('Hooray!', icon='🎉')
+        time.sleep(0.3)
+        st.toast('Hooray!', icon='🎉')
+        time.sleep(0.3)
+        st.toast('Hooray!', icon='🎉')
+        st.balloons()
+    else:
+        st.toast('Omg..', icon='😅')
+        time.sleep(0.3)
+        st.toast('Omg..', icon='😅')
+        time.sleep(0.3)
+        st.toast('Omg..', icon='😅')
+        st.snow()
+
+
+    qna_answer_response = (
+        f"Your selected answer is `{user_selected_answer}`, "
+        f"but the actual answer is `{qna_answer_alphabet}`. "
+    )
+
+    return qna_answer_response
+
 get_qna_question_tool = FunctionTool.from_defaults(
                             fn=get_qna_question,
                             name="Extract_Question",
-                            description=description,
-                            fn_schema=QnA_Model
+                            description=qna_question_description,
+                            fn_schema=Question_Model
+)
+
+evaluate_qna_answer_tool = FunctionTool.from_defaults(
+                            fn=evaluate_qna_answer,
+                            name="Evaluate_Answer",
+                            description=qna_answer_description,
+                            fn_schema=Answer_Model
 )

reset_database.py

@@ -0,0 +1,13 @@
+import os
+import sqlite3
+
+db_path = "./database/mock_qna.sqlite"
+
+con = sqlite3.connect(db_path)
+cur = con.cursor()
+sql_string = "DELETE FROM answer_tbl"
+res = cur.execute(sql_string)
+con.commit()
+con.close()
+
+print("done cleaning up `answer_tbl`.")

streamlit_app.py

@@ -22,7 +22,7 @@ from llama_index.agent.openai import OpenAIAgent
 from llama_index.core import Settings
 
 from vision_api import get_transcribed_text
-from qna_prompting import get_qna_question_tool
+from qna_prompting import get_qna_question_tool, evaluate_qna_answer_tool
 
 import nest_asyncio
 nest_asyncio.apply()
@@ -107,6 +107,12 @@ if "feedback_key" not in st.session_state:
 if "release_file" not in st.session_state:
     st.session_state.release_file = "false"
 
+if "question_id" not in st.session_state:
+    st.session_state.question_id = None
+
+if "qna_answer" not in st.session_state:
+    st.session_state.qna_answer = None
+
 def clear_chat_history():
     st.session_state.messages = [{"role": "assistant", 
                                   "content": "How may I assist you today?",
@@ -193,18 +199,24 @@ def get_query_engine(input_files, llm_model, temperature,
                             streaming=True
     )
 
+    hi_textbook_query_description = """
+        Use this tool to extract content from Health Insurance textbook 
+        that has 15 chapters in total. When user wants to learn more about a 
+        particular chapter, this tool will help to assist user to get better
+        understanding of the content of the textbook.
+    """
     hi_query_tool = QueryEngineTool.from_defaults(
                         query_engine=hi_content_engine,
                         name="vector_tool",
-                        description=(
-                            "Provides information about Health Insurance landscape in Singapore. "
-                            "Use a detailed plain text question as input to the tool."
-                        )
+                        description=hi_textbook_query_description
     )
+
     agent = OpenAIAgent.from_tools(tools=[
                                         hi_query_tool, 
-                                        get_qna_question_tool
+                                        get_qna_question_tool,
+                                        evaluate_qna_answer_tool
                                     ],
+                                   max_function_calls=1,
                                    llm=llm, 
                                    verbose=True)
     print("loaded AI agent, let's begin the chat!")
@@ -299,6 +311,7 @@ if st.session_state.messages[-1]["role"] != "assistant":
             placeholder = st.empty()
             full_response = ""
             for token in response.response_gen:
+                token = token.replace("\n", "  \n")
                 full_response += token
                 placeholder.markdown(full_response)
             placeholder.markdown(full_response)