perfection is perfected

app.py

@@ -1,21 +1,28 @@
 #@title Model Loading
 from sentence_transformers import SentenceTransformer, util
-from utils import get_latest_user_input_from_prompt, get_top_intents, create_embedding
-from intents import intents, intents_sentence_similarity_en
+from utils import get_history_from_prompt, get_latest_user_input_from_prompt, get_top_intents, create_embedding
+from intents import intents, intents_sentence_similarity_en, chatbot_intents
+from prompt import prompt_template
 import flows
+import os
 import gradio as gr
 import pandas as pd
+import langchain
+from langchain import PromptTemplate, LLMChain
+from langchain.chat_models import ChatOpenAI
 
 model_en = SentenceTransformer("intfloat/multilingual-e5-base")
 intents_embedding = create_embedding(intents_sentence_similarity_en, model_en)
 
+llm = None
+llm_chain = None
 
 def raw_inference(input, state, n_samples, threshold):
   state = flows.STATE_FLOWS_MAP[state]
   query_embedding = model_en.encode(input)
   similarity = util.pytorch_cos_sim(query_embedding, intents_embedding)
   result = get_top_intents(intents, similarity, n=n_samples, threshold=threshold, flow=state)
-  return result
+  return result, gr.Button("Ask intent with Language Model", visible=True)
 
 def process_csv(files):
   global df
@@ -28,28 +35,34 @@ def process_csv(files):
   df_length = len(df.index)
 
   chat = get_latest_user_input_from_prompt(df.iloc[1]["prompt"])
+  history = get_history_from_prompt(df.iloc[1]["prompt"])
   state = flows.STATE_FLOWS_MAP[df.iloc[1]['state']]
   label = df.iloc[1]['chatbot_response']
 
-  accuracy = gr.Markdown("""
+  # accuracy = gr.Markdown("""
 
-        You can also check accuracy on how well the model predict the intents based on your provided CSV files. This might take 1-2 minutes.
+  #       You can also check accuracy on how well the model predict the intents based on your provided CSV files. This might take 1-2 minutes.
 
-        """, visible=True)
-  accuracy_button = gr.Button("Calculate Accuracy", visible=True)
+  #       """, visible=True)
+  # accuracy_button = gr.Button("Calculate Accuracy", visible=True)
 
   return (gr.UploadButton("Upload CSV...", file_types=["file"], file_count="single", visible=False),
           files,
           gr.Slider(1, df_length, value=1, step=1, visible=True, label="Index", info="Select which index of data to check the intents"),
           gr.Textbox(label="Input Chat", info="Input in index", visible=True, value=chat, interactive=False),
           gr.Textbox(label="State", info="State on which the chat currently on. Some state will exclude some intents", visible=True, value=state, interactive=False),
-          gr.Textbox(label="Ground Truth", info="The label in which the IntentClassification predict in the CSV", visible=True, value=label, interactive=False))
+          gr.Textbox(label="Ground Truth", info="The label in which the IntentClassification predict in the CSV", visible=True, value=label, interactive=False),
+          gr.Textbox(label="History or summary", info="Chat history or summary, if available", visible=True, value=history, interactive=False))
 
 def update_index(index):
   chat = get_latest_user_input_from_prompt(df.iloc[int(index)]["prompt"])
+  history = get_history_from_prompt(df.iloc[int(index)]["prompt"])
   state = df.iloc[int(index)]['state']
   label = df.iloc[int(index)]['chatbot_response']
-  return gr.Textbox(label="Input Chat", info="Input in index", visible=True, value=chat, interactive=False), gr.Textbox(label="State", info="State on which the chat currently on. Some state will exclude some intents", visible=True, value=state, interactive=False), gr.Textbox(label="Ground Truth", info="The label in which the IntentClassification predict in the CSV", visible=True, value=label, interactive=False)
+  return (gr.Textbox(label="Input Chat", info="Input in index", visible=True, value=chat, interactive=False), 
+          gr.Textbox(label="State", info="State on which the chat currently on. Some state will exclude some intents", visible=True, value=state, interactive=False), 
+          gr.Textbox(label="Ground Truth", info="The label in which the IntentClassification predict in the CSV", visible=True, value=label, interactive=False),
+          gr.Textbox(label="History or summary", info="Chat history or summary, if available", visible=True, value=history, interactive=False))
 
 def check_accuracy(n_samples, threshold):
   global df
@@ -82,6 +95,19 @@ def check_accuracy(n_samples, threshold):
   print(score, grouped_data)
   return score, grouped_data
 
+def classify_intent(input_text:str, history:str, answer, api_key):
+  os.environ['OPENAI_API_KEY'] = api_key
+  llm = ChatOpenAI(model='gpt-3.5-turbo', temperature='0.1')
+  prompt = PromptTemplate(template=prompt_template, input_variables=["intents", "INPUT", "chatHistory"])
+  llm_chain = LLMChain(prompt=prompt, llm=llm, verbose=False)
+
+  inp_intents = ''
+  for i in range(len(answer)):
+    inp_intents += answer[i][0]+": "+chatbot_intents[answer[i][0]]+"\n"
+  predicted_intent = llm_chain.run({"intents":inp_intents, "INPUT": input_text, "chatHistory": history})
+  prompt_result = llm_chain.prompt.format_prompt(intents = inp_intents, INPUT = input_text, chatHistory = history).to_string()
+  return predicted_intent, prompt_result
+
 theme = gr.themes.Default(
     primary_hue="indigo",
     secondary_hue="pink",
@@ -90,8 +116,12 @@ theme = gr.themes.Default(
 
 with gr.Blocks(title="Intent Classification Demo", theme=theme) as interface:
   gr.Markdown("""# Demo for Intent Classification""")
+
   with gr.Row(equal_height=True):
     with gr.Column():
+      api_key = gr.Textbox(label="OpenAI API Key", info="get it at https://platform.openai.com/account/api-keys",visible=True, lines=1, type="password")
+      n_samples = gr.Slider(1, 10, value=5, step=1, label="N samples", info="Number of samples to be retrieved. Default is 5")
+      threshold = gr.Slider(0.0, 1.0, value=0.75, step=0.01, label="Threshold", info="Threshold of cosine similarity which intent will be considered similar to the input. The higher, the more similar the intent will be. Default is 0.75")
       with gr.Tab("Input from raw text"):
         raw_input_text = gr.Textbox(label="Input Chat", info="Input your chat here, the model will predict the intent")
         raw_state = gr.Dropdown(["GeneralState",
@@ -103,7 +133,9 @@ with gr.Blocks(title="Intent Classification Demo", theme=theme) as interface:
                             "AssessKnowledgeState"],
                             label="State",
                             info="Select state on which the chat currently on. Some state will exclude some intents")
+        raw_history = gr.Textbox(label="History or summary", info="Chat history or summary, if available", visible=True)
         raw_ask_button = gr.Button("Ask")
+        ask_llm_button_raw = gr.Button("Ask intent with Language Model", visible=False)
 
       with gr.Tab("Input from Big Query data"):
         gr.Markdown("""
@@ -129,12 +161,14 @@ with gr.Blocks(title="Intent Classification Demo", theme=theme) as interface:
         index = gr.Slider(1, 1000, value=5, step=1, visible=False, label="Index", info="Select which index of data to check the intents")
         input_text = gr.Textbox(label="Input Chat", info="Input in index", visible=False)
         state = gr.Textbox(label="State", info="State on which the chat currently on. Some state will exclude some intents", visible=False)
+        history = gr.Textbox(label="History or summary", info="Chat history or summary, if available", visible=False)
         gt = gr.Textbox(label="Ground Truth", info="The label in which the IntentClassification predict in the CSV", visible=False)
         ask_button = gr.Button("Ask With CSV")
+        ask_llm_button = gr.Button("Ask intent with Language Model", visible=False)
 
 
-        index.change(fn=update_index, inputs=index, outputs=[input_text, state, gt])
-        upload_button.upload(process_csv, upload_button, [upload_button, file_output, index, input_text, state, gt])
+        index.change(fn=update_index, inputs=index, outputs=[input_text, state, gt, history])
+        upload_button.upload(process_csv, upload_button, [upload_button, file_output, index, input_text, state, gt, history])
         with gr.Column():
           with gr.Row():
             accuracy = gr.Markdown("""
@@ -143,18 +177,18 @@ with gr.Blocks(title="Intent Classification Demo", theme=theme) as interface:
 
             """, visible=True)
             accuracy_button = gr.Button("Calculate Accuracy", visible=True)
-          accuracy_score = gr.Textbox(label="Accuracy", info="Accuracy result tested on CSV file you uploaded", visible=True)
+          accuracy_score = gr.Label(label="Accuracy result", visible=True)
           accuracy_table = gr.Dataframe(visible=True)
 
     with gr.Column():
-      n_samples = gr.Slider(1, 10, value=5, step=1, label="N samples", info="Number of samples to be retrieved. Default is 5")
-      threshold = gr.Slider(0.0, 1.0, value=0.75, step=0.01, label="Threshold", info="Threshold of cosine similarity which intent will be considered similar to the input. The higher, the more similar the intent will be. Default is 0.75")
-      answer = gr.JSON(label="Prediction", show_label=True)
-
+      answer = gr.JSON(label="Sentence Similarity Prediction", show_label=True)
+      LLM_prediction = gr.Label(label="LLM Prediction Result", visible=True)
+      LLM_prompt = gr.Textbox(label="Prompt Used for Language Model", info="Showing prompt used in language model", visible=True)
 
   accuracy_button.click(fn=check_accuracy, inputs=[n_samples, threshold], outputs=[accuracy_score, accuracy_table])
-  raw_ask_button.click(fn=raw_inference, inputs=[raw_input_text, raw_state, n_samples, threshold], outputs=answer)
-  ask_button.click(fn=raw_inference, inputs=[input_text, state, n_samples, threshold], outputs=answer)
-
+  raw_ask_button.click(fn=raw_inference, inputs=[raw_input_text, raw_state, n_samples, threshold], outputs=[answer, ask_llm_button_raw])
+  ask_button.click(fn=raw_inference, inputs=[input_text, state, n_samples, threshold], outputs=[answer, ask_llm_button])
+  ask_llm_button.click(fn=classify_intent, inputs=[input_text, history, answer, api_key], outputs=[LLM_prediction, LLM_prompt])
+  ask_llm_button_raw.click(fn=classify_intent, inputs=[raw_input_text, raw_history, answer, api_key], outputs=[LLM_prediction, LLM_prompt])
 
 interface.launch(debug=True)

intents.py

@@ -21,11 +21,11 @@ intents_sentence_similarity_en = {
     "exercise_multiple_question":"Wants to do exercise with a number of question",
     "exercise_multiple_question_with_topic":"Wants to do exercise with a number of question with topics",
     "personal":"Sharing personal story (aku mau curhat, kak lagi sedih)",
-    "general_intent":"Greetings, Asking/desire about study and exams in general. (aku pusing, aku mau belajar kak, besok ulangan, butuh temen belajar, besok ujian, temenin aku, bantu aku, aku butuh bantuan)",
+    "general_intent":" casual chatting, Greetings (halo kak, selamat pagi), Asking help about study and exams in general (aku pusing, aku stres, aku mau belajar kak, besok ulangan, butuh temen belajar, besok ujian, temenin aku, bantu aku, aku butuh bantuan)",
     "provide_topic":"Lesson material Topic",
     "provide_subject":"Lesson material Subject (Fisika, Kimia, Matematika, Biologi, Sejarah, Bahasa Indonesia, Geografi, Biologi, Bahasa Inggris)",
     "provide_subtopic":"Lesson material Subtopic",
-    "provide_concept":"Concept",
+    "provide_concept":"Explain concept",
     "finish_learning":"Finished/already learned (Terimakasih /Thank you/No/Sudah paham!/Ok/Tidak)",
     "back_to_study":"Return to study (Kembali Belajar) /Mulai Belajar?/langsung mulai belajar",
     "similar_question":"Create similar exercise questions (Buatkan Soal Serupa / Cari Soal Serupa / Soal lagi yang mirip / Latihan soal yang mirip)",
@@ -66,3 +66,39 @@ intents = ['homework',
  'provide_answer',
  'diagnosis_result',
  'explain_drill']
+
+chatbot_intents = {
+    "homework":"The student needs help to do their homework (PR). The student doesn't provide the question, only the intention. (Bantu Kerjakan PR, Bantu PR,tanya soal, tanya pr, tanya soal lain, tanya soal yang lain, tanya yang lain",
+    "homework_with_question":"Help students with their homework (PR), the student provide the specific question about their homework. ('berapa 1+1 ?', 'berapa kecepatan mobil yang menempuh jarak 100km dalam waktu 2 jam ?', 'apa yang dimaksud dengan monokotil dan dikotil ?')",
+    "explain_answer":"Student is curious or wants more explanation about the answer (Belajar Lebih Dalam/Ceritakan lebih lanjut/jelaskan dari awal/jelaskan/jelasin lebih lanjut/belum paham)",
+    "explain_answer_with_topic":"Student is curious or wants more explanation about specific topics (Belajar Lebih Dalam terkait termodinamika/Ceritakan lebih lanjut bab matematika integral/jelaskan dari awal materi mitosis/jelaskan lebih lanjut pelajaran listrik/belum paham terkait materi past tenses)",
+    "core_explanation_repeat":"Student wants repeat explanation (Jelasin ulang/Jelaskan ulang/Jelaskan lagi)",
+    "core_explanation_exercise":"Student wants explain with exercise (Contoh Soal/Contoh soal dong/bahas dengan soal)",
+    "core_explanation_video":"Student wants explanation with video (Video Penjelasan/Bahas dengan video/mau video penjelasan)",
+    "core_explanation_protips":"Student wants tips and tricks (protips/tips dan trik)",
+    "negative_respond":"Student respond they dont understand the explanation (aku belum paham, masih belum paham, aku gangerti, aku belum mengerti, masih bingung)",
+    "positive_respond":"Student respond they understand the explanation (aku sudah paham, oke paham kak, oke mengerti, oke aman, lanjut kak)",
+    "recommend":"The student want learning materials suggestions (Rekomendasi Materi/Suggested Materials/Materi Terkait)",
+    "recommend_with_topic":"The student want learning materials suggestions, students provide the topic (Aku mau materi integral/lagi pengen materi aritmatika/mau materi kimia dong kak/Rekomendasi materi tata surya kak/Materi terkait termodinamika)",
+    "assess_knowledge":"The student want their skill/knowledge to be assessed (Cek Kemampuan/Uji kemampuan/diagnosis kemampuan/mengasah kemampuan/Belajar Lebih Efektif/Cek pemahamanku)",
+    "assess_knowledge_with_topic":"The student want their skill/knowledge to be assessed and student provide topics(Cek Kemampuan integral/Uji kemampuan tata surya/diagnosis kemampuan termodinamika/mengasah kemampuan hukum newton/Cek pemahaman aljabar linear)",
+    "assess_knowledge_answer":"The student answer to follow up Diagnosis Kemampuan result. (jawabanku kok salah ya, kenapa bisa salah ya jawabanku, aku gabisa jawab, wah jawabanku bener wkwk, yeay bener semua jawabanku, jawabanku salah 1 aja yeay, cuman bener 1 jawabanku, secara garis besar udah bagus ya hasilku, perlu peningkatan belajar lagi nih)",
+    "exercise":"The student want to do exercise (Latihan Soal, aku mau soal, aku butuh soal, bantu carikan soal dong, mau soal dong, mau pertanyaan dong, latihan pertanyaan, coba ulang pertanyaan nya, ulangi pertanyaanya karena tadi kurang jelas, minta soal)",
+    "exercise_with_image":"The student want to do exercise with an image (aku mau soal dengan gambar, aku butuh soal pake gambar, bantu carikan soal menggunakan gambar dong, mau soal pake gambar, minta soal yang ada gambarnya)",
+    "exercise_with_topic":"The student want to do exercise and students provide topic (carikan soal integral, mau soal fisika, berikan aku soal tata surya, soal geografi, latihan soal ekonomi, soal biologi)",
+    "exercise_multiple_question":"The student wants exercise with multiple / more than one question without topics (langsung 2 soal dong, mau 10 soal sekaligus, berikan aku 4 soal)",
+    "exercise_multiple_question_with_topic":"The student wants exercise with multiple / more than one question with topics (minta soal 2 fotosintesis kak, mau 10 soal sekaligus aritmatika, berikan aku 4 soal termodinamika fisika)",
+    "personal":"The student want to share their personal story (aku mau curhat, kak lagi sedih)",
+    "general_intent":"The student intent in general about study. (aku pusing, aku mau belajar kak, besok ulangan, butuh temen belajar, besok ujian, temenin aku, bantu aku, aku butuh bantuan)",
+    "provide_topic":"The student provide topics. the topic is more specific hierarchy rather than subject example topic is integral, stoikiometri, termodinamika, mitosis. Usually only one words",
+    "provide_subject":"The student provide subjects. example subjects are matematika, fisika, kimia, biologi, sejarah, geografi, fisika. Usually only one words",
+    "provide_subtopic":"The student provide subtopic. subtopic is more specific compared to subjects and topics. for example subjects: fisika, topics: energi, subtopics: energi termal. for example subtopics: energi kinetik, aljabar linear, fungsi trigonometri. Usually more than one words",
+    "provide_concept":"The student provide concept. concept is more specific compared to subjects, topics, and subtopics. for example subjects: fisika, topics: energi, subtopics: energi termal, concept: mengerti suhu dan kesetimbangan termal. for example concepts: sistem persamaan linear dua variabel, prinsip kekekalan energi termal, hukum termodinamika pertama. Usually more than one words",
+    "finish_learning":"Students feel that they have enough/finish about the learning they are doing.",
+    "back_to_study":"students wants to back study (Kembali Belajar)",
+    "similar_question":"students wants to create variations of questions (Buatkan Soal Serupa / Cari Soal Serupa / Soal lagi yang mirip / Latihan soal yang mirip)",
+    "provide_answer":"students answer the questions from chatbot multiple questions. Sometimes students only answer with 1 characters like (A, B, C, D) or more than one characters (Jawabanya B, B. tata surya, D.trigonometri, Menurutku jawabanya C)",
+    "diagnosis_result":"students want to see their diagnosis result (Lihat hasil diagnosis / hasil drill)",
+    "explain_drill":"students want to know more about the drill material topics/subtopics (bahas lewat chat/bahas soal nomer 1 ya/bahas soal nomer 2 ya/bahas soal nomer 3 ya/bahas soal nomer 4 ya/bahas soal nomer 5 ya)"
+  }
+

prompt.py

@@ -0,0 +1,14 @@
+#@title prompt.py
+prompt_template = """
+You are to classify the following message into exactly one of the intents described below:
+
+{intents}
+unknown: You don't find the matching intent from the above list
+
+Message: {INPUT}
+
+Here are previous chats or summary conversation for your reference (only use this if you need further information to infer the intent):
+{chatHistory}
+
+Intent: 
+"""