Add question validation model in front of VQA

chatbot.py

@@ -33,6 +33,9 @@ def predict(image, input):
     answer = st.session_state.predictor.predict_answer_from_text(image, input)
     st.session_state.question.append(input)
     st.session_state.answer.append(answer)
+    while len(st.session_state.question) >= 5:
+        st.session_state.answer.pop(0)
+        st.session_state.question.pop(0)
 
 
 def show():

model/predictor.py

@@ -4,12 +4,13 @@ from transformers import ViltProcessor
 from transformers import ViltForQuestionAnswering
 from transformers import AutoTokenizer
 from transformers import AutoModelForSeq2SeqLM
+from joblib import load
 
 import os
 import re
 import string
 import torch
-
+import pandas as pd
 
 '''
 Visual Question Answering Model to generate answer statement for
@@ -32,16 +33,26 @@ class Predictor:
         self.happy_tt = HappyTextToText(
             "T5", "vennify/t5-base-grammar-correction")
         self.tt_args = TTSettings(num_beams=5, min_length=1)
+        model_path= os.path.join( os.path.dirname(os.path.abspath(__file__)), 'qa_classifier.joblib')
+        self.qa_classifier = load(model_path)
+
+    def is_valid_question(self, question):
+        df=pd.DataFrame()
+        df['sentence']=[question]
+        return self.qa_classifier.predict(df['sentence'])[0] == 1
 
     def predict_answer_from_text(self, image, input):
         if image is None:
-            return 'Please select an image...'
+            return 'Please select an image and ask a question...'
 
         chars = re.escape(string.punctuation)
         question = re.sub(r'['+chars+']', '', input)
         if not question or len(question.split()) < 3:
             return 'I cannot understand, please ask a valid question...'
 
+        if not self.is_valid_question(question):
+            return 'I can understand only questions, can you please ask a valid question...'
+
         # process question using image model
         encoding = self.vqa_processor(image, question, return_tensors='pt')
         with torch.no_grad():

model/qa_classifier.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37ab3777a05c935a42303a7ad07e4dc8bfc6ea07ce8a3da4dbfab16104f9c7af
+size 197373

requirements.txt

@@ -97,11 +97,14 @@ requests==2.28.0
 responses==0.18.0
 rich==12.4.4
 say==1.6.6
+scikit-learn==1.1.1
+scipy==1.8.1
 semver==2.13.0
 Send2Trash==1.8.0
 sentencepiece==0.1.96
 simplere==1.2.13
 six==1.12.0
+sklearn==0.0
 smmap==5.0.0
 soupsieve==2.3.2.post1
 stack-data==0.3.0
@@ -110,6 +113,7 @@ streamlit-bokeh-events==0.1.2
 streamlit-chat==0.0.2.1
 terminado==0.15.0
 textwrap3==0.9.2
+threadpoolctl==3.1.0
 tinycss2==1.1.1
 tokenizers==0.12.1
 toml==0.10.2