feat: Add VQA feature

controllers/config.py

@@ -1,4 +1,5 @@
 from controllers.fw import add_fw_option, del_fw_option, fetch_fw_options
+from services.state import set_state
 
 
 def on_fw_add_btn_click(url: str):
@@ -16,3 +17,7 @@ def on_del_btn_click(url: str):
     options.pop(url)
     # Return new options
     return options, ""
+
+
+def on_fw_radio_change(option: str):
+    set_state('fw_option', option)

controllers/fd.py

@@ -18,10 +18,9 @@ def draw_bboxs(image: Image, faces: list, color: str):
     return np_image
 
 
-def fd_controller(image: Image, model: str, color: str, fw_option: str = 'none'):
+def fd_controller(image: Image, color: str, fw_option: str = 'none'):
     request_data = {
         "image": image_to_bytes(image),
-        "model": model
     }
     response = post_data(
         f"/api/fd/{get_fw_query_params(fw_option)}", files=request_data)

controllers/vqa.py

@@ -0,0 +1,19 @@
+from PIL import Image
+from controllers.utils import image_to_bytes
+from services.api_service import post_data
+from .fw import get_fw_query_params
+
+
+def vqa_controller(image: Image, question: str, fw_option: str = 'none'):
+    request_data_files = {
+        "image": image_to_bytes(image),
+    }
+    request_data_form = {
+        "question": question,
+    }
+    response = post_data(
+        f"/api/vqa/{get_fw_query_params(fw_option)}", files=request_data_files, data=request_data_form)
+    print(response)
+    if response is None:
+        return None
+    return response['answer']

db/state.json

@@ -0,0 +1 @@
+{"fw_option": "auto"}

services/api_service.py

@@ -15,10 +15,10 @@ def get_data(url, params=None, parse_json=True):
         return None
 
 
-def post_data(url, json=None, files=None, parse_json=True):
+def post_data(url, json=None, files=None, data=None, parse_json=True):
     if url.startswith("/"):
         url = API_ENDPOINT + url
-    response = post(url, json=json, files=files)
+    response = post(url, json=json, files=files, data=data)
     if response.status_code == 200:
         if parse_json:
             return response.json()

services/state.py

@@ -1,19 +1,16 @@
-from utilities.constants import FW_DEFAULT_OPTION
+import json
+from utilities.constants import STATE_JSON_FILE
 
 
-class AppState():
-    def __init__(self):
-        self._state = {}
+def get_state(key: str, default=None):
+    with open(STATE_JSON_FILE, 'r') as f:
+        state: dict = json.load(f)
+    return state.get(key, default)
 
-    def get(self, key):
-        return self._state.get(key)
 
-    def set(self, key, value):
-        self._state[key] = value
-
-
-# Create app state
-app_state = AppState()
-
-# Initialize app state
-app_state.set('fw_option', FW_DEFAULT_OPTION)
+def set_state(key: str, value):
+    with open(STATE_JSON_FILE, 'r') as f:
+        state = json.load(f)
+    state[key] = value
+    with open(STATE_JSON_FILE, 'w') as f:
+        json.dump(state, f)

utilities/constants.py

@@ -1,4 +1,8 @@
+import os
+
 API_ENDPOINT = "https://gdscfptu-ai-service-hf.hf.space"
+STATE_JSON_FILE = os.path.join(os.getcwd(), "db", "state.json")
+
 FW_DEFAULT_OPTION = "auto"
 CHAT_EXAMPLES = ["Where should I go in Da Nang?",
                  "Best destinations in Quy Nhon"]

views/__init__.py

@@ -3,6 +3,7 @@ import gradio as gr
 from .rembg import rembg_interface
 from .fd import fd_interface
 from .chat import chat_interface
+from .vqa import vqa_interface
 from .config import config_interface
 
 
@@ -11,10 +12,11 @@ def create_interface():
     fd_ui = fd_interface()
     chat_ui = chat_interface()
     config_ui = config_interface()
+    vqa_ui = vqa_interface()
 
     return gr.TabbedInterface(
         title="GDSC AI Service Inference",
-        interface_list=[rembg_ui, fd_ui, chat_ui.queue(), config_ui],
+        interface_list=[rembg_ui, fd_ui, chat_ui.queue(), vqa_ui, config_ui],
         tab_names=[rembg_ui.title, fd_ui.title,
-                   chat_ui.title, config_ui.title],
+                   chat_ui.title, vqa_ui.title, config_ui.title],
     )

views/chat.py

@@ -1,13 +1,11 @@
 import gradio as gr
 from controllers.chat import chat_controller
-from services.state import app_state
-
-CHAT_EXAMPLES = ["Where should I go in Da Nang?",
-                 "Best destinations in Quy Nhon"]
+from services.state import get_state
+from utilities.constants import FW_DEFAULT_OPTION, CHAT_EXAMPLES
 
 
 def chats_yielding(prompt: str, histories: list[list]):
-    for msg in chat_controller(prompt, histories, app_state.get('fw_option')):
+    for msg in chat_controller(prompt, histories, get_state('fw_option', FW_DEFAULT_OPTION)):
         yield msg
 
 

views/config.py

@@ -1,13 +1,13 @@
 import gradio as gr
-from controllers.config import on_del_btn_click, on_fw_add_btn_click
+from controllers.config import on_del_btn_click, on_fw_add_btn_click, on_fw_radio_change
 from controllers.fw import fetch_fw_options
-from utilities.constants import FW_RADIO_CHOICES, FW_DEFAULT_OPTION
+from utilities.constants import FW_RADIO_CHOICES
 
 
 def config_interface():
     with gr.Blocks() as ui:
         gr.Markdown("### Forwarding Endpoint")
-        radio_btn = gr.Radio(choices=FW_RADIO_CHOICES, value=FW_DEFAULT_OPTION,
+        radio_btn = gr.Radio(choices=FW_RADIO_CHOICES,
                              label="Select Forwarding Option")
         fw_list = gr.Label(value=fetch_fw_options,
                            label="Current Endpoints", every=2)
@@ -17,13 +17,13 @@ def config_interface():
             with gr.Row():
                 add_btn = gr.Button(value="Add Endpoint", variant="secondary")
                 del_btn = gr.Button(value="Delete Endpoint", variant="stop")
-        gr.Markdown("### OpenAI API Key")
-        with gr.Column():
-            input_box2 = gr.Textbox(
-                label="OpenAI API Key", placeholder="sk_...")
-            with gr.Row():
-                add_btn2 = gr.Button(value="Add Key", variant="secondary")
-                del_btn2 = gr.Button(value="Reset Key", variant="stop")
+        # gr.Markdown("### OpenAI API Key")
+        # with gr.Column():
+        #     input_box2 = gr.Textbox(
+        #         label="OpenAI API Key", placeholder="sk_...")
+        #     with gr.Row():
+        #         add_btn2 = gr.Button(value="Add Key", variant="secondary")
+        #         del_btn2 = gr.Button(value="Reset Key", variant="stop")
 
         # Assign actions
         add_btn.click(
@@ -36,6 +36,10 @@ def config_interface():
             inputs=[input_box],
             outputs=[fw_list, input_box]
         )
+        radio_btn.change(
+            fn=on_fw_radio_change,
+            inputs=[radio_btn],
+        )
 
     ui.title = "Configuration"
     return ui

views/fd.py

@@ -1,17 +1,15 @@
 import gradio as gr
-from utilities.constants import FD_DROPDOWN_CHOICES
+from utilities.constants import FW_DEFAULT_OPTION
 from controllers.fd import fd_controller
-from services.state import app_state
+from services.state import get_state
 
 
 def fd_interface():
     return gr.Interface(
         title="Face Detection",
-        fn=lambda model, image, color: fd_controller(
-            image, model, color, app_state.get('fw_option')),
+        fn=lambda image, color: fd_controller(
+            image, color, get_state('fw_option', FW_DEFAULT_OPTION)),
         inputs=[
-            gr.Dropdown(choices=FD_DROPDOWN_CHOICES,
-                        value=FD_DROPDOWN_CHOICES[0], label="Select Model. *Not affect in forwarding mode*"),
             gr.Image(type="pil", label="Input Image"),
             gr.ColorPicker(label="Box Color")
         ],

views/rembg.py

@@ -1,14 +1,14 @@
 import gradio as gr
 from controllers.rembg import rembg_controller
-from services.state import app_state
-from utilities.constants import REMBG_DROPDOWN_CHOICES
+from services.state import get_state
+from utilities.constants import REMBG_DROPDOWN_CHOICES, FW_DEFAULT_OPTION
 
 
 def rembg_interface():
     return gr.Interface(
         title="Remove Background",
         fn=lambda resolution, image: rembg_controller(
-            image, resolution, app_state.get('fw_option')),
+            image, resolution, get_state('fw_option', FW_DEFAULT_OPTION)),
         inputs=[
             gr.Dropdown(choices=REMBG_DROPDOWN_CHOICES,
                         value=REMBG_DROPDOWN_CHOICES[0], label="Resolution"),

views/vqa.py

@@ -0,0 +1,20 @@
+import gradio as gr
+from services.state import get_state
+from controllers.vqa import vqa_controller
+from utilities.constants import FW_DEFAULT_OPTION
+
+
+def vqa_interface():
+    return gr.Interface(
+        title="Visual Question Answering",
+        fn=lambda image, question: vqa_controller(
+            image, question, get_state('fw_option', FW_DEFAULT_OPTION)),
+        inputs=[
+            gr.Image(type="pil", label="Input Image"),
+            gr.Textbox(label="Question")
+        ],
+        outputs=[
+            gr.Textbox(label="Answer")
+        ],
+        flagging_options=[]
+    )