update

app.py

@@ -1,11 +1,13 @@
 import argparse
 import os
 import random
+
 # import sys
 # import os
 #
 # BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 # sys.path.append(BASE_DIR)
+os.system("CUDA_HOME=/usr/local/cuda python -m pip install -e git+https://github.com/IDEA-Research/GroundingDINO.git")
 
 import numpy as np
 import torch
@@ -29,7 +31,6 @@ from imagebind.models.image_bind import ModalityType
 from tagging_model import TaggingModule
 
 
-
 def parse_args():
     parser = argparse.ArgumentParser(description="Qualitative")
     parser.add_argument("--cfg-path", help="path to configuration file.", default='./eval_configs/mmgpt4_eval.yaml')
@@ -105,11 +106,11 @@ def gradio_reset(chat_state, emb_list):
     if emb_list is not None:
         emb_list = []
     return None, gr.update(value=None, interactive=True), gr.update(value=None, interactive=False), \
-           gr.update(value=None, interactive=True), \
-           gr.update(placeholder='Please upload your image/audio first', interactive=False), \
-           gr.update(value=None), \
-           gr.update(value="Upload & Start Chat", interactive=True), \
-           chat_state, emb_list, gr.update(value={})
+        gr.update(value=None, interactive=True), \
+        gr.update(placeholder='Please upload your image/audio first', interactive=False), \
+        gr.update(value=None), \
+        gr.update(value="Upload & Start Chat", interactive=True), \
+        chat_state, emb_list, gr.update(value={})
 
 
 def upload_x(gr_img, gr_aud, chat_state):
@@ -128,15 +129,15 @@ def upload_x(gr_img, gr_aud, chat_state):
     if gr_aud is not None:
         chat.upload_aud(gr_aud, chat_state, emb_list)
     return gr.update(interactive=False), gr.update(interactive=False), \
-           gr.update(interactive=True, placeholder='Type and press Enter'), \
-           gr.update(value="Start Chatting", interactive=False), \
-           chat_state, emb_list, state
+        gr.update(interactive=True, placeholder='Type and press Enter'), \
+        gr.update(value="Start Chatting", interactive=False), \
+        chat_state, emb_list, state
 
 
 def gradio_ask(user_message, chatbot, chat_state, text_output, last_answer):
     if len(user_message) == 0:
         return gr.update(interactive=True, placeholder='Input should not be empty!'), chatbot, chat_state, \
-               gr.update(value=None, color_map=None, show_legend=False), gr.update(value=None)
+            gr.update(value=None, color_map=None, show_legend=False), gr.update(value=None)
     if last_answer is not None:
         chatbot[-1][1] = last_answer
     chat.ask(user_message, chat_state)
@@ -188,6 +189,7 @@ def gradio_answer(image, chatbot, chat_state, emb_list, num_beams, temperature,
             entity_state, \
             gr.update(value=None), gr.update(value=None)
 
+
 def grounding_fn(image, chatbot, entity_state):
     # print("Grounding fn: ", entity_state)
     if image and entity_state:

grounding_model.py

@@ -64,7 +64,7 @@ class GroundingModule(nn.Module):
         print('==> Box grounding with "{}"...'.format(prompt))
         with torch.cuda.amp.autocast(enabled=True):
             boxes, logits, phrases = predict(self.grounding_model,
-                                             image_tensor, prompt, box_threshold, text_threshold, device='cpu')
+                                             image_tensor, prompt, box_threshold, text_threshold, device=self.device)
         print(phrases)
         # from PIL import Image, ImageDraw, ImageFont
         H, W = original_image.size[1], original_image.size[0]

groundingdino/models/GroundingDINO/ms_deform_attn.py

@@ -26,11 +26,11 @@ from torch.autograd.function import once_differentiable
 from torch.nn.init import constant_, xavier_uniform_
 
 try:
-    # from groundingdino import _C
-    from mmcv.utils import ext_loader
-
-    _C = ext_loader.load_ext(
-        '_ext', ['ms_deform_attn_backward', 'ms_deform_attn_forward'])
+    from groundingdino import _C
+    # from mmcv.utils import ext_loader
+    #
+    # _C = ext_loader.load_ext(
+    #     '_ext', ['ms_deform_attn_backward', 'ms_deform_attn_forward'])
 except:
     print("Failed to load custom C++ ops. Running on CPU mode Only!")