Penghao Wu commited on
Commit
b11ae09
1 Parent(s): 3672502
Files changed (2) hide show
  1. visual_search.py +567 -0
  2. vstar_bench_eval.py +294 -0
visual_search.py ADDED
@@ -0,0 +1,567 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import os
3
+ import sys
4
+ import json
5
+ import tqdm
6
+ import copy
7
+ from queue import PriorityQueue
8
+ import functools
9
+ import spacy
10
+ nlp = spacy.load("en_core_web_sm")
11
+
12
+ import cv2
13
+ from PIL import Image
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn.functional as F
17
+ from transformers import AutoTokenizer, CLIPImageProcessor
18
+ from transformers import OwlViTProcessor
19
+
20
+ from VisualSearch.model.VSM import VSMForCausalLM
21
+ from VisualSearch.model.llava import conversation as conversation_lib
22
+ from VisualSearch.model.llava.mm_utils import tokenizer_image_token
23
+ from VisualSearch.utils.utils import expand2square
24
+ from VisualSearch.utils.utils import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN,
25
+ DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX)
26
+
27
+
28
+ def parse_args(args):
29
+ parser = argparse.ArgumentParser(description="Visual Search Evaluation")
30
+ parser.add_argument("--version", default="craigwu/seal_vsm_7b")
31
+ parser.add_argument("--benchmark-folder", default="vstar_bench", type=str)
32
+ parser.add_argument("--visualization", action="store_true", default=False)
33
+ parser.add_argument("--output_path", default="", type=str)
34
+ parser.add_argument("--confidence_low", default=0.3, type=float)
35
+ parser.add_argument("--confidence_high", default=0.5, type=float)
36
+ parser.add_argument("--target_cue_threshold", default=6.0, type=float)
37
+ parser.add_argument("--target_cue_threshold_decay", default=0.7, type=float)
38
+ parser.add_argument("--target_cue_threshold_minimum", default=3.0, type=float)
39
+ parser.add_argument("--minimum_size_scale", default=4.0, type=float)
40
+ parser.add_argument("--minimum_size", default=224, type=int)
41
+ parser.add_argument("--model_max_length", default=512, type=int)
42
+ parser.add_argument(
43
+ "--vision-tower", default="openai/clip-vit-large-patch14", type=str
44
+ )
45
+ parser.add_argument("--use_mm_start_end", action="store_true", default=True)
46
+ parser.add_argument(
47
+ "--conv_type",
48
+ default="llava_v1",
49
+ type=str,
50
+ choices=["llava_v1", "llava_llama_2"],
51
+ )
52
+ return parser.parse_args(args)
53
+
54
+ def tranverse(token):
55
+ children = [_ for _ in token.children]
56
+ if len(children) == 0:
57
+ return token.i, token.i
58
+ left_i = token.i
59
+ right_i = token.i
60
+ for child in children:
61
+ child_left_i, child_right_i = tranverse(child)
62
+ left_i = min(left_i, child_left_i)
63
+ right_i = max(right_i, child_right_i)
64
+ return left_i, right_i
65
+ def get_noun_chunks(token):
66
+ left_children = []
67
+ right_children = []
68
+ for child in token.children:
69
+ if child.i < token.i:
70
+ left_children.append(child)
71
+ else:
72
+ right_children.append(child)
73
+
74
+ start_token_i = token.i
75
+ for left_child in left_children[::-1]:
76
+ if left_child.dep_ in ['amod', 'compound', 'poss']:
77
+ start_token_i, _ = tranverse(left_child)
78
+ else:
79
+ break
80
+ end_token_i = token.i
81
+ for right_child in right_children:
82
+ if right_child.dep_ in ['relcl', 'prep']:
83
+ _, end_token_i = tranverse(right_child)
84
+ else:
85
+ break
86
+ return start_token_i, end_token_i
87
+
88
+ def filter_chunk_list(chunks):
89
+ def overlap(min1, max1, min2, max2):
90
+ return min(max1, max2) - max(min1, min2)
91
+ chunks = sorted(chunks, key=lambda chunk: chunk[1]-chunk[0], reverse=True)
92
+ filtered_chunks = []
93
+ for chunk in chunks:
94
+ flag=True
95
+ for exist_chunk in filtered_chunks:
96
+ if overlap(exist_chunk[0], exist_chunk[1], chunk[0], chunk[1]) >= 0:
97
+ flag = False
98
+ break
99
+ if flag:
100
+ filtered_chunks.append(chunk)
101
+ return sorted(filtered_chunks, key=lambda chunk: chunk[0])
102
+
103
+ def extract_noun_chunks(expression):
104
+ doc = nlp(expression)
105
+ cur_chunks = []
106
+ for token in doc:
107
+ if token.pos_ not in ["NOUN", "PRON"]:
108
+ continue
109
+ cur_chunks.append(get_noun_chunks(token))
110
+ cur_chunks = filter_chunk_list(cur_chunks)
111
+ cur_chunks = [doc[chunk[0]:chunk[1]+1].text for chunk in cur_chunks]
112
+ return cur_chunks
113
+
114
+ def preprocess(
115
+ x,
116
+ pixel_mean=torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1),
117
+ pixel_std=torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1),
118
+ img_size=1024,
119
+ ) -> torch.Tensor:
120
+ """Normalize pixel values and pad to a square input."""
121
+ # Normalize colors
122
+ x = (x - pixel_mean) / pixel_std
123
+ # Pad
124
+ h, w = x.shape[-2:]
125
+ padh = img_size - h
126
+ padw = img_size - w
127
+ x = F.pad(x, (0, padw, 0, padh))
128
+ return x
129
+
130
+ def box_cxcywh_to_xyxy(x):
131
+ x_c, y_c, w, h = x.unbind(1)
132
+ b = [(x_c - 0.5 * w), (y_c - 0.5 * h),
133
+ (x_c + 0.5 * w), (y_c + 0.5 * h)]
134
+ return torch.stack(b, dim=1)
135
+
136
+ def rescale_bboxes(out_bbox, size):
137
+ img_w, img_h = size
138
+ b = box_cxcywh_to_xyxy(out_bbox)
139
+ b = b * torch.tensor([img_w, img_h, img_w, img_h], dtype=torch.float32)
140
+ return b
141
+
142
+ class VSM:
143
+ def __init__(self, args):
144
+ kwargs = {}
145
+ kwargs['torch_dtype'] = torch.bfloat16
146
+ kwargs['device_map'] = 'cuda'
147
+ kwargs['is_eval'] = True
148
+ vsm_tokenizer = AutoTokenizer.from_pretrained(
149
+ args.version,
150
+ cache_dir=None,
151
+ model_max_length=args.model_max_length,
152
+ padding_side="right",
153
+ use_fast=False,
154
+ )
155
+ vsm_tokenizer.pad_token = vsm_tokenizer.unk_token
156
+ loc_token_idx = vsm_tokenizer("[LOC]", add_special_tokens=False).input_ids[0]
157
+ vsm_model = VSMForCausalLM.from_pretrained(
158
+ args.version, low_cpu_mem_usage=True, vision_tower=args.vision_tower, loc_token_idx=loc_token_idx, **kwargs
159
+ )
160
+ vsm_model.get_model().initialize_vision_modules(vsm_model.get_model().config)
161
+ vision_tower = vsm_model.get_model().get_vision_tower().cuda().to(dtype=torch.bfloat16)
162
+ vsm_image_processor = vision_tower.image_processor
163
+ vsm_model.eval()
164
+ clip_image_processor = CLIPImageProcessor.from_pretrained(vsm_model.config.vision_tower)
165
+ transform = OwlViTProcessor.from_pretrained("google/owlvit-base-patch16")
166
+ self.model = vsm_model
167
+ self.vsm_tokenizer = vsm_tokenizer
168
+ self.vsm_image_processor = vsm_image_processor
169
+ self.clip_image_processor = clip_image_processor
170
+ self.transform = transform
171
+ self.conv_type = args.conv_type
172
+ self.use_mm_start_end = args.use_mm_start_end
173
+
174
+ @torch.inference_mode()
175
+ def inference(self, image, question, mode='segmentation'):
176
+ conv = conversation_lib.conv_templates[self.conv_type].copy()
177
+ conv.messages = []
178
+ prompt = DEFAULT_IMAGE_TOKEN + "\n" + question
179
+ if self.use_mm_start_end:
180
+ replace_token = ( DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN)
181
+ prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replace_token)
182
+ conv.append_message(conv.roles[0], prompt)
183
+ conv.append_message(conv.roles[1], "")
184
+ prompt = conv.get_prompt()
185
+
186
+ background_color = tuple(int(x*255) for x in self.clip_image_processor.image_mean)
187
+ image_clip = self.clip_image_processor.preprocess(expand2square(image, background_color), return_tensors="pt")["pixel_values"][0].unsqueeze(0).cuda()
188
+
189
+ image_clip = image_clip.bfloat16()
190
+ image = np.array(image)
191
+ original_size_list = [image.shape[:2]]
192
+ image = self.transform(images=image, return_tensors="pt")['pixel_values'].cuda()
193
+ resize_list = [image.shape[:2]]
194
+ image = image.bfloat16()
195
+ input_ids = tokenizer_image_token(prompt, self.vsm_tokenizer, return_tensors="pt")
196
+ input_ids = input_ids.unsqueeze(0).cuda()
197
+
198
+ output_ids, pred_masks, det_result = self.model.inference(
199
+ image_clip,
200
+ image,
201
+ input_ids,
202
+ resize_list,
203
+ original_size_list,
204
+ max_new_tokens=100,
205
+ tokenizer=self.vsm_tokenizer,
206
+ mode = mode
207
+ )
208
+ if mode == 'segmentation':
209
+ pred_mask = pred_masks[0]
210
+ pred_mask = torch.clamp(pred_mask, min=0)
211
+ return pred_mask[-1]
212
+
213
+ elif mode == 'vqa':
214
+ input_token_len = input_ids.shape[1]
215
+ n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
216
+ if n_diff_input_output > 0:
217
+ print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
218
+ text_output = self.vsm_tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
219
+ text_output = text_output.replace("\n", "").replace(" ", " ").strip()
220
+ return text_output
221
+
222
+ elif mode == 'detection':
223
+ pred_mask = pred_masks[0]
224
+ pred_mask = torch.clamp(pred_mask, min=0)
225
+ return det_result['pred_boxes'][0].cpu(), det_result['pred_logits'][0].sigmoid().cpu(), pred_mask[-1]
226
+
227
+ def refine_bbox(bbox, image_width, image_height):
228
+ bbox[0] = max(0, bbox[0])
229
+ bbox[1] = max(0, bbox[1])
230
+ bbox[2] = min(bbox[2], image_width-bbox[0])
231
+ bbox[3] = min(bbox[3], image_height-bbox[1])
232
+ return bbox
233
+
234
+ def split_4subpatches(current_patch_bbox):
235
+ hw_ratio = current_patch_bbox[3] / current_patch_bbox[2]
236
+ if hw_ratio >= 2:
237
+ return 1, 4
238
+ elif hw_ratio <= 0.5:
239
+ return 4, 1
240
+ else:
241
+ return 2, 2
242
+
243
+ def get_sub_patches(current_patch_bbox, num_of_width_patches, num_of_height_patches):
244
+ width_stride = int(current_patch_bbox[2]//num_of_width_patches)
245
+ height_stride = int(current_patch_bbox[3]/num_of_height_patches)
246
+ sub_patches = []
247
+ for j in range(num_of_height_patches):
248
+ for i in range(num_of_width_patches):
249
+ sub_patch_width = current_patch_bbox[2] - i*width_stride if i == num_of_width_patches-1 else width_stride
250
+ sub_patch_height = current_patch_bbox[3] - j*height_stride if j == num_of_height_patches-1 else height_stride
251
+ sub_patch = [current_patch_bbox[0]+i*width_stride, current_patch_bbox[1]+j*height_stride, sub_patch_width, sub_patch_height]
252
+ sub_patches.append(sub_patch)
253
+ return sub_patches, width_stride, height_stride
254
+
255
+ def get_subpatch_scores(score_heatmap, current_patch_bbox, sub_patches):
256
+ total_sum = (score_heatmap/(current_patch_bbox[2]*current_patch_bbox[3])).sum()
257
+ sub_scores = []
258
+ for sub_patch in sub_patches:
259
+ bbox = [(sub_patch[0]-current_patch_bbox[0]), sub_patch[1]-current_patch_bbox[1], sub_patch[2], sub_patch[3]]
260
+ score = (score_heatmap[bbox[1]:bbox[1]+bbox[3], bbox[0]:bbox[0]+bbox[2]]/(current_patch_bbox[2]*current_patch_bbox[3])).sum()
261
+ if total_sum > 0:
262
+ score /= total_sum
263
+ else:
264
+ score *= 0
265
+ sub_scores.append(score)
266
+ return sub_scores
267
+
268
+ def normalize_score(score_heatmap):
269
+ max_score = score_heatmap.max()
270
+ min_score = score_heatmap.min()
271
+ if max_score != min_score:
272
+ score_heatmap = (score_heatmap - min_score) / (max_score - min_score)
273
+ else:
274
+ score_heatmap = score_heatmap * 0
275
+ return score_heatmap
276
+
277
+ def iou(bbox1, bbox2):
278
+ x1 = max(bbox1[0], bbox2[0])
279
+ y1 = max(bbox1[1], bbox2[1])
280
+ x2 = min(bbox1[0]+bbox1[2], bbox2[0]+bbox2[2])
281
+ y2 = min(bbox1[1]+bbox1[3],bbox2[1]+bbox2[3])
282
+ inter_area = max(0, x2 - x1) * max(0, y2 - y1)
283
+ return inter_area/(bbox1[2]*bbox1[3]+bbox2[2]*bbox2[3]-inter_area)
284
+
285
+ BOX_COLOR = (255, 0, 0) # Red
286
+ TEXT_COLOR = (255, 255, 255) # White
287
+ import cv2
288
+ from matplotlib import pyplot as plt
289
+ def visualize_bbox(img, bbox, class_name, color=BOX_COLOR, thickness=2):
290
+ """Visualizes a single bounding box on the image"""
291
+ x_min, y_min, w, h = bbox
292
+ x_min, x_max, y_min, y_max = int(x_min), int(x_min + w), int(y_min), int(y_min + h)
293
+
294
+ cv2.rectangle(img, (x_min, y_min), (x_max, y_max), color=color, thickness=thickness)
295
+
296
+ ((text_width, text_height), _) = cv2.getTextSize(class_name, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
297
+ cv2.rectangle(img, (x_min, y_min - int(1.3 * text_height)), (x_min + text_width, y_min), BOX_COLOR, -1)
298
+ cv2.putText(
299
+ img,
300
+ text=class_name,
301
+ org=(x_min, y_min - int(0.3 * text_height)),
302
+ fontFace=cv2.FONT_HERSHEY_SIMPLEX,
303
+ fontScale=0.5,
304
+ color=TEXT_COLOR,
305
+ lineType=cv2.LINE_AA,
306
+ )
307
+ return img
308
+ def show_heatmap_on_image(img: np.ndarray,
309
+ mask: np.ndarray,
310
+ use_rgb: bool = False,
311
+ colormap: int = cv2.COLORMAP_JET,
312
+ image_weight: float = 0.5) -> np.ndarray:
313
+ mask = np.clip(mask, 0, 1)
314
+ heatmap = cv2.applyColorMap(np.uint8(255 * mask), colormap)
315
+ if use_rgb:
316
+ heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)
317
+ heatmap = np.float32(heatmap) / 255
318
+
319
+ if np.max(img) > 1:
320
+ raise Exception(
321
+ "The input image should np.float32 in the range [0, 1]")
322
+
323
+ if image_weight < 0 or image_weight > 1:
324
+ raise Exception(
325
+ f"image_weight should be in the range [0, 1].\
326
+ Got: {image_weight}")
327
+
328
+ cam = (1 - image_weight) * heatmap + image_weight * img
329
+ cam = cam / np.max(cam)
330
+ return np.uint8(255 * cam)
331
+ def vis_heatmap(image, heatmap, use_rgb=False):
332
+ max_v = np.max(heatmap)
333
+ min_v = np.min(heatmap)
334
+ if max_v != min_v:
335
+ heatmap = (heatmap - min_v) / (max_v - min_v)
336
+ heatmap_image = show_heatmap_on_image(image.astype(float)/255., heatmap, use_rgb=use_rgb)
337
+ return heatmap_image
338
+
339
+ def visualize_search_path(image, search_path, search_length, target_bbox, label, save_path):
340
+ context_cue_list = []
341
+ whole_image = image
342
+ os.makedirs(save_path, exist_ok=True)
343
+ whole_image.save(os.path.join(save_path, 'whole_image.jpg'))
344
+
345
+ whole_image = np.array(whole_image)
346
+ if target_bbox is not None:
347
+ whole_image = visualize_bbox(whole_image.copy(), target_bbox, class_name="gt: "+label, color=(255,0,0))
348
+ for step_i, node in enumerate(search_path):
349
+ if step_i + 1 > search_length:
350
+ break
351
+ current_patch_box = node['bbox']
352
+ if 'detection_result' in node:
353
+ final_patch_image = image.crop((current_patch_box[0],current_patch_box[1],current_patch_box[0]+current_patch_box[2], current_patch_box[1]+current_patch_box[3]))
354
+ final_patch_image.save(os.path.join(save_path, 'final_patch_image.jpg'))
355
+ final_search_result = visualize_bbox(np.array(final_patch_image), node['detection_result'], class_name='search result', color=(255,0,0))
356
+ final_search_result = cv2.cvtColor(final_search_result, cv2.COLOR_RGB2BGR)
357
+ cv2.imwrite(os.path.join(save_path, 'search_result.jpg'), final_search_result)
358
+ cur_whole_image = visualize_bbox(whole_image.copy(), current_patch_box, class_name="step-{}".format(step_i+1), color=(0,0,255))
359
+ # if step_i != len(search_path)-1:
360
+ # next_patch_box = search_path[step_i+1]['bbox']
361
+ # cur_whole_image = visualize_bbox(cur_whole_image, next_patch_box, class_name="next-step", color=(0,255,0))
362
+ cur_whole_image = cv2.cvtColor(cur_whole_image, cv2.COLOR_RGB2BGR)
363
+ cv2.imwrite(os.path.join(save_path, 'step_{}.jpg'.format(step_i+1)), cur_whole_image)
364
+
365
+ cur_patch_image = image.crop((current_patch_box[0],current_patch_box[1],current_patch_box[0]+current_patch_box[2], current_patch_box[1]+current_patch_box[3]))
366
+ if 'context_cue' in node:
367
+ context_cue = node['context_cue']
368
+ context_cue_list.append('step{}: {}'.format(step_i+1, context_cue)+'\n')
369
+ if 'final_heatmap' in node:
370
+ score_map = node['final_heatmap']
371
+ score_map = vis_heatmap(np.array(cur_patch_image), score_map, use_rgb=True)
372
+ score_map = cv2.cvtColor(score_map, cv2.COLOR_RGB2BGR)
373
+ cv2.imwrite(os.path.join(save_path, 'step_{}_heatmap.jpg'.format(step_i+1)), score_map)
374
+
375
+ with open(os.path.join(save_path, 'context_cue.txt'),"w") as f:
376
+ f.writelines(context_cue_list)
377
+
378
+ @functools.total_ordering
379
+ class Prioritize:
380
+
381
+ def __init__(self, priority, item):
382
+ self.priority = priority
383
+ self.item = item
384
+
385
+ def __eq__(self, other):
386
+ return self.priority == other.priority
387
+
388
+ def __lt__(self, other):
389
+ return self.priority < other.priority
390
+ def visual_search_queue(vsm, image, target_object_name, current_patch, search_path, queue, smallest_size=224, confidence_high=0.5, target_cue_threshold=6.0, target_cue_threshold_decay=0.7, target_cue_threshold_minimum=3.0):
391
+ current_patch_bbox = current_patch['bbox']
392
+ current_patch_scale_level = current_patch['scale_level']
393
+
394
+ image_patch = image.crop((int(current_patch_bbox[0]), int(current_patch_bbox[1]), int(current_patch_bbox[0]+current_patch_bbox[2]), int(current_patch_bbox[1]+current_patch_bbox[3])))
395
+ # whehter we can detect the target object on the current image patch
396
+ question = "Please locate the {} in this image.".format(target_object_name)
397
+ pred_bboxes, pred_logits, target_cue_heatmap = vsm.inference(copy.deepcopy(image_patch), question, mode='detection')
398
+ if len(pred_logits) > 0:
399
+ top_index = pred_logits.view(-1).argmax()
400
+ top_logit = pred_logits.view(-1).max()
401
+ final_bbox = pred_bboxes[top_index].view(4)
402
+ final_bbox = final_bbox * torch.Tensor([image_patch.width, image_patch.height, image_patch.width, image_patch.height])
403
+ final_bbox[:2] -= final_bbox[2:] / 2
404
+ if top_logit > confidence_high:
405
+ search_path[-1]['detection_result'] = final_bbox
406
+ # only return multiple detected instances on the whole image
407
+ if len(search_path) == 1:
408
+ all_valid_boxes = pred_bboxes[pred_logits.view(-1)>0.5].view(-1, 4)
409
+ all_valid_boxes = all_valid_boxes * torch.Tensor([[image_patch.width, image_patch.height, image_patch.width, image_patch.height]])
410
+ all_valid_boxes[:, :2] -= all_valid_boxes[:, 2:] / 2
411
+ return True, search_path, all_valid_boxes
412
+ return True, search_path, None
413
+ else:
414
+ search_path[-1]['temp_detection_result'] = (top_logit, final_bbox)
415
+
416
+ ### current patch is already the smallest unit
417
+ if min(current_patch_bbox[2], current_patch_bbox[3]) <= smallest_size:
418
+ return False, search_path, None
419
+
420
+ target_cue_heatmap = target_cue_heatmap.view(current_patch_bbox[3], current_patch_bbox[2], 1)
421
+ score_max = target_cue_heatmap.max().item()
422
+ # check whether the target cue is prominent
423
+ threshold = max(target_cue_threshold_minimum, target_cue_threshold*(target_cue_threshold_decay)**(current_patch_scale_level-1))
424
+ if score_max > threshold:
425
+ target_cue_heatmap = normalize_score(target_cue_heatmap)
426
+ final_heatmap = target_cue_heatmap
427
+ else:
428
+ question = "According to the common sense knowledge and possible visual cues, what is the most likely location of the {} in the image?".format(target_object_name)
429
+ vqa_results = vsm.inference(copy.deepcopy(image_patch), question, mode='vqa')
430
+
431
+ possible_location_phrase = vqa_results.split('most likely to appear')[-1].strip()
432
+ if possible_location_phrase.endswith('.'):
433
+ possible_location_phrase = possible_location_phrase[:-1]
434
+ possible_location_phrase = possible_location_phrase.split(target_object_name)[-1]
435
+ noun_chunks = extract_noun_chunks(possible_location_phrase)
436
+ if len(noun_chunks) == 1:
437
+ possible_location_phrase = noun_chunks[0]
438
+ else:
439
+ possible_location_phrase = "region {}".format(possible_location_phrase)
440
+ question = "Please locate the {} in this image.".format(possible_location_phrase)
441
+ context_cue_heatmap = vsm.inference(copy.deepcopy(image_patch), question, mode='segmentation').view(current_patch_bbox[3], current_patch_bbox[2], 1)
442
+ context_cue_heatmap = normalize_score(context_cue_heatmap)
443
+ final_heatmap = context_cue_heatmap
444
+
445
+ current_patch_index = len(search_path)-1
446
+ if score_max <= threshold:
447
+ search_path[current_patch_index]['context_cue'] = vqa_results + "#" + possible_location_phrase
448
+ search_path[current_patch_index]['final_heatmap'] = final_heatmap.cpu().numpy()
449
+
450
+ ### split the current patch into 4 sub-patches
451
+ basic_sub_patches, sub_patch_width, sub_patch_height = get_sub_patches(current_patch_bbox, *split_4subpatches(current_patch_bbox))
452
+
453
+ tmp_patch = current_patch
454
+ basic_sub_scores = [0]*len(basic_sub_patches)
455
+ while True:
456
+ tmp_score_heatmap = tmp_patch['final_heatmap']
457
+ tmp_sub_scores = get_subpatch_scores(tmp_score_heatmap, tmp_patch['bbox'], basic_sub_patches)
458
+ basic_sub_scores = [basic_sub_scores[patch_i]+tmp_sub_scores[patch_i]/(4**tmp_patch['scale_level']) for patch_i in range(len(basic_sub_scores))]
459
+ if tmp_patch['parent_index'] == -1:
460
+ break
461
+ else:
462
+ tmp_patch = search_path[tmp_patch['parent_index']]
463
+
464
+ sub_patches = basic_sub_patches
465
+ sub_scores = basic_sub_scores
466
+
467
+ for sub_patch, sub_score in zip(sub_patches, sub_scores):
468
+ new_patch_info = dict()
469
+ new_patch_info['bbox'] = sub_patch
470
+ new_patch_info['scale_level'] = current_patch_scale_level + 1
471
+ new_patch_info['score'] = sub_score
472
+ new_patch_info['parent_index'] = current_patch_index
473
+ queue.put(Prioritize(-new_patch_info['score'], new_patch_info))
474
+
475
+ while(not queue.empty()):
476
+ patch_chosen = queue.get().item
477
+ search_path.append(patch_chosen)
478
+ success, search_path, all_valid_boxes = visual_search_queue(vsm, image, target_object_name, patch_chosen, search_path, queue, smallest_size=smallest_size, confidence_high=confidence_high, target_cue_threshold=target_cue_threshold, target_cue_threshold_decay=target_cue_threshold_decay, target_cue_threshold_minimum=target_cue_threshold_minimum)
479
+ if success:
480
+ return success, search_path, all_valid_boxes
481
+ return False, search_path, None
482
+
483
+
484
+ def visual_search(vsm, image, target_object_name, target_bbox, smallest_size, confidence_high=0.5, confidence_low=0.3, target_cue_threshold=6.0, target_cue_threshold_decay=0.7, target_cue_threshold_minimum=3.0, visualize=False, save_path=None):
485
+ if visualize:
486
+ assert save_path is not None
487
+ init_patch = dict()
488
+ init_patch['bbox'] = [0,0,image.width,image.height]
489
+ init_patch['scale_level'] = 1
490
+ init_patch['score'] = None
491
+ init_patch['parent_index'] = -1
492
+ search_path = [init_patch]
493
+
494
+ queue = PriorityQueue()
495
+ search_successful, search_path, all_valid_boxes = visual_search_queue(vsm, image, target_object_name, init_patch, search_path, queue, smallest_size=smallest_size, confidence_high=confidence_high, target_cue_threshold=target_cue_threshold, target_cue_threshold_decay=target_cue_threshold_decay, target_cue_threshold_minimum=target_cue_threshold_minimum)
496
+ path_length = len(search_path)
497
+ final_step = search_path[-1]
498
+ if not search_successful:
499
+ # if no target is found with confidence passing confidence_high, select the target with the highest confidence during search and compare its confidence with confidence_low
500
+ max_logit = 0
501
+ final_step = None
502
+ path_length = 0
503
+ for i, search_step in enumerate(search_path):
504
+ if 'temp_detection_result' in search_step:
505
+ if search_step['temp_detection_result'][0] > max_logit:
506
+ max_logit = search_step['temp_detection_result'][0]
507
+ final_step = search_step
508
+ path_length = i+1
509
+ final_step['detection_result'] = final_step['temp_detection_result'][1]
510
+ if max_logit >= confidence_low:
511
+ search_successful = True
512
+ if visualize:
513
+ vis_path_length = path_length if search_successful else len(search_path)
514
+ visualize_search_path(image, search_path, vis_path_length, target_bbox, target_object_name, save_path)
515
+ del queue
516
+ return final_step, path_length, search_successful, all_valid_boxes
517
+
518
+
519
+
520
+ def main(args):
521
+ args = parse_args(args)
522
+ vsm = VSM(args)
523
+
524
+ benchmark_folder = args.benchmark_folder
525
+
526
+ acc_list = []
527
+ search_path_length_list = []
528
+
529
+ for test_type in ['direct_attributes', 'relative_position']:
530
+ folder = os.path.join(benchmark_folder, test_type)
531
+ output_folder = None
532
+ if args.visualization:
533
+ output_folder = os.path.join(args.output_path, test_type)
534
+ os.makedirs(output_folder, exist_ok=True)
535
+ image_files = filter(lambda file: '.json' not in file, os.listdir(folder))
536
+ for image_file in tqdm.tqdm(image_files):
537
+ image_path = os.path.join(folder, image_file)
538
+ annotation_path = image_path.split('.')[0] + '.json'
539
+ annotation = json.load(open(annotation_path))
540
+ bboxs = annotation['bbox']
541
+ object_names = annotation['target_object']
542
+
543
+ for i, (gt_bbox, object_name) in enumerate(zip(bboxs, object_names)):
544
+ image = Image.open(image_path).convert('RGB')
545
+ smallest_size = max(int(np.ceil(min(image.width, image.height)/args.minimum_size_scale)), args.minimum_size)
546
+ if args.visualization:
547
+ vis_path = os.path.join(output_folder, "{}_{}".format(image_file.split('.')[0],i))
548
+ else:
549
+ vis_path = None
550
+ final_step, path_length, search_successful, all_valid_boxes = visual_search(vsm, image, object_name, target_bbox=gt_bbox, smallest_size=smallest_size, confidence_high=args.confidence_high, confidence_low=args.confidence_low, target_cue_threshold=args.target_cue_threshold, target_cue_threshold_decay=args.target_cue_threshold_decay, target_cue_threshold_minimum=args.target_cue_threshold_minimum, save_path=vis_path, visualize=args.visualization)
551
+ if search_successful:
552
+ search_bbox = final_step['detection_result']
553
+ search_final_patch = final_step['bbox']
554
+ search_bbox[0] += search_final_patch[0]
555
+ search_bbox[1] += search_final_patch[1]
556
+ iou_i = iou(search_bbox, gt_bbox).item()
557
+ det_acc = 1.0 if iou_i > 0.5 else 0.0
558
+ acc_list.append(det_acc)
559
+ search_path_length_list.append(path_length)
560
+ else:
561
+ acc_list.append(0)
562
+ search_path_length_list.append(0)
563
+ print('Avg search path length:', np.mean([search_path_length_list[i] for i in range(len(search_path_length_list)) if acc_list[i]]))
564
+ print('Top 1 Acc:', np.mean(acc_list))
565
+
566
+ if __name__ == "__main__":
567
+ main(sys.argv[1:])
vstar_bench_eval.py ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import os
3
+ import json
4
+ from tqdm import tqdm
5
+ from collections import defaultdict
6
+ from copy import deepcopy
7
+
8
+ from PIL import Image
9
+ import numpy as np
10
+ import torch
11
+ from torch.nn import CrossEntropyLoss
12
+
13
+ from LLaVA.llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN
14
+ from LLaVA.llava.conversation import conv_templates, SeparatorStyle
15
+ from LLaVA.llava.model.builder import load_pretrained_model
16
+ from LLaVA.llava.utils import disable_torch_init
17
+ from LLaVA.llava.mm_utils import get_model_name_from_path, KeywordsStoppingCriteria, tokenizer_image_object_token
18
+
19
+ from visual_search import parse_args, VSM, visual_search
20
+
21
+ def normalize_bbox(bbox, image_width, image_height):
22
+ normalized_bbox = [bbox[0]/image_width, bbox[1]/image_height, (bbox[0]+bbox[2])/image_width, (bbox[1]+bbox[3])/image_height]
23
+ normalized_bbox = [np.clip(_, 0, 1) for _ in normalized_bbox]
24
+ return normalized_bbox
25
+ def expand2square(pil_img, background_color):
26
+ width, height = pil_img.size
27
+ if width == height:
28
+ return pil_img, 0, 0
29
+ elif width > height:
30
+ result = Image.new(pil_img.mode, (width, width), background_color)
31
+ result.paste(pil_img, (0, (width - height) // 2))
32
+ return result, 0, (width - height) // 2
33
+ else:
34
+ result = Image.new(pil_img.mode, (height, height), background_color)
35
+ result.paste(pil_img, ((height - width) // 2, 0))
36
+ return result, (height - width) // 2, 0
37
+
38
+ class VQA_LLM:
39
+ def __init__(self, args):
40
+ disable_torch_init()
41
+ model_path = args.vqa_model_path
42
+ model_name = get_model_name_from_path(model_path)
43
+ model_name += 'llava'
44
+ model_base = None
45
+ device_map = "auto"
46
+ self.tokenizer, self.model, self.image_processor, self.context_len = load_pretrained_model(model_path, model_base, model_name)
47
+ self.conv_type = args.conv_type
48
+
49
+ def get_patch(self, bbox, image_width, image_height, patch_size=224, patch_scale=None):
50
+ object_width = int(np.ceil(bbox[2]))
51
+ object_height = int(np.ceil(bbox[3]))
52
+
53
+ object_center_x = int(bbox[0] + bbox[2]/2)
54
+ object_center_y = int(bbox[1] + bbox[3]/2)
55
+
56
+ if patch_scale is None:
57
+ patch_width = max(object_width, patch_size)
58
+ patch_height = max(object_height, patch_size)
59
+ else:
60
+ patch_width = int(object_width*patch_scale)
61
+ patch_height = int(object_height*patch_scale)
62
+
63
+ left = max(0, object_center_x-patch_width//2)
64
+ right = min(left+patch_width, image_width)
65
+
66
+ top = max(0, object_center_y-patch_height//2)
67
+ bottom = min(top+patch_height, image_height)
68
+
69
+ return [left, top, right, bottom]
70
+
71
+ def get_object_crop(self, image, bbox, patch_scale):
72
+ resized_bbox = self.get_patch(bbox, image.width, image.height, patch_scale=patch_scale)
73
+ object_crop = image.crop((resized_bbox[0], resized_bbox[1], resized_bbox[2], resized_bbox[3]))
74
+ object_crop = object_crop.resize((self.image_processor.crop_size['width'],self.image_processor.crop_size['height']))
75
+ object_crop = self.image_processor.preprocess(object_crop, return_tensors='pt')['pixel_values'][0]
76
+ return object_crop
77
+
78
+ @torch.inference_mode()
79
+ def free_form_inference(self, image, question, temperature=0, top_p=None, num_beams=1, max_new_tokens=200, object_crops=None, images_long=None, objects_long=None):
80
+ conv = conv_templates[self.conv_type].copy()
81
+ qs = DEFAULT_IMAGE_TOKEN + '\n' + question
82
+ conv.append_message(conv.roles[0], qs)
83
+ conv.append_message(conv.roles[1], None)
84
+ prompt = conv.get_prompt()
85
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
86
+ keywords = [stop_str]
87
+ input_ids = tokenizer_image_object_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
88
+ image_tensor = self.image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
89
+ stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
90
+
91
+ output_ids = self.model.generate(
92
+ input_ids,
93
+ images=image_tensor.unsqueeze(0).half().cuda(),
94
+ object_features=object_crops.half().cuda() if object_crops is not None else None,
95
+ images_long = images_long,
96
+ objects_long = objects_long,
97
+ do_sample= True if temperature > 0 else False,
98
+ num_beams=num_beams,
99
+ temperature=temperature,
100
+ top_p = top_p,
101
+ max_new_tokens=max_new_tokens,
102
+ use_cache=True,
103
+ stopping_criteria=[stopping_criteria])
104
+
105
+ input_token_len = input_ids.shape[1]
106
+ n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
107
+ if n_diff_input_output > 0:
108
+ print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
109
+ outputs = self.tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
110
+ outputs = outputs.strip()
111
+ if outputs.endswith(stop_str):
112
+ outputs = outputs[:-len(stop_str)]
113
+ outputs = outputs.strip()
114
+ return outputs
115
+
116
+ @torch.inference_mode()
117
+ def multiple_choices_inference(self, image, question, options, object_crops=None, images_long=None, objects_long=None):
118
+ conv = conv_templates[self.conv_type].copy()
119
+ qs = DEFAULT_IMAGE_TOKEN + '\n' + question
120
+ conv.append_message(conv.roles[0], qs)
121
+ conv.append_message(conv.roles[1], None)
122
+ prompt = conv.get_prompt()
123
+
124
+ question_input_ids = tokenizer_image_object_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
125
+ image_tensor = self.image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
126
+
127
+ output_question = self.model(
128
+ question_input_ids,
129
+ use_cache=True,
130
+ images=image_tensor.unsqueeze(0).half().cuda(),
131
+ object_features=object_crops.half().cuda() if object_crops is not None else None,
132
+ images_long = images_long,
133
+ objects_long = objects_long)
134
+
135
+ question_logits = output_question.logits
136
+ question_past_key_values = output_question.past_key_values
137
+
138
+ loss_list = []
139
+
140
+ for option in options:
141
+ conv = conv_templates[self.conv_type].copy()
142
+ conv.append_message(conv.roles[0], qs)
143
+ conv.append_message(conv.roles[1], option)
144
+ full_prompt = conv.get_prompt()
145
+
146
+ full_input_ids = tokenizer_image_object_token(full_prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
147
+ option_answer_input_ids = full_input_ids[:, question_input_ids.shape[1]:]
148
+
149
+ output_option = self.model(input_ids=option_answer_input_ids,
150
+ use_cache=True,
151
+ attention_mask=torch.ones(1, question_logits.shape[1]+option_answer_input_ids.shape[1], device=full_input_ids.device),
152
+ past_key_values=question_past_key_values)
153
+
154
+ logits = torch.cat([question_logits[:, -1:], output_option.logits[:, :-1]], 1)
155
+
156
+ loss_fct = CrossEntropyLoss()
157
+ logits = logits.view(-1, self.model.config.vocab_size)
158
+ labels = option_answer_input_ids.view(-1)
159
+ loss = loss_fct(logits, labels)
160
+
161
+ loss_list.append(loss)
162
+
163
+ option_chosen = torch.stack(loss_list).argmin()
164
+
165
+ return option_chosen.cpu().item()
166
+
167
+
168
+ def eval_model(args):
169
+ # init VQA LLM
170
+ vqa_llm = VQA_LLM(args)
171
+ # init VSM
172
+ vsm_args = parse_args({})
173
+ vsm_args.version = args.vsm_model_path
174
+ vsm = VSM(vsm_args)
175
+
176
+ results = {}
177
+ per_type_acc = defaultdict(list)
178
+ all_acc = []
179
+
180
+ missing_objects_msg = "Sorry, I can not answer the question. Some visual information about the following objects is missing or unclear:"
181
+ focus_msg = "Additional visual information to focus on: "
182
+ for test_type in ['direct_attributes', 'relative_position']:
183
+ results[test_type] = []
184
+ folder = os.path.join(args.benchmark_folder, test_type)
185
+ image_files = list(filter(lambda file: '.json' not in file, os.listdir(folder)))
186
+ for image_file in tqdm(image_files):
187
+ result_single_sample = {}
188
+ image_path = os.path.join(folder, image_file)
189
+ annotation_path = image_path.split('.')[0] + '.json'
190
+ image = Image.open(image_path).convert('RGB')
191
+ annotation = json.load(open(annotation_path))
192
+ image, _, _ = expand2square(image, tuple(int(x*255) for x in vqa_llm.image_processor.image_mean))
193
+
194
+ question = annotation['question']
195
+ # generate free-form response to check whether visual search needs to be activated
196
+ prediction = vqa_llm.free_form_inference(image, question)
197
+ missing_objects = []
198
+ if missing_objects_msg in prediction:
199
+ missing_objects = prediction.split(missing_objects_msg)[-1]
200
+ if missing_objects.endswith('.'):
201
+ missing_objects = missing_objects[:-1]
202
+ missing_objects = missing_objects.split(',')
203
+ missing_objects = [missing_object.strip() for missing_object in missing_objects]
204
+
205
+ search_result = []
206
+ if len(missing_objects) > 0:
207
+ # visual search
208
+ for object_name in missing_objects:
209
+ image = Image.open(image_path).convert('RGB')
210
+ smallest_size = max(int(np.ceil(min(image.width, image.height)/args.minimum_size_scale)), args.minimum_size)
211
+ final_step, path_length, search_successful, all_valid_boxes = visual_search(vsm, image, object_name, target_bbox=None, smallest_size=smallest_size)
212
+ if all_valid_boxes is not None:
213
+ # might exist multiple target instances
214
+ for search_bbox in all_valid_boxes:
215
+ search_final_patch = final_step['bbox']
216
+ search_bbox[0] += search_final_patch[0]
217
+ search_bbox[1] += search_final_patch[1]
218
+ search_result.append({'bbox':search_bbox.tolist(),'name':object_name})
219
+ else:
220
+ search_bbox = final_step['detection_result']
221
+ search_final_patch = final_step['bbox']
222
+ search_bbox[0] += search_final_patch[0]
223
+ search_bbox[1] += search_final_patch[1]
224
+ search_result.append({'bbox':search_bbox.tolist(),'name':object_name})
225
+ # predict the multiple-choice option
226
+ options = annotation['options']
227
+ image = Image.open(image_path).convert('RGB')
228
+ if len(missing_objects) > 0:
229
+ object_names = [_['name'] for _ in search_result]
230
+ bboxs = deepcopy([_['bbox'] for _ in search_result])
231
+ if len(object_names) <= 2:
232
+ images_long = [False]
233
+ objects_long = [True]*len(object_names)
234
+ else:
235
+ images_long = [False]
236
+ objects_long = [False]*len(object_names)
237
+ object_crops = []
238
+ for bbox in bboxs:
239
+ object_crop = vqa_llm.get_object_crop(image, bbox, patch_scale=1.2)
240
+ object_crops.append(object_crop)
241
+ object_crops = torch.stack(object_crops, 0)
242
+ image, left, top = expand2square(image, tuple(int(x*255) for x in vqa_llm.image_processor.image_mean))
243
+ bbox_list = []
244
+ for bbox in bboxs:
245
+ bbox[0] += left
246
+ bbox[1] += top
247
+ bbox_list.append(bbox)
248
+ bbox_list = [normalize_bbox(bbox, image.width, image.height) for bbox in bbox_list]
249
+ cur_focus_msg = focus_msg
250
+ for i, (object_name, bbox) in enumerate(zip(object_names, bbox_list)):
251
+ cur_focus_msg = cur_focus_msg + "{} <object> at location [{:.3f},{:.3f},{:.3f},{:.3f}]".format(object_name, bbox[0], bbox[1], bbox[2], bbox[3])
252
+ if i != len(bbox_list)-1:
253
+ cur_focus_msg = cur_focus_msg+"; "
254
+ else:
255
+ cur_focus_msg = cur_focus_msg +'.'
256
+ question_with_focus = cur_focus_msg+"\n"+question
257
+ option_chosen = vqa_llm.multiple_choices_inference(image, question_with_focus, options, object_crops, images_long=images_long, objects_long=objects_long)
258
+ else:
259
+ option_chosen = vqa_llm.multiple_choices_inference(image, question, options)
260
+
261
+ correct = 1 if option_chosen==0 else 0
262
+ per_type_acc[test_type].append(correct)
263
+ all_acc.append(correct)
264
+
265
+ result_single_sample['question'] = question
266
+ result_single_sample['options'] = options
267
+ result_single_sample['image'] = image_file
268
+ result_single_sample['prediction_freeform'] = prediction
269
+ result_single_sample['missing_objects'] = missing_objects
270
+ result_single_sample['search_result'] = search_result
271
+ result_single_sample['option_chosen'] = option_chosen
272
+ result_single_sample['correct'] = correct
273
+ results[test_type].append(result_single_sample)
274
+
275
+ print(test_type, np.mean(per_type_acc[test_type]))
276
+
277
+ print(np.mean(all_acc))
278
+
279
+ with open(args.output_path, 'w') as f:
280
+ json.dump(results, f, indent=4)
281
+
282
+ if __name__ == "__main__":
283
+ parser = argparse.ArgumentParser()
284
+ parser.add_argument("--vqa-model-path", type=str, default="craigwu/seal_vqa_7b")
285
+ parser.add_argument("--vqa-model-base", type=str, default=None)
286
+ parser.add_argument("--conv_type", default="v1", type=str,)
287
+ parser.add_argument("--benchmark-folder", type=str, default="vstar_bench")
288
+ parser.add_argument("--vsm-model-path", type=str, default="craigwu/seal_vsm_7b")
289
+ parser.add_argument("--output-path", type=str, default="eval_result.json")
290
+ parser.add_argument("--minimum_size_scale", default=4.0, type=float, help="minimum sub-image scale for the termination of search")
291
+ parser.add_argument("--minimum_size", default=224, type=int, help="minimum sub-image size for the termination of search")
292
+
293
+ args = parser.parse_args()
294
+ eval_model(args)