Spaces:
Sleeping
Sleeping
| # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. | |
| # Set up custom environment before nearly anything else is imported | |
| # NOTE: this should be the first import (no not reorder) | |
| from maskrcnn_benchmark.utils.env import setup_environment # noqa F401 isort:skip | |
| import argparse | |
| import os | |
| import functools | |
| import io | |
| import datetime | |
| import itertools | |
| import json | |
| from tqdm import tqdm | |
| import numpy as np | |
| import torch | |
| import torch.distributed as dist | |
| from collections import defaultdict | |
| from maskrcnn_benchmark.config import cfg | |
| from maskrcnn_benchmark.data import make_data_loader | |
| from maskrcnn_benchmark.engine.inference import inference, create_positive_dict, clean_name | |
| from maskrcnn_benchmark.modeling.detector import build_detection_model | |
| from maskrcnn_benchmark.utils.checkpoint import DetectronCheckpointer | |
| from maskrcnn_benchmark.utils.collect_env import collect_env_info | |
| from maskrcnn_benchmark.utils.comm import synchronize, get_rank, is_main_process, all_gather | |
| from maskrcnn_benchmark.utils.logger import setup_logger | |
| from maskrcnn_benchmark.utils.miscellaneous import mkdir | |
| from maskrcnn_benchmark.utils.stats import get_model_complexity_info | |
| from omnilabeltools import OmniLabel, OmniLabelEval, visualize_image_sample | |
| import time | |
| import json | |
| import tempfile | |
| import matplotlib.pyplot as plt | |
| from transformers import AutoTokenizer, CLIPTokenizerFast | |
| import omnilabeltools as olt | |
| from omnilabeltools import OmniLabel, OmniLabelEval | |
| import pdb | |
| import wandb | |
| from multiprocessing import Pool | |
| class LLM: | |
| def __init__(self, version, prompt_file = None, temp = 1.0): | |
| self.version = version | |
| self.prompt_file = prompt_file | |
| self.temp = temp | |
| with open(self.prompt_file, "r") as f: | |
| self.prompt = f.read() | |
| def __call__(self, entity): | |
| time.sleep(0.1) | |
| success = False | |
| fail_count = 0 | |
| if isinstance(entity, list): | |
| prompt = [self.prompt.replace("PROMPT", e) for e in entity] | |
| else: | |
| if self.version == "chat": | |
| raw_prompt = self.prompt.replace("PROMPT", entity) | |
| try: | |
| prompt = json.loads(raw_prompt) | |
| except: | |
| prompt = [{"role": "user", "content": raw_prompt}] | |
| else: | |
| prompt = self.prompt.replace("PROMPT", entity) | |
| while not success: | |
| try: | |
| if self.version == "chat": | |
| model = "gpt-3.5-turbo" | |
| response = openai.ChatCompletion.create( | |
| model=model, | |
| messages = prompt, | |
| temperature=self.temp, | |
| ) | |
| else: | |
| if self.version == "curie": | |
| model = "curie" | |
| else: | |
| model = "text-davinci-003" | |
| response = openai.Completion.create( | |
| model=model, | |
| prompt=prompt, | |
| temperature=self.temp, | |
| max_tokens=128, | |
| top_p=1, | |
| frequency_penalty=0.0, | |
| presence_penalty=0.0, | |
| ) | |
| success = True | |
| fail_count = 0 | |
| except Exception as e: | |
| print(f"Exception: {e}") | |
| time.sleep(0.1) | |
| fail_count += 1 | |
| if fail_count > 10: | |
| print("Too many failures") | |
| return "Too many failures" | |
| if isinstance(entity, list): | |
| if self.version == "chat": | |
| return [r["message"]["content"] for r in response["choices"]] | |
| else: | |
| return [r["text"] for r in response["choices"]] | |
| else: | |
| if self.version == "chat": | |
| return response["choices"][0]["message"]["content"] | |
| else: | |
| return response["choices"][0]["text"] | |
| def init_distributed_mode(args): | |
| """Initialize distributed training, if appropriate""" | |
| if "RANK" in os.environ and "WORLD_SIZE" in os.environ: | |
| args.rank = int(os.environ["RANK"]) | |
| args.world_size = int(os.environ["WORLD_SIZE"]) | |
| args.gpu = int(os.environ["LOCAL_RANK"]) | |
| elif "SLURM_PROCID" in os.environ: | |
| args.rank = int(os.environ["SLURM_PROCID"]) | |
| args.gpu = args.rank % torch.cuda.device_count() | |
| else: | |
| print("Not using distributed mode") | |
| args.distributed = False | |
| return | |
| # args.distributed = True | |
| torch.cuda.set_device(args.gpu) | |
| args.dist_backend = "nccl" | |
| print("| distributed init (rank {}): {}".format(args.rank, args.dist_url), flush=True) | |
| dist.init_process_group( | |
| backend=args.dist_backend, | |
| init_method=args.dist_url, | |
| world_size=args.world_size, | |
| rank=args.rank, | |
| timeout=datetime.timedelta(0, 7200), | |
| ) | |
| dist.barrier() | |
| setup_for_distributed(args.rank == 0) | |
| def setup_for_distributed(is_master): | |
| """ | |
| This function disables printing when not in master process | |
| """ | |
| import builtins as __builtin__ | |
| builtin_print = __builtin__.print | |
| def print(*args, **kwargs): | |
| force = kwargs.pop("force", False) | |
| if is_master or force: | |
| builtin_print(*args, **kwargs) | |
| __builtin__.print = print | |
| def remove_full_stop(description_list): | |
| ret_list = [] | |
| for descript in description_list: | |
| if descript[-1] == '.': | |
| descript = descript[:-1] # remove '.' | |
| ret_list.append(descript) | |
| return ret_list | |
| def num_of_words(text): | |
| return len(text.split(' ')) | |
| def create_queries_and_maps(labels, label_list, tokenizer, additional_labels=None, cfg=None, center_nouns_length = None, override_tokens_positive = None): | |
| # Clean label list | |
| label_list = [clean_name(i) for i in label_list] | |
| # Form the query and get the mapping | |
| tokens_positive = [] | |
| start_i = 0 | |
| end_i = 0 | |
| objects_query = "Detect: " | |
| #objects_query = "" | |
| prefix_length = len(objects_query) | |
| # sep between tokens, follow training | |
| separation_tokens = cfg.DATASETS.SEPARATION_TOKENS | |
| caption_prompt = cfg.DATASETS.CAPTION_PROMPT | |
| use_caption_prompt = cfg.DATASETS.USE_CAPTION_PROMPT and caption_prompt is not None | |
| for _index, label in enumerate(label_list): | |
| if use_caption_prompt: | |
| objects_query += caption_prompt[_index]["prefix"] | |
| start_i = len(objects_query) | |
| if use_caption_prompt: | |
| objects_query += caption_prompt[_index]["name"] | |
| else: | |
| objects_query += label | |
| if "a kind of " in label: | |
| end_i = len(label.split(",")[0]) + start_i | |
| else: | |
| end_i = len(objects_query) | |
| tokens_positive.append([(start_i, end_i)]) # Every label has a [(start, end)] | |
| if use_caption_prompt: | |
| objects_query += caption_prompt[_index]["suffix"] | |
| if _index != len(label_list) - 1: | |
| objects_query += separation_tokens | |
| if additional_labels is not None: | |
| objects_query += separation_tokens | |
| for _index, label in enumerate(additional_labels): | |
| objects_query += label | |
| if _index != len(additional_labels) - 1: | |
| objects_query += separation_tokens | |
| # print(objects_query) | |
| if cfg.MODEL.LANGUAGE_BACKBONE.TOKENIZER_TYPE == "bert-base-uncased": | |
| tokenized = tokenizer(objects_query, return_tensors="pt") | |
| elif cfg.MODEL.LANGUAGE_BACKBONE.TOKENIZER_TYPE == "roberta-base": | |
| tokenized = tokenizer(objects_query, return_tensors="pt") | |
| elif cfg.MODEL.LANGUAGE_BACKBONE.TOKENIZER_TYPE == "clip": | |
| tokenized = tokenizer( | |
| objects_query, max_length=cfg.MODEL.LANGUAGE_BACKBONE.MAX_QUERY_LEN, truncation=True, return_tensors="pt" | |
| ) | |
| else: | |
| raise NotImplementedError | |
| if override_tokens_positive is not None: | |
| new_tokens_positive = [] | |
| for override in override_tokens_positive: | |
| new_tokens_positive.append((override[0] + prefix_length, override[1] + prefix_length)) | |
| tokens_positive = [new_tokens_positive] # this is because we only have one label | |
| # Create the mapping between tokenized sentence and the original label | |
| # if one_hot: | |
| # positive_map_token_to_label, positive_map_label_to_token = create_one_hot_dict(labels, no_minus_one_for_one_hot=cfg.DATASETS.NO_MINUS_ONE_FOR_ONE_HOT) | |
| # else: | |
| positive_map_token_to_label, positive_map_label_to_token = create_positive_dict( | |
| tokenized, tokens_positive, labels=labels | |
| ) # from token position to original label | |
| return objects_query, positive_map_label_to_token | |
| def main(): | |
| parser = argparse.ArgumentParser(description="PyTorch Detection to Grounding Inference") | |
| parser.add_argument( | |
| "--config-file", | |
| default="configs/pretrain/glip_Swin_T_O365_GoldG.yaml", | |
| metavar="FILE", | |
| help="path to config file", | |
| ) | |
| parser.add_argument( | |
| "--weight", | |
| default=None, | |
| metavar="FILE", | |
| help="path to config file", | |
| ) | |
| parser.add_argument("--local_rank", type=int, default=0) | |
| parser.add_argument( | |
| "opts", help="Modify config options using the command-line", default=None, nargs=argparse.REMAINDER | |
| ) | |
| parser.add_argument("--world-size", default=1, type=int, help="number of distributed processes") | |
| parser.add_argument("--dist-url", default="env://", help="url used to set up distributed training") | |
| parser.add_argument("--task_config", default=None) | |
| parser.add_argument("--chunk_size", default=20, type=int, help="number of descriptions each time") | |
| parser.add_argument("--threshold", default=None, type=float, help="number of boxes stored in each run") | |
| parser.add_argument("--topk_per_eval", default=None, type=int, help="number of boxes stored in each run") | |
| parser.add_argument("--group_query", action="store_true", help="group query") | |
| parser.add_argument("--noun_phrase_file", default=None, type=str, help="noun phrase file") | |
| args = parser.parse_args() | |
| num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1 | |
| distributed = num_gpus > 1 | |
| if distributed: | |
| # torch.cuda.set_device(args.local_rank) | |
| # torch.distributed.init_process_group( | |
| # backend="nccl", init_method="env://" | |
| # ) | |
| init_distributed_mode(args) | |
| print("Passed distributed init") | |
| cfg.local_rank = args.local_rank | |
| cfg.num_gpus = num_gpus | |
| cfg.merge_from_file(args.config_file) | |
| cfg.merge_from_list(args.opts) | |
| cfg.freeze() | |
| log_dir = cfg.OUTPUT_DIR | |
| if args.weight: | |
| log_dir = os.path.join(log_dir, "eval", os.path.splitext(os.path.basename(args.weight))[0]) | |
| if log_dir: | |
| mkdir(log_dir) | |
| logger = setup_logger("maskrcnn_benchmark", log_dir, get_rank()) | |
| logger.info(args) | |
| logger.info("Using {} GPUs".format(num_gpus)) | |
| logger.info(cfg) | |
| # logger.info("Collecting env info (might take some time)") | |
| # logger.info("\n" + collect_env_info()) | |
| device = cfg.MODEL.DEVICE | |
| cpu_device = torch.device("cpu") | |
| model = build_detection_model(cfg) | |
| model.to(device) | |
| # we currently disable this | |
| # params, flops = get_model_complexity_info(model, | |
| # (3, cfg.INPUT.MAX_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST), | |
| # input_constructor=lambda x: {'images': [torch.rand(x).cuda()]}) | |
| # print("FLOPs: {}, #Parameter: {}".format(params, flops)) | |
| checkpointer = DetectronCheckpointer(cfg, model, save_dir=cfg.OUTPUT_DIR) | |
| if args.weight: | |
| _ = checkpointer.load(args.weight, force=True) | |
| else: | |
| _ = checkpointer.load(cfg.MODEL.WEIGHT) | |
| if args.weight: | |
| weight_iter = os.path.splitext(os.path.basename(args.weight))[0].split("_")[-1] | |
| try: | |
| weight_iter = int(weight_iter) | |
| except: | |
| weight_iter = 1 | |
| else: | |
| weight_iter = 1 | |
| # get the wandb name | |
| train_wandb_name = os.path.basename(cfg.OUTPUT_DIR) | |
| eval_wandb_name = train_wandb_name + "_eval" + "_Fixed{}_Chunk{}".format(not cfg.DATASETS.LVIS_USE_NORMAL_AP, cfg.TEST.CHUNKED_EVALUATION) | |
| if is_main_process() and train_wandb_name != "__test__": | |
| api = wandb.Api() | |
| runs = api.runs('haroldli/language_det_eval') | |
| matched_run = None | |
| history = [] | |
| exclude_keys = ['_runtime', '_timestamp'] | |
| for run in runs: | |
| if run.name == eval_wandb_name and str(run._state) == "finished": | |
| print("run found", run.name) | |
| print(run.summary) | |
| matched_run = run | |
| run_his = matched_run.scan_history() | |
| #print([len(i) for i in run_his]) | |
| for stat in run_his: | |
| stat_i = {k: v for k, v in stat.items() if k not in exclude_keys and v is not None} | |
| if len(stat_i) > 1: | |
| history.append(stat_i) | |
| #matched_run.delete() | |
| break # only update one | |
| wandb_run = wandb.init( | |
| project = 'language_det_eval', | |
| job_type = 'evaluate', | |
| name = eval_wandb_name, | |
| ) | |
| #pprint(history) | |
| # exclude_keys = ['_step', '_runtime', '_timestamp'] | |
| # for stat in history: | |
| # wandb.log( | |
| # {k: v for k, v in stat.items() if k not in exclude_keys}, | |
| # step = stat['_step'], | |
| # ) | |
| else: | |
| wandb_run = None | |
| history = None | |
| print("weight_iter: ", weight_iter) | |
| print("train_wandb_name: ", train_wandb_name) | |
| print("eval_wandb_name: ", eval_wandb_name) | |
| # build tokenizer to process data | |
| # tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") | |
| if cfg.MODEL.LANGUAGE_BACKBONE.TOKENIZER_TYPE == "bert-base-uncased": | |
| tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") | |
| elif cfg.MODEL.LANGUAGE_BACKBONE.TOKENIZER_TYPE == "roberta-base": | |
| tokenizer = AutoTokenizer.from_pretrained("roberta-base") | |
| elif cfg.MODEL.LANGUAGE_BACKBONE.TOKENIZER_TYPE == "clip": | |
| if cfg.MODEL.DYHEAD.FUSE_CONFIG.MLM_LOSS: | |
| tokenizer = CLIPTokenizerFast.from_pretrained( | |
| "openai/clip-vit-base-patch32", from_slow=True, mask_token="ðŁĴij</w>" | |
| ) | |
| else: | |
| tokenizer = CLIPTokenizerFast.from_pretrained("openai/clip-vit-base-patch32", from_slow=True) | |
| else: | |
| tokenizer = None | |
| raise NotImplementedError | |
| ### inference & evaluation | |
| topk_per_eval = args.topk_per_eval | |
| threshold = args.threshold | |
| model.eval() | |
| chunk_size = args.chunk_size # num of texts each time | |
| if cfg.MODEL.RPN_ARCHITECTURE == "VLDYHEAD": | |
| class_plus = 1 | |
| else: | |
| class_plus = 0 | |
| task_config = args.task_config | |
| assert task_config is not None, "task_config should be assigned" | |
| cfg_ = cfg.clone() | |
| cfg_.defrost() | |
| cfg_.merge_from_file(task_config) | |
| cfg_.merge_from_list(args.opts) | |
| dataset_name = cfg_.DATASETS.TEST[0] | |
| output_folder = os.path.join(log_dir, "inference", dataset_name) | |
| if not os.path.exists(output_folder): | |
| mkdir(output_folder) | |
| data_loaders_val = make_data_loader(cfg_, is_train=False, is_distributed=distributed) | |
| _iterator = tqdm(data_loaders_val[0]) # only for the first test set | |
| predictions = [] | |
| # adhoclly | |
| # if "coco" in cfg_.DATASETS.TEST[0]: | |
| # gt_json = 'DATASET/omnilabel/dataset_all_val_v0.1.3_coco.json' | |
| # elif "oi_v5" in cfg_.DATASETS.TEST[0]: | |
| # gt_json = 'DATASET/omnilabel/dataset_all_val_v0.1.3_openimagesv5.json' | |
| # elif "oi_v6" in cfg_.DATASETS.TEST[0]: | |
| # gt_json = 'DATASET/omnilabel/dataset_all_val_v0.1.3_openimagesv6.json' | |
| # else: | |
| # assert(0) | |
| # omni_label = OmniLabel(path_json=gt_json) | |
| if args.noun_phrase_file is not None: | |
| try: | |
| noun_phrase = json.load(open(args.noun_phrase_file)) | |
| except: | |
| noun_phrase = {} | |
| print("No noun phrase file found, will generate one") | |
| llm = LLM(version="chat", prompt_file="tools/data_process/prompts/noun.v1.txt", temp=0.0) | |
| else: | |
| noun_phrase = {} | |
| # stats | |
| pos_rates = [] | |
| query_length = [] | |
| all_info = [] | |
| for iidx, batch in enumerate(_iterator): | |
| images, targets, image_ids, *_ = batch | |
| # import ipdb | |
| # ipdb.set_trace() | |
| images = images.to(device) | |
| text_queries = targets[0].get_field('inference_obj_descriptions') | |
| text_queries_ids = targets[0].get_field("inference_obj_description_ids") | |
| image_size = targets[0].size | |
| image_id = image_ids[0] | |
| # pdb.set_trace() | |
| #print(data_loaders_val[0].dataset.dataset_dicts[iidx]) | |
| #all_info.append(data_loaders_val[0].dataset.dataset_dicts[iidx]) | |
| # get the positive label if there is one | |
| try: | |
| positive_info = omni_label.get_image_sample(image_id) | |
| positive_instances = positive_info['instances'] | |
| positive_labels = [] | |
| for i in positive_instances: positive_labels.extend(i['description_ids']) | |
| positive_labels = list(set(positive_labels)) | |
| except: | |
| positive_labels = None | |
| des_id_start = 0 | |
| # rearrange the queries | |
| query_indexes = [i for i in range(len(text_queries_ids)) if num_of_words(text_queries[i]) > 2] | |
| cat_indexes = [i for i in range(len(text_queries_ids)) if num_of_words(text_queries[i]) <= 2] | |
| # rearrange the queries | |
| if args.group_query: | |
| text_queries_ids = [text_queries_ids[i] for i in query_indexes] + [text_queries_ids[i] for i in cat_indexes] | |
| text_queries = [text_queries[i] for i in query_indexes] + [text_queries[i] for i in cat_indexes] | |
| while des_id_start < len(text_queries_ids): | |
| # sinlge descriptions each time | |
| if args.group_query: | |
| if num_of_words(text_queries[des_id_start]) > 2: | |
| description_list = remove_full_stop(text_queries[des_id_start:des_id_start+8]) | |
| description_id_list = text_queries_ids[des_id_start:des_id_start+8] | |
| des_id_start += 8 | |
| else: | |
| description_list = remove_full_stop(text_queries[des_id_start:des_id_start+chunk_size]) | |
| description_id_list = text_queries_ids[des_id_start:des_id_start+chunk_size] | |
| des_id_start += chunk_size | |
| else: | |
| if num_of_words(text_queries[des_id_start]) > 2: | |
| _det_phrase = True | |
| description_list = remove_full_stop([text_queries[des_id_start]]) | |
| description_id_list = [text_queries_ids[des_id_start]] | |
| des_id_start += 1 | |
| else: | |
| _det_phrase = False | |
| description_list = remove_full_stop(text_queries[des_id_start:des_id_start+chunk_size]) | |
| description_id_list = text_queries_ids[des_id_start:des_id_start+chunk_size] | |
| des_id_start += chunk_size | |
| # create postive map, always use continuous labels starting from 1 | |
| continue_labels = np.arange(0, chunk_size) + class_plus | |
| if _det_phrase and args.noun_phrase_file is not None: | |
| # try to find the centern noun phrase | |
| center_noun = noun_phrase.get(description_list[0], None) | |
| if center_noun is None: | |
| center_noun = llm(description_list[0]) | |
| if len(center_noun) == 0: | |
| center_noun = description_list[0] # failed case | |
| noun_phrase[description_list[0]] = center_noun | |
| start = description_list[0].lower().find(center_noun.lower()) | |
| end = start + len(center_noun) | |
| override_tokens_positive = [(start, end)] | |
| print(description_list[0], center_noun, override_tokens_positive) | |
| cur_queries, positive_map_label_to_token = create_queries_and_maps(continue_labels, description_list, tokenizer, cfg=cfg, override_tokens_positive=override_tokens_positive) | |
| else: | |
| cur_queries, positive_map_label_to_token = create_queries_and_maps(continue_labels, description_list, tokenizer, cfg=cfg) | |
| set_description_id_list = set(description_id_list) | |
| # intersection between positive labels and current description ids | |
| if positive_labels is not None: | |
| pos_rate = len(set_description_id_list.intersection(set(positive_labels))) / len(set_description_id_list) | |
| pos_rates.append(pos_rate) | |
| query_length.append(len(set_description_id_list)) | |
| # print(cur_queries) | |
| with torch.no_grad(): | |
| output = model(images, captions=[cur_queries], positive_map=positive_map_label_to_token) | |
| output = output[0].to(cpu_device).convert(mode="xywh") | |
| output = output.resize(image_size) # to the oringinal scale | |
| # print(output) | |
| # import ipdb | |
| # ipdb.set_trace() | |
| # thresolding | |
| if threshold is not None: | |
| scores = output.get_field('scores') | |
| output = output[scores > threshold] | |
| # sorted by scores | |
| if topk_per_eval is not None: | |
| scores = output.get_field('scores') | |
| _, sortIndices = scores.sort(descending=True) | |
| output = output[sortIndices] | |
| # topk | |
| output = output[:topk_per_eval] | |
| # map continuous id to description id | |
| cont_ids_2_descript_ids = {i:v for i, v in enumerate(description_id_list)} | |
| pred_boxes = output.bbox | |
| pred_labels = output.get_field('labels') - class_plus # continuous ids, starting from 0 | |
| pred_scores = output.get_field('scores') | |
| # convert continuous id to description id | |
| for box_idx, box in enumerate(pred_boxes): | |
| predictions.append({ | |
| "image_id": image_id, | |
| "bbox": box.cpu().tolist(), | |
| "description_ids": [cont_ids_2_descript_ids[pred_labels[box_idx].item()]], | |
| "scores": [pred_scores[box_idx].item()], | |
| }) | |
| #print("pos_rate: %.2f"%(np.mean(pos_rates)), pos_rates) | |
| #print("query_length: %.2f"%(np.mean(query_length)), query_length) | |
| # draw a histogram of pos_rate | |
| plt.hist(pos_rates, bins=10) | |
| plt.savefig(os.path.join(output_folder, "pos_rate.png")) | |
| plt.close() | |
| if args.noun_phrase_file is not None: | |
| with open(args.noun_phrase_file, "w") as f: | |
| json.dump(noun_phrase, f, indent=4) | |
| # collect predictions from all GPUs | |
| synchronize() | |
| all_predictions = all_gather(predictions) | |
| all_predictions = list(itertools.chain(*all_predictions)) | |
| if not is_main_process(): | |
| return | |
| result_save_json = "%s_results.json"%(dataset_name) | |
| results_path = os.path.join(output_folder, result_save_json) | |
| print('Saving to', results_path) | |
| json.dump(all_predictions, open(results_path, 'w')) | |
| from maskrcnn_benchmark.config.paths_catalog import DatasetCatalog | |
| datasetMeta = DatasetCatalog.get(dataset_name) | |
| gt_path_json = datasetMeta['args']['ann_file'] | |
| # import ipdb | |
| # ipdb.set_trace() | |
| # evaluation | |
| gt = OmniLabel(gt_path_json) # load ground truth dataset | |
| dt = gt.load_res(results_path) # load prediction results | |
| ole = OmniLabelEval(gt, dt) | |
| # ole.params.resThrs = ... # set evaluation parameters as desired | |
| ole.evaluate() | |
| ole.accumulate() | |
| score = ole.summarize() | |
| # OUTPUTS/GLIP_MODEL17/eval/model_0270000/inference/omnilabel_val/omnilabel_val_results.json | |
| #with open("tools/files/omnilabel_coco.json", "a") as f: | |
| # json.dump(all_info, f) | |
| if is_main_process(): | |
| if wandb_run is not None: | |
| # | |
| dataset_name = cfg.DATASETS.TEST[0] | |
| write_to_wandb_log(score, dataset_name, weight_iter, history) | |
| with open("{}/detailed.json".format(output_folder), "w") as f: | |
| json.dump(score, f) | |
| wandb_run.save("{}/detailed.json".format(output_folder)) | |
| print(score) | |
| def write_to_wandb_log(score, dataset_name, weight_iter, history): | |
| all_results = defaultdict(dict) | |
| exclude_keys = ['_step', '_runtime', '_timestamp'] | |
| if history is not None: | |
| for stat in history: | |
| all_results[stat['_step']].update({k: v for k, v in stat.items() if k not in exclude_keys}) | |
| result_dict = {} | |
| for score_i in score: | |
| if score_i["metric"]['metric'] == "AP" and score_i["metric"]['iou'] == "0.50:0.95" and score_i["metric"]['area'] == "all": | |
| result_dict[f"{dataset_name}_AP_{score_i['metric']['description']}"] = score_i['value'] | |
| #wandb.log({f"{dataset_name}_mAP_all": mAP_all, f"{dataset_name}_mAP_rare": mAP_rare, f"{dataset_name}_mAP_common": mAP_common, f"{dataset_name}_mAP_frequent": mAP_frequent}, step = weight_iter) | |
| all_results[weight_iter].update(result_dict) | |
| # sort all results | |
| max_key = max(all_results.keys()) | |
| for i in range(max_key + 1): | |
| if i in all_results: | |
| wandb.log(all_results[i], step = i) | |
| else: | |
| wandb.log({}, step = i) | |
| # for k in sorted(all_results.keys()): | |
| # # need to do consecutive logging | |
| # wandb.log(all_results[k], step = k) | |
| if __name__ == "__main__": | |
| main() | |
| ''' | |
| from omnilabeltools import OmniLabel, OmniLabelEval | |
| gt = OmniLabel('DATASET/omnilabel/dataset_all_val_v0.1.3_openimagesv5.json') # load ground truth dataset | |
| dt = gt.load_res("OUTPUTS/GLIP_MODEL17/eval/model_0270000/inference/omnilabel_val/omnilabel_val_results.json") # load prediction results | |
| ole = OmniLabelEval(gt, dt) | |
| ole.evaluate() | |
| ole.accumulate() | |
| ole.summarize() | |
| gt = OmniLabel('DATASET/omnilabel/dataset_all_val_v0.1.3_coco.json') # load ground truth dataset | |
| dt = gt.load_res("OUTPUTS/GLIP_MODEL17/eval/model_0270000/inference/omnilabel_val/omnilabel_val_results.json") | |
| gt = OmniLabel('DATASET/omnilabel/dataset_all_val_v0.1.3_object365.json') # load ground truth dataset | |
| dt = gt.load_res("OUTPUTS/GLIP_MODEL17/eval/model_0270000/inference/omnilabel_val/omnilabel_val_results.json") | |
| ''' | |