Spaces:
Sleeping
Sleeping
| # -------------------------------------------------------- | |
| # Python Single Object Tracking Evaluation | |
| # Licensed under The MIT License [see LICENSE for details] | |
| # Written by Fangyi Zhang | |
| # @author fangyi.zhang@vipl.ict.ac.cn | |
| # @project https://github.com/StrangerZhang/pysot-toolkit.git | |
| # Revised for SiamMask by foolwood | |
| # -------------------------------------------------------- | |
| import warnings | |
| import itertools | |
| import numpy as np | |
| from colorama import Style, Fore | |
| from ..utils import calculate_failures, calculate_accuracy | |
| class AccuracyRobustnessBenchmark: | |
| """ | |
| Args: | |
| dataset: | |
| burnin: | |
| """ | |
| def __init__(self, dataset, burnin=10): | |
| self.dataset = dataset | |
| self.burnin = burnin | |
| def eval(self, eval_trackers=None): | |
| """ | |
| Args: | |
| eval_tags: list of tag | |
| eval_trackers: list of tracker name | |
| Returns: | |
| ret: dict of results | |
| """ | |
| if eval_trackers is None: | |
| eval_trackers = self.dataset.tracker_names | |
| if isinstance(eval_trackers, str): | |
| eval_trackers = [eval_trackers] | |
| result = {} | |
| for tracker_name in eval_trackers: | |
| accuracy, failures = self._calculate_accuracy_robustness(tracker_name) | |
| result[tracker_name] = {'overlaps': accuracy, | |
| 'failures': failures} | |
| return result | |
| def show_result(self, result, eao_result=None, show_video_level=False, helight_threshold=0.5): | |
| """pretty print result | |
| Args: | |
| result: returned dict from function eval | |
| """ | |
| tracker_name_len = max((max([len(x) for x in result.keys()])+2), 12) | |
| if eao_result is not None: | |
| header = "|{:^"+str(tracker_name_len)+"}|{:^10}|{:^12}|{:^13}|{:^7}|" | |
| header = header.format('Tracker Name', | |
| 'Accuracy', 'Robustness', 'Lost Number', 'EAO') | |
| formatter = "|{:^"+str(tracker_name_len)+"}|{:^10.3f}|{:^12.3f}|{:^13.1f}|{:^7.3f}|" | |
| else: | |
| header = "|{:^"+str(tracker_name_len)+"}|{:^10}|{:^12}|{:^13}|" | |
| header = header.format('Tracker Name', | |
| 'Accuracy', 'Robustness', 'Lost Number') | |
| formatter = "|{:^"+str(tracker_name_len)+"}|{:^10.3f}|{:^12.3f}|{:^13.1f}|" | |
| bar = '-'*len(header) | |
| print(bar) | |
| print(header) | |
| print(bar) | |
| if eao_result is not None: | |
| tracker_eao = sorted(eao_result.items(), | |
| key=lambda x:x[1]['all'], | |
| reverse=True)[:20] | |
| tracker_names = [x[0] for x in tracker_eao] | |
| else: | |
| tracker_names = list(result.keys()) | |
| for tracker_name in tracker_names: | |
| ret = result[tracker_name] | |
| overlaps = list(itertools.chain(*ret['overlaps'].values())) | |
| accuracy = np.nanmean(overlaps) | |
| length = sum([len(x) for x in ret['overlaps'].values()]) | |
| failures = list(ret['failures'].values()) | |
| lost_number = np.mean(np.sum(failures, axis=0)) | |
| robustness = np.mean(np.sum(np.array(failures), axis=0) / length) * 100 | |
| if eao_result is None: | |
| print(formatter.format(tracker_name, accuracy, robustness, lost_number)) | |
| else: | |
| print(formatter.format(tracker_name, accuracy, robustness, lost_number, eao_result[tracker_name]['all'])) | |
| print(bar) | |
| if show_video_level and len(result) < 10: | |
| print('\n\n') | |
| header1 = "|{:^14}|".format("Tracker name") | |
| header2 = "|{:^14}|".format("Video name") | |
| for tracker_name in result.keys(): | |
| header1 += ("{:^17}|").format(tracker_name) | |
| header2 += "{:^8}|{:^8}|".format("Acc", "LN") | |
| print('-'*len(header1)) | |
| print(header1) | |
| print('-'*len(header1)) | |
| print(header2) | |
| print('-'*len(header1)) | |
| videos = list(result[tracker_name]['overlaps'].keys()) | |
| for video in videos: | |
| row = "|{:^14}|".format(video) | |
| for tracker_name in result.keys(): | |
| overlaps = result[tracker_name]['overlaps'][video] | |
| accuracy = np.nanmean(overlaps) | |
| failures = result[tracker_name]['failures'][video] | |
| lost_number = np.mean(failures) | |
| accuracy_str = "{:^8.3f}".format(accuracy) | |
| if accuracy < helight_threshold: | |
| row += f'{Fore.RED}{accuracy_str}{Style.RESET_ALL}|' | |
| else: | |
| row += accuracy_str+'|' | |
| lost_num_str = "{:^8.3f}".format(lost_number) | |
| if lost_number > 0: | |
| row += f'{Fore.RED}{lost_num_str}{Style.RESET_ALL}|' | |
| else: | |
| row += lost_num_str+'|' | |
| print(row) | |
| print('-'*len(header1)) | |
| def _calculate_accuracy_robustness(self, tracker_name): | |
| overlaps = {} | |
| failures = {} | |
| all_length = {} | |
| for i in range(len(self.dataset)): | |
| video = self.dataset[i] | |
| gt_traj = video.gt_traj | |
| if tracker_name not in video.pred_trajs: | |
| tracker_trajs = video.load_tracker(self.dataset.tracker_path, tracker_name, False) | |
| else: | |
| tracker_trajs = video.pred_trajs[tracker_name] | |
| overlaps_group = [] | |
| num_failures_group = [] | |
| for tracker_traj in tracker_trajs: | |
| num_failures = calculate_failures(tracker_traj)[0] | |
| overlaps_ = calculate_accuracy(tracker_traj, gt_traj, | |
| burnin=10, bound=(video.width, video.height))[1] | |
| overlaps_group.append(overlaps_) | |
| num_failures_group.append(num_failures) | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore", category=RuntimeWarning) | |
| overlaps[video.name] = np.nanmean(overlaps_group, axis=0).tolist() | |
| failures[video.name] = num_failures_group | |
| return overlaps, failures | |