Spaces:
Sleeping
Sleeping
__author__ = 'tsungyi' | |
import numpy as np | |
import datetime | |
import time | |
from collections import defaultdict | |
from . import mask as maskUtils | |
import copy | |
class COCOeval: | |
# Interface for evaluating detection on the Microsoft COCO dataset. | |
# | |
# The usage for CocoEval is as follows: | |
# cocoGt=..., cocoDt=... # load dataset and results | |
# E = CocoEval(cocoGt,cocoDt); # initialize CocoEval object | |
# E.params.recThrs = ...; # set parameters as desired | |
# E.evaluate(); # run per image evaluation | |
# E.accumulate(); # accumulate per image results | |
# E.summarize(); # display summary metrics of results | |
# For example usage see evalDemo.m and http://mscoco.org/. | |
# | |
# The evaluation parameters are as follows (defaults in brackets): | |
# imgIds - [all] N img ids to use for evaluation | |
# catIds - [all] K cat ids to use for evaluation | |
# iouThrs - [.5:.05:.95] T=10 IoU thresholds for evaluation | |
# recThrs - [0:.01:1] R=101 recall thresholds for evaluation | |
# areaRng - [...] A=4 object area ranges for evaluation | |
# maxDets - [1 10 100] M=3 thresholds on max detections per image | |
# iouType - ['segm'] set iouType to 'segm', 'bbox' or 'keypoints' | |
# iouType replaced the now DEPRECATED useSegm parameter. | |
# useCats - [1] if true use category labels for evaluation | |
# Note: if useCats=0 category labels are ignored as in proposal scoring. | |
# Note: multiple areaRngs [Ax2] and maxDets [Mx1] can be specified. | |
# | |
# evaluate(): evaluates detections on every image and every category and | |
# concats the results into the "evalImgs" with fields: | |
# dtIds - [1xD] id for each of the D detections (dt) | |
# gtIds - [1xG] id for each of the G ground truths (gt) | |
# dtMatches - [TxD] matching gt id at each IoU or 0 | |
# gtMatches - [TxG] matching dt id at each IoU or 0 | |
# dtScores - [1xD] confidence of each dt | |
# gtIgnore - [1xG] ignore flag for each gt | |
# dtIgnore - [TxD] ignore flag for each dt at each IoU | |
# | |
# accumulate(): accumulates the per-image, per-category evaluation | |
# results in "evalImgs" into the dictionary "eval" with fields: | |
# params - parameters used for evaluation | |
# date - date evaluation was performed | |
# counts - [T,R,K,A,M] parameter dimensions (see above) | |
# precision - [TxRxKxAxM] precision for every evaluation setting | |
# recall - [TxKxAxM] max recall for every evaluation setting | |
# Note: precision and recall==-1 for settings with no gt objects. | |
# | |
# See also coco, mask, pycocoDemo, pycocoEvalDemo | |
# | |
# Microsoft COCO Toolbox. version 2.0 | |
# Data, paper, and tutorials available at: http://mscoco.org/ | |
# Code written by Piotr Dollar and Tsung-Yi Lin, 2015. | |
# Licensed under the Simplified BSD License [see coco/license.txt] | |
def __init__(self, cocoGt=None, cocoDt=None, iouType='segm'): | |
''' | |
Initialize CocoEval using coco APIs for gt and dt | |
:param cocoGt: coco object with ground truth annotations | |
:param cocoDt: coco object with detection results | |
:return: None | |
''' | |
if not iouType: | |
print('iouType not specified. use default iouType segm') | |
self.cocoGt = cocoGt # ground truth COCO API | |
self.cocoDt = cocoDt # detections COCO API | |
self.params = {} # evaluation parameters | |
self.evalImgs = defaultdict(list) # per-image per-category evaluation results [KxAxI] elements | |
self.eval = {} # accumulated evaluation results | |
self._gts = defaultdict(list) # gt for evaluation | |
self._dts = defaultdict(list) # dt for evaluation | |
self.params = Params(iouType=iouType) # parameters | |
self._paramsEval = {} # parameters for evaluation | |
self.stats = [] # result summarization | |
self.ious = {} # ious between all gts and dts | |
if not cocoGt is None: | |
self.params.imgIds = sorted(cocoGt.getImgIds()) | |
self.params.catIds = sorted(cocoGt.getCatIds()) | |
def _prepare(self): | |
''' | |
Prepare ._gts and ._dts for evaluation based on params | |
:return: None | |
''' | |
def _toMask(anns, coco): | |
# modify ann['segmentation'] by reference | |
for ann in anns: | |
rle = coco.annToRLE(ann) | |
ann['segmentation'] = rle | |
p = self.params | |
if p.useCats: | |
gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) | |
dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) | |
else: | |
gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds)) | |
dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds)) | |
# convert ground truth to mask if iouType == 'segm' | |
if p.iouType == 'segm': | |
_toMask(gts, self.cocoGt) | |
_toMask(dts, self.cocoDt) | |
# set ignore flag | |
for gt in gts: | |
gt['ignore'] = gt['ignore'] if 'ignore' in gt else 0 | |
gt['ignore'] = 'iscrowd' in gt and gt['iscrowd'] | |
if p.iouType == 'keypoints': | |
gt['ignore'] = (gt['num_keypoints'] == 0) or gt['ignore'] | |
self._gts = defaultdict(list) # gt for evaluation | |
self._dts = defaultdict(list) # dt for evaluation | |
for gt in gts: | |
self._gts[gt['image_id'], gt['category_id']].append(gt) | |
for dt in dts: | |
self._dts[dt['image_id'], dt['category_id']].append(dt) | |
self.evalImgs = defaultdict(list) # per-image per-category evaluation results | |
self.eval = {} # accumulated evaluation results | |
def evaluate(self): | |
''' | |
Run per image evaluation on given images and store results (a list of dict) in self.evalImgs | |
:return: None | |
''' | |
tic = time.time() | |
print('Running per image evaluation...') | |
p = self.params | |
# add backward compatibility if useSegm is specified in params | |
if not p.useSegm is None: | |
p.iouType = 'segm' if p.useSegm == 1 else 'bbox' | |
print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType)) | |
print('Evaluate annotation type *{}*'.format(p.iouType)) | |
p.imgIds = list(np.unique(p.imgIds)) | |
if p.useCats: | |
p.catIds = list(np.unique(p.catIds)) | |
p.maxDets = sorted(p.maxDets) | |
self.params=p | |
self._prepare() | |
# loop through images, area range, max detection number | |
catIds = p.catIds if p.useCats else [-1] | |
if p.iouType == 'segm' or p.iouType == 'bbox': | |
computeIoU = self.computeIoU | |
elif p.iouType == 'keypoints': | |
computeIoU = self.computeOks | |
self.ious = {(imgId, catId): computeIoU(imgId, catId) \ | |
for imgId in p.imgIds | |
for catId in catIds} | |
evaluateImg = self.evaluateImg | |
maxDet = p.maxDets[-1] | |
self.evalImgs = [evaluateImg(imgId, catId, areaRng, maxDet) | |
for catId in catIds | |
for areaRng in p.areaRng | |
for imgId in p.imgIds | |
] | |
self._paramsEval = copy.deepcopy(self.params) | |
toc = time.time() | |
print('DONE (t={:0.2f}s).'.format(toc-tic)) | |
def computeIoU(self, imgId, catId): | |
p = self.params | |
if p.useCats: | |
gt = self._gts[imgId,catId] | |
dt = self._dts[imgId,catId] | |
else: | |
gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]] | |
dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]] | |
if len(gt) == 0 and len(dt) ==0: | |
return [] | |
inds = np.argsort([-d['score'] for d in dt], kind='mergesort') | |
dt = [dt[i] for i in inds] | |
if len(dt) > p.maxDets[-1]: | |
dt=dt[0:p.maxDets[-1]] | |
if p.iouType == 'segm': | |
g = [g['segmentation'] for g in gt] | |
d = [d['segmentation'] for d in dt] | |
elif p.iouType == 'bbox': | |
g = [g['bbox'] for g in gt] | |
d = [d['bbox'] for d in dt] | |
else: | |
raise Exception('unknown iouType for iou computation') | |
# compute iou between each dt and gt region | |
iscrowd = [int(o['iscrowd']) for o in gt] | |
ious = maskUtils.iou(d,g,iscrowd) | |
return ious | |
def computeOks(self, imgId, catId): | |
p = self.params | |
# dimention here should be Nxm | |
gts = self._gts[imgId, catId] | |
dts = self._dts[imgId, catId] | |
inds = np.argsort([-d['score'] for d in dts], kind='mergesort') | |
dts = [dts[i] for i in inds] | |
if len(dts) > p.maxDets[-1]: | |
dts = dts[0:p.maxDets[-1]] | |
# if len(gts) == 0 and len(dts) == 0: | |
if len(gts) == 0 or len(dts) == 0: | |
return [] | |
ious = np.zeros((len(dts), len(gts))) | |
sigmas = np.array([.26, .25, .25, .35, .35, .79, .79, .72, .72, .62,.62, 1.07, 1.07, .87, .87, .89, .89])/10.0 | |
vars = (sigmas * 2)**2 | |
k = len(sigmas) | |
# compute oks between each detection and ground truth object | |
for j, gt in enumerate(gts): | |
# create bounds for ignore regions(double the gt bbox) | |
g = np.array(gt['keypoints']) | |
xg = g[0::3]; yg = g[1::3]; vg = g[2::3] | |
k1 = np.count_nonzero(vg > 0) | |
bb = gt['bbox'] | |
x0 = bb[0] - bb[2]; x1 = bb[0] + bb[2] * 2 | |
y0 = bb[1] - bb[3]; y1 = bb[1] + bb[3] * 2 | |
for i, dt in enumerate(dts): | |
d = np.array(dt['keypoints']) | |
xd = d[0::3]; yd = d[1::3] | |
if k1>0: | |
# measure the per-keypoint distance if keypoints visible | |
dx = xd - xg | |
dy = yd - yg | |
else: | |
# measure minimum distance to keypoints in (x0,y0) & (x1,y1) | |
z = np.zeros((k)) | |
dx = np.max((z, x0-xd),axis=0)+np.max((z, xd-x1),axis=0) | |
dy = np.max((z, y0-yd),axis=0)+np.max((z, yd-y1),axis=0) | |
e = (dx**2 + dy**2) / vars / (gt['area']+np.spacing(1)) / 2 | |
if k1 > 0: | |
e=e[vg > 0] | |
ious[i, j] = np.sum(np.exp(-e)) / e.shape[0] | |
return ious | |
def evaluateImg(self, imgId, catId, aRng, maxDet): | |
''' | |
perform evaluation for single category and image | |
:return: dict (single image results) | |
''' | |
p = self.params | |
if p.useCats: | |
gt = self._gts[imgId,catId] | |
dt = self._dts[imgId,catId] | |
else: | |
gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]] | |
dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]] | |
if len(gt) == 0 and len(dt) ==0: | |
return None | |
for g in gt: | |
if g['ignore'] or (g['area']<aRng[0] or g['area']>aRng[1]): | |
g['_ignore'] = 1 | |
else: | |
g['_ignore'] = 0 | |
# sort dt highest score first, sort gt ignore last | |
gtind = np.argsort([g['_ignore'] for g in gt], kind='mergesort') | |
gt = [gt[i] for i in gtind] | |
dtind = np.argsort([-d['score'] for d in dt], kind='mergesort') | |
dt = [dt[i] for i in dtind[0:maxDet]] | |
iscrowd = [int(o['iscrowd']) for o in gt] | |
# load computed ious | |
ious = self.ious[imgId, catId][:, gtind] if len(self.ious[imgId, catId]) > 0 else self.ious[imgId, catId] | |
T = len(p.iouThrs) | |
G = len(gt) | |
D = len(dt) | |
gtm = np.zeros((T,G)) | |
dtm = np.zeros((T,D)) | |
gtIg = np.array([g['_ignore'] for g in gt]) | |
dtIg = np.zeros((T,D)) | |
if not len(ious)==0: | |
for tind, t in enumerate(p.iouThrs): | |
for dind, d in enumerate(dt): | |
# information about best match so far (m=-1 -> unmatched) | |
iou = min([t,1-1e-10]) | |
m = -1 | |
for gind, g in enumerate(gt): | |
# if this gt already matched, and not a crowd, continue | |
if gtm[tind,gind]>0 and not iscrowd[gind]: | |
continue | |
# if dt matched to reg gt, and on ignore gt, stop | |
if m>-1 and gtIg[m]==0 and gtIg[gind]==1: | |
break | |
# continue to next gt unless better match made | |
if ious[dind,gind] < iou: | |
continue | |
# if match successful and best so far, store appropriately | |
iou=ious[dind,gind] | |
m=gind | |
# if match made store id of match for both dt and gt | |
if m ==-1: | |
continue | |
dtIg[tind,dind] = gtIg[m] | |
dtm[tind,dind] = gt[m]['id'] | |
gtm[tind,m] = d['id'] | |
# set unmatched detections outside of area range to ignore | |
a = np.array([d['area']<aRng[0] or d['area']>aRng[1] for d in dt]).reshape((1, len(dt))) | |
dtIg = np.logical_or(dtIg, np.logical_and(dtm==0, np.repeat(a,T,0))) | |
# store results for given image and category | |
return { | |
'image_id': imgId, | |
'category_id': catId, | |
'aRng': aRng, | |
'maxDet': maxDet, | |
'dtIds': [d['id'] for d in dt], | |
'gtIds': [g['id'] for g in gt], | |
'dtMatches': dtm, | |
'gtMatches': gtm, | |
'dtScores': [d['score'] for d in dt], | |
'gtIgnore': gtIg, | |
'dtIgnore': dtIg, | |
} | |
def accumulate(self, p = None): | |
''' | |
Accumulate per image evaluation results and store the result in self.eval | |
:param p: input params for evaluation | |
:return: None | |
''' | |
print('Accumulating evaluation results...') | |
tic = time.time() | |
if not self.evalImgs: | |
print('Please run evaluate() first') | |
# allows input customized parameters | |
if p is None: | |
p = self.params | |
p.catIds = p.catIds if p.useCats == 1 else [-1] | |
T = len(p.iouThrs) | |
R = len(p.recThrs) | |
K = len(p.catIds) if p.useCats else 1 | |
A = len(p.areaRng) | |
M = len(p.maxDets) | |
precision = -np.ones((T,R,K,A,M)) # -1 for the precision of absent categories | |
recall = -np.ones((T,K,A,M)) | |
scores = -np.ones((T,R,K,A,M)) | |
# create dictionary for future indexing | |
_pe = self._paramsEval | |
catIds = _pe.catIds if _pe.useCats else [-1] | |
setK = set(catIds) | |
setA = set(map(tuple, _pe.areaRng)) | |
setM = set(_pe.maxDets) | |
setI = set(_pe.imgIds) | |
# get inds to evaluate | |
k_list = [n for n, k in enumerate(p.catIds) if k in setK] | |
m_list = [m for n, m in enumerate(p.maxDets) if m in setM] | |
a_list = [n for n, a in enumerate(map(lambda x: tuple(x), p.areaRng)) if a in setA] | |
i_list = [n for n, i in enumerate(p.imgIds) if i in setI] | |
I0 = len(_pe.imgIds) | |
A0 = len(_pe.areaRng) | |
# retrieve E at each category, area range, and max number of detections | |
for k, k0 in enumerate(k_list): | |
Nk = k0*A0*I0 | |
for a, a0 in enumerate(a_list): | |
Na = a0*I0 | |
for m, maxDet in enumerate(m_list): | |
E = [self.evalImgs[Nk + Na + i] for i in i_list] | |
E = [e for e in E if not e is None] | |
if len(E) == 0: | |
continue | |
dtScores = np.concatenate([e['dtScores'][0:maxDet] for e in E]) | |
# different sorting method generates slightly different results. | |
# mergesort is used to be consistent as Matlab implementation. | |
inds = np.argsort(-dtScores, kind='mergesort') | |
dtScoresSorted = dtScores[inds] | |
dtm = np.concatenate([e['dtMatches'][:,0:maxDet] for e in E], axis=1)[:,inds] | |
dtIg = np.concatenate([e['dtIgnore'][:,0:maxDet] for e in E], axis=1)[:,inds] | |
gtIg = np.concatenate([e['gtIgnore'] for e in E]) | |
npig = np.count_nonzero(gtIg==0 ) | |
if npig == 0: | |
continue | |
tps = np.logical_and( dtm, np.logical_not(dtIg) ) | |
fps = np.logical_and(np.logical_not(dtm), np.logical_not(dtIg) ) | |
tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float) | |
fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float) | |
for t, (tp, fp) in enumerate(zip(tp_sum, fp_sum)): | |
tp = np.array(tp) | |
fp = np.array(fp) | |
nd = len(tp) | |
rc = tp / npig | |
pr = tp / (fp+tp+np.spacing(1)) | |
q = np.zeros((R,)) | |
ss = np.zeros((R,)) | |
if nd: | |
recall[t,k,a,m] = rc[-1] | |
else: | |
recall[t,k,a,m] = 0 | |
# numpy is slow without cython optimization for accessing elements | |
# use python array gets significant speed improvement | |
pr = pr.tolist(); q = q.tolist() | |
for i in range(nd-1, 0, -1): | |
if pr[i] > pr[i-1]: | |
pr[i-1] = pr[i] | |
inds = np.searchsorted(rc, p.recThrs, side='left') | |
try: | |
for ri, pi in enumerate(inds): | |
q[ri] = pr[pi] | |
ss[ri] = dtScoresSorted[pi] | |
except: | |
pass | |
precision[t,:,k,a,m] = np.array(q) | |
scores[t,:,k,a,m] = np.array(ss) | |
self.eval = { | |
'params': p, | |
'counts': [T, R, K, A, M], | |
'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), | |
'precision': precision, | |
'recall': recall, | |
'scores': scores, | |
} | |
toc = time.time() | |
print('DONE (t={:0.2f}s).'.format( toc-tic)) | |
def summarize(self): | |
''' | |
Compute and display summary metrics for evaluation results. | |
Note this functin can *only* be applied on the default parameter setting | |
''' | |
def _summarize( ap=1, iouThr=None, areaRng='all', maxDets=100 ): | |
p = self.params | |
iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}' | |
titleStr = 'Average Precision' if ap == 1 else 'Average Recall' | |
typeStr = '(AP)' if ap==1 else '(AR)' | |
iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \ | |
if iouThr is None else '{:0.2f}'.format(iouThr) | |
aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] | |
mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] | |
if ap == 1: | |
# dimension of precision: [TxRxKxAxM] | |
s = self.eval['precision'] | |
# IoU | |
if iouThr is not None: | |
t = np.where(iouThr == p.iouThrs)[0] | |
s = s[t] | |
s = s[:,:,:,aind,mind] | |
else: | |
# dimension of recall: [TxKxAxM] | |
s = self.eval['recall'] | |
if iouThr is not None: | |
t = np.where(iouThr == p.iouThrs)[0] | |
s = s[t] | |
s = s[:,:,aind,mind] | |
if len(s[s>-1])==0: | |
mean_s = -1 | |
else: | |
mean_s = np.mean(s[s>-1]) | |
print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) | |
return mean_s | |
def _summarizeDets(): | |
stats = np.zeros((12,)) | |
stats[0] = _summarize(1) | |
stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2]) | |
stats[2] = _summarize(1, iouThr=.75, maxDets=self.params.maxDets[2]) | |
stats[3] = _summarize(1, areaRng='small', maxDets=self.params.maxDets[2]) | |
stats[4] = _summarize(1, areaRng='medium', maxDets=self.params.maxDets[2]) | |
stats[5] = _summarize(1, areaRng='large', maxDets=self.params.maxDets[2]) | |
stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) | |
stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) | |
stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) | |
stats[9] = _summarize(0, areaRng='small', maxDets=self.params.maxDets[2]) | |
stats[10] = _summarize(0, areaRng='medium', maxDets=self.params.maxDets[2]) | |
stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2]) | |
return stats | |
def _summarizeKps(): | |
stats = np.zeros((10,)) | |
stats[0] = _summarize(1, maxDets=20) | |
stats[1] = _summarize(1, maxDets=20, iouThr=.5) | |
stats[2] = _summarize(1, maxDets=20, iouThr=.75) | |
stats[3] = _summarize(1, maxDets=20, areaRng='medium') | |
stats[4] = _summarize(1, maxDets=20, areaRng='large') | |
stats[5] = _summarize(0, maxDets=20) | |
stats[6] = _summarize(0, maxDets=20, iouThr=.5) | |
stats[7] = _summarize(0, maxDets=20, iouThr=.75) | |
stats[8] = _summarize(0, maxDets=20, areaRng='medium') | |
stats[9] = _summarize(0, maxDets=20, areaRng='large') | |
return stats | |
if not self.eval: | |
raise Exception('Please run accumulate() first') | |
iouType = self.params.iouType | |
if iouType == 'segm' or iouType == 'bbox': | |
summarize = _summarizeDets | |
elif iouType == 'keypoints': | |
summarize = _summarizeKps | |
self.stats = summarize() | |
def __str__(self): | |
self.summarize() | |
class Params: | |
''' | |
Params for coco evaluation api | |
''' | |
def setDetParams(self): | |
self.imgIds = [] | |
self.catIds = [] | |
# np.arange causes trouble. the data point on arange is slightly larger than the true value | |
self.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True) | |
self.recThrs = np.linspace(.0, 1.00, np.round((1.00 - .0) / .01) + 1, endpoint=True) | |
self.maxDets = [1, 10, 100] | |
self.areaRng = [[0 ** 2, 1e5 ** 2], [0 ** 2, 32 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]] | |
self.areaRngLbl = ['all', 'small', 'medium', 'large'] | |
self.useCats = 1 | |
def setKpParams(self): | |
self.imgIds = [] | |
self.catIds = [] | |
# np.arange causes trouble. the data point on arange is slightly larger than the true value | |
self.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True) | |
self.recThrs = np.linspace(.0, 1.00, np.round((1.00 - .0) / .01) + 1, endpoint=True) | |
self.maxDets = [20] | |
self.areaRng = [[0 ** 2, 1e5 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]] | |
self.areaRngLbl = ['all', 'medium', 'large'] | |
self.useCats = 1 | |
def __init__(self, iouType='segm'): | |
if iouType == 'segm' or iouType == 'bbox': | |
self.setDetParams() | |
elif iouType == 'keypoints': | |
self.setKpParams() | |
else: | |
raise Exception('iouType not supported') | |
self.iouType = iouType | |
# useSegm is deprecated | |
self.useSegm = None |