|
import os |
|
import numpy as np |
|
import pickle |
|
import cv2 |
|
from metrics.new_utils import * |
|
|
|
|
|
class Metric(): |
|
def calc(self, gt_data, conv_data, thresh=8.0, iou_thresh=0.7): |
|
|
|
gts = gt_data['corners'] |
|
dets = conv_data['corners'] |
|
|
|
per_sample_corner_tp = 0.0 |
|
per_sample_corner_fp = 0.0 |
|
per_sample_corner_length = gts.shape[0] |
|
found = [False] * gts.shape[0] |
|
c_det_annot = {} |
|
|
|
|
|
|
|
for i, det in enumerate(dets): |
|
|
|
near_gt = [0, 999999.0, (0.0, 0.0)] |
|
for k, gt in enumerate(gts): |
|
dist = np.linalg.norm(gt - det) |
|
if dist < near_gt[1]: |
|
near_gt = [k, dist, gt] |
|
if near_gt[1] <= thresh and not found[near_gt[0]]: |
|
per_sample_corner_tp += 1.0 |
|
found[near_gt[0]] = True |
|
c_det_annot[i] = near_gt[0] |
|
else: |
|
per_sample_corner_fp += 1.0 |
|
|
|
per_corner_score = { |
|
'recall': per_sample_corner_tp / gts.shape[0], |
|
'precision': per_sample_corner_tp / (per_sample_corner_tp + per_sample_corner_fp + 1e-8) |
|
} |
|
|
|
|
|
per_sample_edge_tp = 0.0 |
|
per_sample_edge_fp = 0.0 |
|
edge_corner_annots = gt_data['edges'] |
|
per_sample_edge_length = edge_corner_annots.shape[0] |
|
|
|
false_edge_ids = [] |
|
match_gt_ids = set() |
|
|
|
for l, e_det in enumerate(conv_data['edges']): |
|
c1, c2 = e_det |
|
|
|
|
|
if (c1 not in c_det_annot.keys()) or (c2 not in c_det_annot.keys()): |
|
per_sample_edge_fp += 1.0 |
|
false_edge_ids.append(l) |
|
continue |
|
|
|
c1_prime = c_det_annot[c1] |
|
c2_prime = c_det_annot[c2] |
|
is_hit = False |
|
|
|
for k, e_annot in enumerate(edge_corner_annots): |
|
c3, c4 = e_annot |
|
if ((c1_prime == c3) and (c2_prime == c4)) or ((c1_prime == c4) and (c2_prime == c3)): |
|
is_hit = True |
|
match_gt_ids.add(k) |
|
break |
|
|
|
|
|
if is_hit: |
|
per_sample_edge_tp += 1.0 |
|
else: |
|
per_sample_edge_fp += 1.0 |
|
false_edge_ids.append(l) |
|
|
|
per_edge_score = { |
|
'recall': per_sample_edge_tp / edge_corner_annots.shape[0], |
|
'precision': per_sample_edge_tp / (per_sample_edge_tp + per_sample_edge_fp + 1e-8) |
|
} |
|
|
|
|
|
conv_mask = render(corners=conv_data['corners'], edges=conv_data['edges'], render_pad=0, edge_linewidth=1)[0] |
|
conv_mask = 1 - conv_mask |
|
conv_mask = conv_mask.astype(np.uint8) |
|
labels, region_mask = cv2.connectedComponents(conv_mask, connectivity=4) |
|
|
|
|
|
|
|
background_label = region_mask[0, 0] |
|
all_conv_masks = [] |
|
for region_i in range(1, labels): |
|
if region_i == background_label: |
|
continue |
|
the_region = region_mask == region_i |
|
if the_region.sum() < 20: |
|
continue |
|
all_conv_masks.append(the_region) |
|
|
|
gt_mask = render(corners=gt_data['corners'], edges=gt_data['edges'], render_pad=0, edge_linewidth=1)[0] |
|
gt_mask = 1 - gt_mask |
|
gt_mask = gt_mask.astype(np.uint8) |
|
labels, region_mask = cv2.connectedComponents(gt_mask, connectivity=4) |
|
|
|
|
|
|
|
background_label = region_mask[0, 0] |
|
all_gt_masks = [] |
|
for region_i in range(1, labels): |
|
if region_i == background_label: |
|
continue |
|
the_region = region_mask == region_i |
|
if the_region.sum() < 20: |
|
continue |
|
all_gt_masks.append(the_region) |
|
|
|
per_sample_region_tp = 0.0 |
|
per_sample_region_fp = 0.0 |
|
per_sample_region_length = len(all_gt_masks) |
|
found = [False] * len(all_gt_masks) |
|
for i, r_det in enumerate(all_conv_masks): |
|
|
|
near_gt = [0, 0, None] |
|
for k, r_gt in enumerate(all_gt_masks): |
|
iou = np.logical_and(r_gt, r_det).sum() / float(np.logical_or(r_gt, r_det).sum()) |
|
if iou > near_gt[1]: |
|
near_gt = [k, iou, r_gt] |
|
if near_gt[1] >= iou_thresh and not found[near_gt[0]]: |
|
per_sample_region_tp += 1.0 |
|
found[near_gt[0]] = True |
|
else: |
|
per_sample_region_fp += 1.0 |
|
|
|
per_region_score = { |
|
'recall': per_sample_region_tp / len(all_gt_masks), |
|
'precision': per_sample_region_tp / (per_sample_region_tp + per_sample_region_fp + 1e-8) |
|
} |
|
|
|
return { |
|
'corner_tp': per_sample_corner_tp, |
|
'corner_fp': per_sample_corner_fp, |
|
'corner_length': per_sample_corner_length, |
|
'edge_tp': per_sample_edge_tp, |
|
'edge_fp': per_sample_edge_fp, |
|
'edge_length': per_sample_edge_length, |
|
'region_tp': per_sample_region_tp, |
|
'region_fp': per_sample_region_fp, |
|
'region_length': per_sample_region_length, |
|
'corner': per_corner_score, |
|
'edge': per_edge_score, |
|
'region': per_region_score |
|
} |
|
|
|
|
|
def compute_metrics(gt_data, pred_data): |
|
metric = Metric() |
|
score = metric.calc(gt_data, pred_data) |
|
return score |
|
|
|
|
|
def get_recall_and_precision(tp, fp, length): |
|
recall = tp / (length + 1e-8) |
|
precision = tp / (tp + fp + 1e-8) |
|
return recall, precision |
|
|
|
|
|
if __name__ == '__main__': |
|
base_path = './' |
|
gt_datapath = '../data/cities_dataset/annot' |
|
metric = Metric() |
|
corner_tp = 0.0 |
|
corner_fp = 0.0 |
|
corner_length = 0.0 |
|
edge_tp = 0.0 |
|
edge_fp = 0.0 |
|
edge_length = 0.0 |
|
region_tp = 0.0 |
|
region_fp = 0.0 |
|
region_length = 0.0 |
|
for file_name in os.listdir(base_path): |
|
if len(file_name) < 10: |
|
continue |
|
f = open(os.path.join(base_path, file_name), 'rb') |
|
gt_data = np.load(os.path.join(gt_datapath, file_name + '.npy'), allow_pickle=True).tolist() |
|
candidate = pickle.load(f) |
|
conv_corners = candidate.graph.getCornersArray() |
|
conv_edges = candidate.graph.getEdgesArray() |
|
conv_data = {'corners': conv_corners, 'edges': conv_edges} |
|
score = metric.calc(gt_data, conv_data) |
|
corner_tp += score['corner_tp'] |
|
corner_fp += score['corner_fp'] |
|
corner_length += score['corner_length'] |
|
edge_tp += score['edge_tp'] |
|
edge_fp += score['edge_fp'] |
|
edge_length += score['edge_length'] |
|
region_tp += score['region_tp'] |
|
region_fp += score['region_fp'] |
|
region_length += score['region_length'] |
|
|
|
f = open(os.path.join(base_path, 'score.txt'), 'w') |
|
|
|
recall, precision = get_recall_and_precision(corner_tp, corner_fp, corner_length) |
|
f_score = 2.0 * precision * recall / (recall + precision + 1e-8) |
|
print('corners - precision: %.3f recall: %.3f f_score: %.3f' % (precision, recall, f_score)) |
|
f.write('corners - precision: %.3f recall: %.3f f_score: %.3f\n' % (precision, recall, f_score)) |
|
|
|
|
|
recall, precision = get_recall_and_precision(edge_tp, edge_fp, edge_length) |
|
f_score = 2.0 * precision * recall / (recall + precision + 1e-8) |
|
print('edges - precision: %.3f recall: %.3f f_score: %.3f' % (precision, recall, f_score)) |
|
f.write('edges - precision: %.3f recall: %.3f f_score: %.3f\n' % (precision, recall, f_score)) |
|
|
|
|
|
recall, precision = get_recall_and_precision(region_tp, region_fp, region_length) |
|
f_score = 2.0 * precision * recall / (recall + precision + 1e-8) |
|
print('regions - precision: %.3f recall: %.3f f_score: %.3f' % (precision, recall, f_score)) |
|
f.write('regions - precision: %.3f recall: %.3f f_score: %.3f\n' % (precision, recall, f_score)) |
|
|
|
f.close() |
|
|