|
""" |
|
@date: 2021/8/4 |
|
@description: |
|
""" |
|
import numpy as np |
|
import cv2 |
|
import scipy |
|
|
|
from evaluation.f1_score import f1_score_2d |
|
from loss import GradLoss |
|
from utils.boundary import corners2boundaries, layout2depth |
|
from utils.conversion import depth2xyz, uv2xyz, get_u, depth2uv, xyz2uv, uv2pixel |
|
from utils.height import calc_ceil_ratio |
|
from evaluation.iou import calc_IoU, calc_Iou_height |
|
from visualization.boundary import draw_boundaries |
|
from visualization.floorplan import draw_iou_floorplan |
|
from visualization.grad import show_grad |
|
|
|
|
|
def calc_accuracy(dt, gt, visualization=False, h=512): |
|
visb_iou_2ds = [] |
|
visb_iou_3ds = [] |
|
full_iou_2ds = [] |
|
full_iou_3ds = [] |
|
iou_heights = [] |
|
|
|
visb_iou_floodplans = [] |
|
full_iou_floodplans = [] |
|
pano_bds = [] |
|
|
|
if 'depth' not in dt.keys(): |
|
dt['depth'] = gt['depth'] |
|
|
|
for i in range(len(gt['depth'])): |
|
|
|
dt_xyz = dt['processed_xyz'][i] if 'processed_xyz' in dt else depth2xyz(np.abs(dt['depth'][i])) |
|
visb_gt_xyz = depth2xyz(np.abs(gt['depth'][i])) |
|
corners = gt['corners'][i] |
|
full_gt_corners = corners[corners[..., 0] + corners[..., 1] != 0] |
|
full_gt_xyz = uv2xyz(full_gt_corners) |
|
|
|
dt_xz = dt_xyz[..., ::2] |
|
visb_gt_xz = visb_gt_xyz[..., ::2] |
|
full_gt_xz = full_gt_xyz[..., ::2] |
|
|
|
gt_ratio = gt['ratio'][i][0] |
|
|
|
if 'ratio' not in dt.keys(): |
|
if 'boundary' in dt.keys(): |
|
w = len(dt['boundary'][i]) |
|
boundary = np.clip(dt['boundary'][i], 0.0001, 0.4999) |
|
depth = np.clip(dt['depth'][i], 0.001, 9999) |
|
dt_ceil_boundary = np.concatenate([get_u(w, is_np=True)[..., None], boundary], axis=-1) |
|
dt_floor_boundary = depth2uv(depth) |
|
dt_ratio = calc_ceil_ratio(boundaries=[dt_ceil_boundary, dt_floor_boundary]) |
|
else: |
|
dt_ratio = gt_ratio |
|
else: |
|
dt_ratio = dt['ratio'][i][0] |
|
|
|
visb_iou_2d, visb_iou_3d = calc_IoU(dt_xz, visb_gt_xz, dt_height=1 + dt_ratio, gt_height=1 + gt_ratio) |
|
full_iou_2d, full_iou_3d = calc_IoU(dt_xz, full_gt_xz, dt_height=1 + dt_ratio, gt_height=1 + gt_ratio) |
|
iou_height = calc_Iou_height(dt_height=1 + dt_ratio, gt_height=1 + gt_ratio) |
|
|
|
visb_iou_2ds.append(visb_iou_2d) |
|
visb_iou_3ds.append(visb_iou_3d) |
|
full_iou_2ds.append(full_iou_2d) |
|
full_iou_3ds.append(full_iou_3d) |
|
iou_heights.append(iou_height) |
|
|
|
if visualization: |
|
pano_img = cv2.resize(gt['image'][i].transpose(1, 2, 0), (h*2, h)) |
|
|
|
|
|
visb_iou_floodplans.append(draw_iou_floorplan(dt_xz, visb_gt_xz, side_l=h)) |
|
full_iou_floodplans.append(draw_iou_floorplan(dt_xz, full_gt_xz, side_l=h)) |
|
gt_boundaries = corners2boundaries(gt_ratio, corners_xyz=full_gt_xyz, step=None, length=1024, visible=False) |
|
dt_boundaries = corners2boundaries(dt_ratio, corners_xyz=dt_xyz, step=None, visible=False, |
|
length=1024) |
|
|
|
pano_bd = draw_boundaries(pano_img, boundary_list=gt_boundaries, boundary_color=[0, 0, 1]) |
|
pano_bd = draw_boundaries(pano_bd, boundary_list=dt_boundaries, boundary_color=[0, 1, 0]) |
|
pano_bds.append(pano_bd) |
|
|
|
visb_iou_2d = np.array(visb_iou_2ds).mean() |
|
visb_iou_3d = np.array(visb_iou_3ds).mean() |
|
full_iou_2d = np.array(full_iou_2ds).mean() |
|
full_iou_3d = np.array(full_iou_3ds).mean() |
|
iou_height = np.array(iou_heights).mean() |
|
|
|
if visualization: |
|
visb_iou_floodplans = np.array(visb_iou_floodplans).transpose(0, 3, 1, 2) |
|
full_iou_floodplans = np.array(full_iou_floodplans).transpose(0, 3, 1, 2) |
|
pano_bds = np.array(pano_bds).transpose(0, 3, 1, 2) |
|
return [visb_iou_2d, visb_iou_3d, visb_iou_floodplans],\ |
|
[full_iou_2d, full_iou_3d, full_iou_floodplans], iou_height, pano_bds, full_iou_2ds |
|
|
|
|
|
def calc_ce(dt, gt): |
|
w = 1024 |
|
h = 512 |
|
ce_s = [] |
|
for i in range(len(gt['corners'])): |
|
floor_gt_corners = gt['corners'][i] |
|
|
|
floor_gt_corners = floor_gt_corners[floor_gt_corners[..., 0] + floor_gt_corners[..., 1] != 0] |
|
floor_gt_corners = np.roll(floor_gt_corners, -np.argmin(floor_gt_corners[..., 0]), 0) |
|
gt_ratio = gt['ratio'][i][0] |
|
ceil_gt_corners = corners2boundaries(gt_ratio, corners_uv=floor_gt_corners, step=None)[1] |
|
gt_corners = np.concatenate((floor_gt_corners, ceil_gt_corners)) |
|
gt_corners = uv2pixel(gt_corners, w, h) |
|
|
|
floor_dt_corners = xyz2uv(dt['processed_xyz'][i]) |
|
floor_dt_corners = np.roll(floor_dt_corners, -np.argmin(floor_dt_corners[..., 0]), 0) |
|
dt_ratio = dt['ratio'][i][0] |
|
ceil_dt_corners = corners2boundaries(dt_ratio, corners_uv=floor_dt_corners, step=None)[1] |
|
dt_corners = np.concatenate((floor_dt_corners, ceil_dt_corners)) |
|
dt_corners = uv2pixel(dt_corners, w, h) |
|
|
|
mse = np.sqrt(((gt_corners - dt_corners) ** 2).sum(1)).mean() |
|
ce = 100 * mse / np.sqrt(w ** 2 + h ** 2) |
|
ce_s.append(ce) |
|
|
|
return np.array(ce_s).mean() |
|
|
|
|
|
def calc_pe(dt, gt): |
|
w = 1024 |
|
h = 512 |
|
pe_s = [] |
|
for i in range(len(gt['corners'])): |
|
floor_gt_corners = gt['corners'][i] |
|
|
|
floor_gt_corners = floor_gt_corners[floor_gt_corners[..., 0] + floor_gt_corners[..., 1] != 0] |
|
floor_gt_corners = np.roll(floor_gt_corners, -np.argmin(floor_gt_corners[..., 0]), 0) |
|
gt_ratio = gt['ratio'][i][0] |
|
gt_floor_boundary, gt_ceil_boundary = corners2boundaries(gt_ratio, corners_uv=floor_gt_corners, length=w) |
|
gt_floor_boundary = uv2pixel(gt_floor_boundary, w, h) |
|
gt_ceil_boundary = uv2pixel(gt_ceil_boundary, w, h) |
|
|
|
floor_dt_corners = xyz2uv(dt['processed_xyz'][i]) |
|
floor_dt_corners = np.roll(floor_dt_corners, -np.argmin(floor_dt_corners[..., 0]), 0) |
|
dt_ratio = dt['ratio'][i][0] |
|
dt_floor_boundary, dt_ceil_boundary = corners2boundaries(dt_ratio, corners_uv=floor_dt_corners, length=w) |
|
dt_floor_boundary = uv2pixel(dt_floor_boundary, w, h) |
|
dt_ceil_boundary = uv2pixel(dt_ceil_boundary, w, h) |
|
|
|
gt_surface = np.zeros((h, w), dtype=np.int32) |
|
gt_surface[gt_ceil_boundary[..., 1], np.arange(w)] = 1 |
|
gt_surface[gt_floor_boundary[..., 1], np.arange(w)] = 1 |
|
gt_surface = np.cumsum(gt_surface, axis=0) |
|
|
|
dt_surface = np.zeros((h, w), dtype=np.int32) |
|
dt_surface[dt_ceil_boundary[..., 1], np.arange(w)] = 1 |
|
dt_surface[dt_floor_boundary[..., 1], np.arange(w)] = 1 |
|
dt_surface = np.cumsum(dt_surface, axis=0) |
|
|
|
pe = 100 * (dt_surface != gt_surface).sum() / (h * w) |
|
pe_s.append(pe) |
|
return np.array(pe_s).mean() |
|
|
|
|
|
def calc_rmse_delta_1(dt, gt): |
|
rmse_s = [] |
|
delta_1_s = [] |
|
for i in range(len(gt['depth'])): |
|
gt_boundaries = corners2boundaries(gt['ratio'][i], corners_xyz=depth2xyz(gt['depth'][i]), step=None, |
|
visible=False) |
|
dt_xyz = dt['processed_xyz'][i] if 'processed_xyz' in dt else depth2xyz(np.abs(dt['depth'][i])) |
|
|
|
dt_boundaries = corners2boundaries(dt['ratio'][i], corners_xyz=dt_xyz, step=None, |
|
length=256 if 'processed_xyz' in dt else None, |
|
visible=True if 'processed_xyz' in dt else False) |
|
gt_layout_depth = layout2depth(gt_boundaries, show=False) |
|
dt_layout_depth = layout2depth(dt_boundaries, show=False) |
|
|
|
rmse = ((gt_layout_depth - dt_layout_depth) ** 2).mean() ** 0.5 |
|
threshold = np.maximum(gt_layout_depth / dt_layout_depth, dt_layout_depth / gt_layout_depth) |
|
delta_1 = (threshold < 1.25).mean() |
|
rmse_s.append(rmse) |
|
delta_1_s.append(delta_1) |
|
return np.array(rmse_s).mean(), np.array(delta_1_s).mean() |
|
|
|
|
|
def calc_f1_score(dt, gt, threshold=10): |
|
w = 1024 |
|
h = 512 |
|
f1_s = [] |
|
precision_s = [] |
|
recall_s = [] |
|
for i in range(len(gt['corners'])): |
|
floor_gt_corners = gt['corners'][i] |
|
|
|
floor_gt_corners = floor_gt_corners[floor_gt_corners[..., 0] + floor_gt_corners[..., 1] != 0] |
|
floor_gt_corners = np.roll(floor_gt_corners, -np.argmin(floor_gt_corners[..., 0]), 0) |
|
gt_ratio = gt['ratio'][i][0] |
|
ceil_gt_corners = corners2boundaries(gt_ratio, corners_uv=floor_gt_corners, step=None)[1] |
|
gt_corners = np.concatenate((floor_gt_corners, ceil_gt_corners)) |
|
gt_corners = uv2pixel(gt_corners, w, h) |
|
|
|
floor_dt_corners = xyz2uv(dt['processed_xyz'][i]) |
|
floor_dt_corners = np.roll(floor_dt_corners, -np.argmin(floor_dt_corners[..., 0]), 0) |
|
dt_ratio = dt['ratio'][i][0] |
|
ceil_dt_corners = corners2boundaries(dt_ratio, corners_uv=floor_dt_corners, step=None)[1] |
|
dt_corners = np.concatenate((floor_dt_corners, ceil_dt_corners)) |
|
dt_corners = uv2pixel(dt_corners, w, h) |
|
|
|
Fs, Ps, Rs = f1_score_2d(gt_corners, dt_corners, [threshold]) |
|
f1_s.append(Fs[0]) |
|
precision_s.append(Ps[0]) |
|
recall_s.append(Rs[0]) |
|
|
|
return np.array(f1_s).mean(), np.array(precision_s).mean(), np.array(recall_s).mean() |
|
|
|
|
|
def show_heat_map(dt, gt, vis_w=1024): |
|
dt_heat_map = dt['corner_heat_map'].detach().cpu().numpy() |
|
gt_heat_map = gt['corner_heat_map'].detach().cpu().numpy() |
|
dt_heat_map_imgs = [] |
|
gt_heat_map_imgs = [] |
|
for i in range(len(gt['depth'])): |
|
dt_heat_map_img = dt_heat_map[..., np.newaxis].repeat(3, axis=-1).repeat(20, axis=0) |
|
gt_heat_map_img = gt_heat_map[..., np.newaxis].repeat(3, axis=-1).repeat(20, axis=0) |
|
dt_heat_map_imgs.append(cv2.resize(dt_heat_map_img, (vis_w, dt_heat_map_img.shape[0])).transpose(2, 0, 1)) |
|
gt_heat_map_imgs.append(cv2.resize(gt_heat_map_img, (vis_w, dt_heat_map_img.shape[0])).transpose(2, 0, 1)) |
|
return dt_heat_map_imgs, gt_heat_map_imgs |
|
|
|
|
|
def show_depth_normal_grad(dt, gt, device, vis_w=1024): |
|
grad_conv = GradLoss().to(device).grad_conv |
|
gt_grad_imgs = [] |
|
dt_grad_imgs = [] |
|
|
|
if 'depth' not in dt.keys(): |
|
dt['depth'] = gt['depth'] |
|
|
|
if vis_w == 1024: |
|
h = 5 |
|
else: |
|
h = int(vis_w / (12 * 10)) |
|
|
|
for i in range(len(gt['depth'])): |
|
gt_grad_img = show_grad(gt['depth'][i], grad_conv, h) |
|
dt_grad_img = show_grad(dt['depth'][i], grad_conv, h) |
|
vis_h = dt_grad_img.shape[0] * (vis_w // dt_grad_img.shape[1]) |
|
gt_grad_imgs.append(cv2.resize(gt_grad_img, (vis_w, vis_h), interpolation=cv2.INTER_NEAREST).transpose(2, 0, 1)) |
|
dt_grad_imgs.append(cv2.resize(dt_grad_img, (vis_w, vis_h), interpolation=cv2.INTER_NEAREST).transpose(2, 0, 1)) |
|
|
|
return gt_grad_imgs, dt_grad_imgs |
|
|
