import torch import numpy as np def batch_episym(x1, x2, F): batch_size, num_pts = x1.shape[0], x1.shape[1] x1 = torch.cat([x1, x1.new_ones(batch_size, num_pts, 1)], dim=-1).reshape( batch_size, num_pts, 3, 1 ) x2 = torch.cat([x2, x2.new_ones(batch_size, num_pts, 1)], dim=-1).reshape( batch_size, num_pts, 3, 1 ) F = F.reshape(-1, 1, 3, 3).repeat(1, num_pts, 1, 1) x2Fx1 = torch.matmul(x2.transpose(2, 3), torch.matmul(F, x1)).reshape( batch_size, num_pts ) Fx1 = torch.matmul(F, x1).reshape(batch_size, num_pts, 3) Ftx2 = torch.matmul(F.transpose(2, 3), x2).reshape(batch_size, num_pts, 3) ys = ( x2Fx1**2 * ( 1.0 / (Fx1[:, :, 0] ** 2 + Fx1[:, :, 1] ** 2 + 1e-15) + 1.0 / (Ftx2[:, :, 0] ** 2 + Ftx2[:, :, 1] ** 2 + 1e-15) ) ).sqrt() return ys def CELoss(seed_x1, seed_x2, e, confidence, inlier_th, batch_mask=1): # seed_x: b*k*2 ys = batch_episym(seed_x1, seed_x2, e) mask_pos, mask_neg = (ys <= inlier_th).float(), (ys > inlier_th).float() num_pos, num_neg = ( torch.relu(torch.sum(mask_pos, dim=1) - 1.0) + 1.0, torch.relu(torch.sum(mask_neg, dim=1) - 1.0) + 1.0, ) loss_pos, loss_neg = ( -torch.log(abs(confidence) + 1e-8) * mask_pos, -torch.log(abs(1 - confidence) + 1e-8) * mask_neg, ) classif_loss = torch.mean( loss_pos * 0.5 / num_pos.unsqueeze(-1) + loss_neg * 0.5 / num_neg.unsqueeze(-1), dim=-1, ) classif_loss = classif_loss * batch_mask classif_loss = classif_loss.mean() precision = torch.mean( torch.sum((confidence > 0.5).type(confidence.type()) * mask_pos, dim=1) / (torch.sum((confidence > 0.5).type(confidence.type()), dim=1) + 1e-8) ) recall = torch.mean( torch.sum((confidence > 0.5).type(confidence.type()) * mask_pos, dim=1) / num_pos ) return classif_loss, precision, recall def CorrLoss(desc_mat, batch_num_corr, batch_num_incorr1, batch_num_incorr2): total_loss_corr, total_loss_incorr = 0, 0 total_acc_corr, total_acc_incorr = 0, 0 batch_size = desc_mat.shape[0] log_p = torch.log(abs(desc_mat) + 1e-8) for i in range(batch_size): cur_log_p = log_p[i] num_corr = batch_num_corr[i] num_incorr1, num_incorr2 = batch_num_incorr1[i], batch_num_incorr2[i] # loss and acc loss_corr = -torch.diag(cur_log_p)[:num_corr].mean() loss_incorr = ( -cur_log_p[num_corr : num_corr + num_incorr1, -1].mean() - cur_log_p[-1, num_corr : num_corr + num_incorr2].mean() ) / 2 value_row, row_index = torch.max(desc_mat[i, :-1, :-1], dim=-1) value_col, col_index = torch.max(desc_mat[i, :-1, :-1], dim=-2) acc_incorr = ( (value_row[num_corr : num_corr + num_incorr1] < 0.2).float().mean() + (value_col[num_corr : num_corr + num_incorr2] < 0.2).float().mean() ) / 2 acc_row_mask = row_index[:num_corr] == torch.arange(num_corr).cuda() acc_col_mask = col_index[:num_corr] == torch.arange(num_corr).cuda() acc = (acc_col_mask & acc_row_mask).float().mean() total_loss_corr += loss_corr total_loss_incorr += loss_incorr total_acc_corr += acc total_acc_incorr += acc_incorr total_acc_corr /= batch_size total_acc_incorr /= batch_size total_loss_corr /= batch_size total_loss_incorr /= batch_size return total_loss_corr, total_loss_incorr, total_acc_corr, total_acc_incorr class SGMLoss: def __init__(self, config, model_config): self.config = config self.model_config = model_config def run(self, data, result): loss_corr, loss_incorr, acc_corr, acc_incorr = CorrLoss( result["p"], data["num_corr"], data["num_incorr1"], data["num_incorr2"] ) loss_mid_corr_tower, loss_mid_incorr_tower, acc_mid_tower = [], [], [] # mid loss for i in range(len(result["mid_p"])): mid_p = result["mid_p"][i] loss_mid_corr, loss_mid_incorr, mid_acc_corr, mid_acc_incorr = CorrLoss( mid_p, data["num_corr"], data["num_incorr1"], data["num_incorr2"] ) loss_mid_corr_tower.append(loss_mid_corr), loss_mid_incorr_tower.append( loss_mid_incorr ), acc_mid_tower.append(mid_acc_corr) if len(result["mid_p"]) != 0: loss_mid_corr_tower, loss_mid_incorr_tower, acc_mid_tower = ( torch.stack(loss_mid_corr_tower), torch.stack(loss_mid_incorr_tower), torch.stack(acc_mid_tower), ) else: loss_mid_corr_tower, loss_mid_incorr_tower, acc_mid_tower = ( torch.zeros(1).cuda(), torch.zeros(1).cuda(), torch.zeros(1).cuda(), ) # seed confidence loss classif_loss_tower, classif_precision_tower, classif_recall_tower = [], [], [] for layer in range(len(result["seed_conf"])): confidence = result["seed_conf"][layer] seed_index = result["seed_index"][ (np.asarray(self.model_config.seedlayer) <= layer).nonzero()[0][-1] ] seed_x1, seed_x2 = data["x1"].gather( dim=1, index=seed_index[:, :, 0, None].expand(-1, -1, 2) ), data["x2"].gather( dim=1, index=seed_index[:, :, 1, None].expand(-1, -1, 2) ) classif_loss, classif_precision, classif_recall = CELoss( seed_x1, seed_x2, data["e_gt"], confidence, self.config.inlier_th ) classif_loss_tower.append(classif_loss), classif_precision_tower.append( classif_precision ), classif_recall_tower.append(classif_recall) classif_loss, classif_precision_tower, classif_recall_tower = ( torch.stack(classif_loss_tower).mean(), torch.stack(classif_precision_tower), torch.stack(classif_recall_tower), ) classif_loss *= self.config.seed_loss_weight loss_mid_corr_tower *= self.config.mid_loss_weight loss_mid_incorr_tower *= self.config.mid_loss_weight total_loss = ( loss_corr + loss_incorr + classif_loss + loss_mid_corr_tower.sum() + loss_mid_incorr_tower.sum() ) return { "loss_corr": loss_corr, "loss_incorr": loss_incorr, "acc_corr": acc_corr, "acc_incorr": acc_incorr, "loss_seed_conf": classif_loss, "pre_seed_conf": classif_precision_tower, "recall_seed_conf": classif_recall_tower, "loss_corr_mid": loss_mid_corr_tower, "loss_incorr_mid": loss_mid_incorr_tower, "mid_acc_corr": acc_mid_tower, "total_loss": total_loss, } class SGLoss: def __init__(self, config, model_config): self.config = config self.model_config = model_config def run(self, data, result): loss_corr, loss_incorr, acc_corr, acc_incorr = CorrLoss( result["p"], data["num_corr"], data["num_incorr1"], data["num_incorr2"] ) total_loss = loss_corr + loss_incorr return { "loss_corr": loss_corr, "loss_incorr": loss_incorr, "acc_corr": acc_corr, "acc_incorr": acc_incorr, "total_loss": total_loss, }