import torch import torch.nn as nn import torch.nn.functional as F from lanet_utils import image_grid class ConvBlock(nn.Module): def __init__(self, in_channels, out_channels): super(ConvBlock, self).__init__() self.conv = nn.Sequential( nn.Conv2d( in_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False, ), nn.BatchNorm2d(out_channels), nn.ReLU(inplace=True), nn.Conv2d( out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False, ), nn.BatchNorm2d(out_channels), nn.ReLU(inplace=True), ) def forward(self, x): return self.conv(x) class DilationConv3x3(nn.Module): def __init__(self, in_channels, out_channels): super(DilationConv3x3, self).__init__() self.conv = nn.Conv2d( in_channels, out_channels, kernel_size=3, stride=1, padding=2, dilation=2, bias=False, ) self.bn = nn.BatchNorm2d(out_channels) def forward(self, x): x = self.conv(x) x = self.bn(x) return x class InterestPointModule(nn.Module): def __init__(self, is_test=False): super(InterestPointModule, self).__init__() self.is_test = is_test self.conv1 = ConvBlock(3, 32) self.conv2 = ConvBlock(32, 64) self.conv3 = ConvBlock(64, 128) self.conv4 = ConvBlock(128, 256) self.maxpool2x2 = nn.MaxPool2d(2, 2) # score head self.score_conv = nn.Conv2d( 256, 256, kernel_size=3, stride=1, padding=1, bias=False ) self.score_norm = nn.BatchNorm2d(256) self.score_out = nn.Conv2d(256, 3, kernel_size=3, stride=1, padding=1) self.softmax = nn.Softmax(dim=1) # location head self.loc_conv = nn.Conv2d( 256, 256, kernel_size=3, stride=1, padding=1, bias=False ) self.loc_norm = nn.BatchNorm2d(256) self.loc_out = nn.Conv2d(256, 2, kernel_size=3, stride=1, padding=1) # descriptor out self.des_conv2 = DilationConv3x3(64, 256) self.des_conv3 = DilationConv3x3(128, 256) # cross_head: self.shift_out = nn.Conv2d(256, 1, kernel_size=3, stride=1, padding=1) self.relu = nn.ReLU(inplace=True) def forward(self, x): B, _, H, W = x.shape x = self.conv1(x) x = self.maxpool2x2(x) x2 = self.conv2(x) x = self.maxpool2x2(x2) x3 = self.conv3(x) x = self.maxpool2x2(x3) x = self.conv4(x) B, _, Hc, Wc = x.shape # score head score_x = self.score_out(self.relu(self.score_norm(self.score_conv(x)))) aware = self.softmax(score_x[:, 0:2, :, :]) score = score_x[:, 2, :, :].unsqueeze(1).sigmoid() border_mask = torch.ones(B, Hc, Wc) border_mask[:, 0] = 0 border_mask[:, Hc - 1] = 0 border_mask[:, :, 0] = 0 border_mask[:, :, Wc - 1] = 0 border_mask = border_mask.unsqueeze(1) score = score * border_mask.to(score.device) # location head coord_x = self.relu(self.loc_norm(self.loc_conv(x))) coord_cell = self.loc_out(coord_x).tanh() shift_ratio = self.shift_out(coord_x).sigmoid() * 2.0 step = ((H / Hc) - 1) / 2.0 center_base = ( image_grid( B, Hc, Wc, dtype=coord_cell.dtype, device=coord_cell.device, ones=False, normalized=False, ).mul(H / Hc) + step ) coord_un = center_base.add(coord_cell.mul(shift_ratio * step)) coord = coord_un.clone() coord[:, 0] = torch.clamp(coord_un[:, 0], min=0, max=W - 1) coord[:, 1] = torch.clamp(coord_un[:, 1], min=0, max=H - 1) # descriptor block desc_block = [] desc_block.append(self.des_conv2(x2)) desc_block.append(self.des_conv3(x3)) desc_block.append(aware) if self.is_test: coord_norm = coord[:, :2].clone() coord_norm[:, 0] = (coord_norm[:, 0] / (float(W - 1) / 2.0)) - 1.0 coord_norm[:, 1] = (coord_norm[:, 1] / (float(H - 1) / 2.0)) - 1.0 coord_norm = coord_norm.permute(0, 2, 3, 1) desc2 = torch.nn.functional.grid_sample(desc_block[0], coord_norm) desc3 = torch.nn.functional.grid_sample(desc_block[1], coord_norm) aware = desc_block[2] desc = torch.mul(desc2, aware[:, 0, :, :]) + torch.mul( desc3, aware[:, 1, :, :] ) desc = desc.div( torch.unsqueeze(torch.norm(desc, p=2, dim=1), 1) ) # Divide by norm to normalize. return score, coord, desc return score, coord, desc_block class CorrespondenceModule(nn.Module): def __init__(self, match_type="dual_softmax"): super(CorrespondenceModule, self).__init__() self.match_type = match_type if self.match_type == "dual_softmax": self.temperature = 0.1 else: raise NotImplementedError() def forward(self, source_desc, target_desc): b, c, h, w = source_desc.size() source_desc = source_desc.div( torch.unsqueeze(torch.norm(source_desc, p=2, dim=1), 1) ).view(b, -1, h * w) target_desc = target_desc.div( torch.unsqueeze(torch.norm(target_desc, p=2, dim=1), 1) ).view(b, -1, h * w) if self.match_type == "dual_softmax": sim_mat = ( torch.einsum("bcm, bcn -> bmn", source_desc, target_desc) / self.temperature ) confidence_matrix = F.softmax(sim_mat, 1) * F.softmax(sim_mat, 2) else: raise NotImplementedError() return confidence_matrix