Vincentqyw
fix: roma
8b973ee
raw
history blame
6.81 kB
import torch
import torch.nn as nn
import time
eps = 1e-8
def sinkhorn(M, r, c, iteration):
p = torch.softmax(M, dim=-1)
u = torch.ones_like(r)
v = torch.ones_like(c)
for _ in range(iteration):
u = r / ((p * v.unsqueeze(-2)).sum(-1) + eps)
v = c / ((p * u.unsqueeze(-1)).sum(-2) + eps)
p = p * u.unsqueeze(-1) * v.unsqueeze(-2)
return p
def sink_algorithm(M, dustbin, iteration):
M = torch.cat([M, dustbin.expand([M.shape[0], M.shape[1], 1])], dim=-1)
M = torch.cat([M, dustbin.expand([M.shape[0], 1, M.shape[2]])], dim=-2)
r = torch.ones([M.shape[0], M.shape[1] - 1], device="cuda")
r = torch.cat([r, torch.ones([M.shape[0], 1], device="cuda") * M.shape[1]], dim=-1)
c = torch.ones([M.shape[0], M.shape[2] - 1], device="cuda")
c = torch.cat([c, torch.ones([M.shape[0], 1], device="cuda") * M.shape[2]], dim=-1)
p = sinkhorn(M, r, c, iteration)
return p
class attention_block(nn.Module):
def __init__(self, channels, head, type):
assert type == "self" or type == "cross", "invalid attention type"
nn.Module.__init__(self)
self.head = head
self.type = type
self.head_dim = channels // head
self.query_filter = nn.Conv1d(channels, channels, kernel_size=1)
self.key_filter = nn.Conv1d(channels, channels, kernel_size=1)
self.value_filter = nn.Conv1d(channels, channels, kernel_size=1)
self.attention_filter = nn.Sequential(
nn.Conv1d(2 * channels, 2 * channels, kernel_size=1),
nn.SyncBatchNorm(2 * channels),
nn.ReLU(),
nn.Conv1d(2 * channels, channels, kernel_size=1),
)
self.mh_filter = nn.Conv1d(channels, channels, kernel_size=1)
def forward(self, fea1, fea2):
batch_size, n, m = fea1.shape[0], fea1.shape[2], fea2.shape[2]
query1, key1, value1 = (
self.query_filter(fea1).view(batch_size, self.head_dim, self.head, -1),
self.key_filter(fea1).view(batch_size, self.head_dim, self.head, -1),
self.value_filter(fea1).view(batch_size, self.head_dim, self.head, -1),
)
query2, key2, value2 = (
self.query_filter(fea2).view(batch_size, self.head_dim, self.head, -1),
self.key_filter(fea2).view(batch_size, self.head_dim, self.head, -1),
self.value_filter(fea2).view(batch_size, self.head_dim, self.head, -1),
)
if self.type == "self":
score1, score2 = torch.softmax(
torch.einsum("bdhn,bdhm->bhnm", query1, key1) / self.head_dim**0.5,
dim=-1,
), torch.softmax(
torch.einsum("bdhn,bdhm->bhnm", query2, key2) / self.head_dim**0.5,
dim=-1,
)
add_value1, add_value2 = torch.einsum(
"bhnm,bdhm->bdhn", score1, value1
), torch.einsum("bhnm,bdhm->bdhn", score2, value2)
else:
score1, score2 = torch.softmax(
torch.einsum("bdhn,bdhm->bhnm", query1, key2) / self.head_dim**0.5,
dim=-1,
), torch.softmax(
torch.einsum("bdhn,bdhm->bhnm", query2, key1) / self.head_dim**0.5,
dim=-1,
)
add_value1, add_value2 = torch.einsum(
"bhnm,bdhm->bdhn", score1, value2
), torch.einsum("bhnm,bdhm->bdhn", score2, value1)
add_value1, add_value2 = self.mh_filter(
add_value1.contiguous().view(batch_size, self.head * self.head_dim, n)
), self.mh_filter(
add_value2.contiguous().view(batch_size, self.head * self.head_dim, m)
)
fea11, fea22 = torch.cat([fea1, add_value1], dim=1), torch.cat(
[fea2, add_value2], dim=1
)
fea1, fea2 = fea1 + self.attention_filter(fea11), fea2 + self.attention_filter(
fea22
)
return fea1, fea2
class matcher(nn.Module):
def __init__(self, config):
nn.Module.__init__(self)
self.use_score_encoding = config.use_score_encoding
self.layer_num = config.layer_num
self.sink_iter = config.sink_iter
self.position_encoder = nn.Sequential(
nn.Conv1d(3, 32, kernel_size=1)
if config.use_score_encoding
else nn.Conv1d(2, 32, kernel_size=1),
nn.SyncBatchNorm(32),
nn.ReLU(),
nn.Conv1d(32, 64, kernel_size=1),
nn.SyncBatchNorm(64),
nn.ReLU(),
nn.Conv1d(64, 128, kernel_size=1),
nn.SyncBatchNorm(128),
nn.ReLU(),
nn.Conv1d(128, 256, kernel_size=1),
nn.SyncBatchNorm(256),
nn.ReLU(),
nn.Conv1d(256, config.net_channels, kernel_size=1),
)
self.dustbin = nn.Parameter(torch.tensor(1, dtype=torch.float32, device="cuda"))
self.self_attention_block = nn.Sequential(
*[
attention_block(config.net_channels, config.head, "self")
for _ in range(config.layer_num)
]
)
self.cross_attention_block = nn.Sequential(
*[
attention_block(config.net_channels, config.head, "cross")
for _ in range(config.layer_num)
]
)
self.final_project = nn.Conv1d(
config.net_channels, config.net_channels, kernel_size=1
)
def forward(self, data, test_mode=True):
desc1, desc2 = data["desc1"], data["desc2"]
desc1, desc2 = torch.nn.functional.normalize(
desc1, dim=-1
), torch.nn.functional.normalize(desc2, dim=-1)
desc1, desc2 = desc1.transpose(1, 2), desc2.transpose(1, 2)
if test_mode:
encode_x1, encode_x2 = data["x1"], data["x2"]
else:
encode_x1, encode_x2 = data["aug_x1"], data["aug_x2"]
if not self.use_score_encoding:
encode_x1, encode_x2 = encode_x1[:, :, :2], encode_x2[:, :, :2]
encode_x1, encode_x2 = encode_x1.transpose(1, 2), encode_x2.transpose(1, 2)
x1_pos_embedding, x2_pos_embedding = self.position_encoder(
encode_x1
), self.position_encoder(encode_x2)
aug_desc1, aug_desc2 = x1_pos_embedding + desc1, x2_pos_embedding + desc2
for i in range(self.layer_num):
aug_desc1, aug_desc2 = self.self_attention_block[i](aug_desc1, aug_desc2)
aug_desc1, aug_desc2 = self.cross_attention_block[i](aug_desc1, aug_desc2)
aug_desc1, aug_desc2 = self.final_project(aug_desc1), self.final_project(
aug_desc2
)
desc_mat = torch.matmul(aug_desc1.transpose(1, 2), aug_desc2)
p = sink_algorithm(desc_mat, self.dustbin, self.sink_iter[0])
return {"p": p}