|
import torch.nn as nn |
|
import torch |
|
from einops import rearrange |
|
import numpy |
|
def index_points(device, points, idx): |
|
""" |
|
|
|
Input: |
|
points: input points data, [B, N, C] |
|
idx: sample index data, [B, S] |
|
Return: |
|
new_points:, indexed points data, [B, S, C] |
|
""" |
|
B = points.shape[0] |
|
view_shape = list(idx.shape) |
|
view_shape[1:] = [1] * (len(view_shape) - 1) |
|
repeat_shape = list(idx.shape) |
|
repeat_shape[0] = 1 |
|
|
|
batch_indices = torch.arange(B, dtype=torch.long).cuda().view(view_shape).repeat(repeat_shape) |
|
new_points = points[batch_indices, idx, :] |
|
return new_points |
|
|
|
def knn_l2(device, net, k, u): |
|
''' |
|
Input: |
|
k: int32, number of k in k-nn search |
|
net: (batch_size, npoint, c) float32 array, points |
|
u: int32, block size |
|
Output: |
|
idx: (batch_size, npoint, k) int32 array, indices to input points |
|
''' |
|
INF = 1e8 |
|
batch_size = net.size(0) |
|
npoint = net.size(1) |
|
n_channel = net.size(2) |
|
|
|
square = torch.pow(torch.norm(net, dim=2,keepdim=True),2) |
|
|
|
def u_block(batch_size, npoint, u): |
|
block = numpy.zeros([batch_size, npoint, npoint]) |
|
n = npoint // u |
|
for i in range(n): |
|
block[:, (i*u):(i*u+u), (i*u):(i*u+u)] = numpy.ones([batch_size, u, u]) * (-INF) |
|
return block |
|
|
|
|
|
minus_distance = 2 * torch.matmul(net, net.transpose(2,1)) - square - square.transpose(2,1) + torch.Tensor(u_block(batch_size, npoint, u)).cuda() |
|
_, indices = torch.topk(minus_distance, k, largest=True, sorted=False) |
|
|
|
return indices |
|
|
|
class Residual(nn.Module): |
|
def __init__(self, fn): |
|
super().__init__() |
|
self.fn = fn |
|
def forward(self, x, **kwargs): |
|
return self.fn(x, **kwargs) + x |
|
|
|
class PreNorm(nn.Module): |
|
def __init__(self, dim, fn): |
|
super().__init__() |
|
self.norm = nn.LayerNorm(dim) |
|
self.fn = fn |
|
def forward(self, x, **kwargs): |
|
return self.fn(self.norm(x), **kwargs) |
|
|
|
class FeedForward(nn.Module): |
|
def __init__(self, dim, hidden_dim, dropout = 0.): |
|
super().__init__() |
|
self.net = nn.Sequential( |
|
nn.Linear(dim, hidden_dim), |
|
nn.GELU(), |
|
nn.Dropout(dropout), |
|
nn.Linear(hidden_dim, dim), |
|
nn.Dropout(dropout) |
|
) |
|
def forward(self, x): |
|
return self.net(x) |
|
|
|
class Attention(nn.Module): |
|
def __init__(self, dim, heads = 4, dropout = 0.): |
|
super().__init__() |
|
self.heads = heads |
|
self.scale = dim ** -0.5 |
|
|
|
self.to_qkv = nn.Linear(dim, dim * 3, bias = False) |
|
self.to_out = nn.Sequential( |
|
nn.Linear(dim, dim), |
|
nn.Dropout(dropout) |
|
) |
|
|
|
def forward(self, x, mask = None): |
|
b, n, _, h = *x.shape, self.heads |
|
qkv = self.to_qkv(x).chunk(3, dim = -1) |
|
q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = h), qkv) |
|
|
|
dots = torch.einsum('bhid,bhjd->bhij', q, k) * self.scale |
|
|
|
if mask is not None: |
|
mask = F.pad(mask.flatten(1), (1, 0), value = True) |
|
assert mask.shape[-1] == dots.shape[-1], 'mask has incorrect dimensions' |
|
mask = mask[:, None, :] * mask[:, :, None] |
|
dots.masked_fill_(~mask, float('-inf')) |
|
del mask |
|
|
|
attn = dots.softmax(dim=-1) |
|
|
|
out = torch.einsum('bhij,bhjd->bhid', attn, v) |
|
out = rearrange(out, 'b h n d -> b n (h d)') |
|
out = self.to_out(out) |
|
return out |
|
|
|
class Local_Attention(nn.Module): |
|
def __init__(self, dim, heads = 4,knn=4, dropout = 0.): |
|
super().__init__() |
|
self.heads = heads |
|
self.scale = dim ** -0.5 |
|
|
|
|
|
self.q=nn.Linear(dim,dim,bias=False) |
|
self.k=nn.Linear(dim,dim,bias=False) |
|
self.v=nn.Linear(dim,dim,bias=False) |
|
|
|
self.to_out = nn.Sequential( |
|
nn.Linear(dim, dim), |
|
nn.Dropout(dropout) |
|
) |
|
self.knn=knn |
|
|
|
def forward(self, x, mask = None): |
|
b, n, _, h = *x.shape, self.heads |
|
|
|
point=x*1 |
|
X=x*1 |
|
|
|
idx = knn_l2(0, point.permute(0,2,1), 4, 1) |
|
feat=idx |
|
new_point = index_points(0, point.permute(0,2,1),idx) |
|
|
|
group_point = new_point.permute(0, 3, 2, 1) |
|
|
|
_1,_2,_3,_4=group_point.shape |
|
|
|
q=self.q(X.reshape(_1*_2,1,_4)) |
|
k=self.k(torch.cat([group_point.reshape(_1*_2,self.knn,_4),X.reshape(_1*_2,1,_4)],dim=1)) |
|
v=self.v(torch.cat([group_point.reshape(_1*_2,self.knn,_4),X.reshape(_1*_2,1,_4)],dim=1)) |
|
q, k, v = rearrange(q, 'b n (h d) -> b h n d', h = h),rearrange(k, 'b n (h d) -> b h n d', h = h),rearrange(v, 'b n (h d) -> b h n d', h = h) |
|
|
|
attn_map=q@k.permute(0,1,3,2)*self.scale |
|
attn_map=attn_map.softmax(dim=-1) |
|
|
|
out=attn_map@v |
|
out=out.view(b,out.shape[0]//b,out.shape[1],out.shape[3]).permute(0,2,1,3) |
|
|
|
out = rearrange(out, 'b h n d -> b n (h d)') |
|
out = self.to_out(out) |
|
return out |
|
|
|
|
|
class Transformer(nn.Module): |
|
def __init__(self, dim, depth, heads, mlp_dim, group=5, dropout=0.): |
|
super().__init__() |
|
self.layers = nn.ModuleList([]) |
|
for _ in range(depth): |
|
self.layers.append(nn.ModuleList([ |
|
Residual(PreNorm(dim, Attention(dim, heads = heads, dropout = dropout))), |
|
Residual(PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout))) |
|
])) |
|
self.group=group |
|
def forward(self, x, mask = None): |
|
bs_gp,dim,wid,hei=x.shape[0],x.shape[1],x.shape[2],x.shape[3] |
|
bs=bs_gp//self.group |
|
gp=self.group |
|
x=x.reshape(bs,gp,dim,wid,hei) |
|
x=x.permute(0,1,3,4,2).reshape(bs,gp*wid*hei,dim) |
|
for attn, ff in self.layers: |
|
x = attn(x, mask = mask) |
|
x = ff(x) |
|
|
|
x=x.reshape(bs,gp,wid,hei,dim).permute(0,1,4,2,3).reshape(bs_gp,dim,wid,hei) |
|
|
|
return x |
|
|
|
class Transformer__local(nn.Module): |
|
def __init__(self, dim, depth, heads, mlp_dim,knn_k=4, group=5, dropout=0.): |
|
super().__init__() |
|
self.layers = nn.ModuleList([]) |
|
for _ in range(depth): |
|
self.layers.append(nn.ModuleList([ |
|
Residual(PreNorm(dim, Local_Attention(dim, heads = heads,knn=knn_k, dropout = dropout))), |
|
Residual(PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout))) |
|
])) |
|
self.group=group |
|
def forward(self, x, mask = None): |
|
bs_gp,dim,wid,hei=x.shape[0],x.shape[1],x.shape[2],x.shape[3] |
|
bs=bs_gp//self.group |
|
gp=self.group |
|
x=x.reshape(bs,gp,dim,wid,hei) |
|
x=x.permute(0,1,3,4,2).reshape(bs,gp*wid*hei,dim) |
|
for attn, ff in self.layers: |
|
x = attn(x, mask = mask) |
|
x = ff(x) |
|
|
|
x=x.reshape(bs,gp,wid,hei,dim).permute(0,1,4,2,3).reshape(bs_gp,dim,wid,hei) |
|
|
|
return x |
