from SiamMask.models.siammask_sharp import SiamMask from SiamMask.models.features import MultiStageFeature from SiamMask.models.rpn import RPN, DepthCorr from SiamMask.models.mask import Mask import torch import torch.nn as nn import torch.nn.functional as F from SiamMask.utils.load_helper import load_pretrain from SiamMask.experiments.siammask_sharp.resnet import resnet50 class ResDownS(nn.Module): def __init__(self, inplane, outplane): super(ResDownS, self).__init__() self.downsample = nn.Sequential( nn.Conv2d(inplane, outplane, kernel_size=1, bias=False), nn.BatchNorm2d(outplane)) def forward(self, x): x = self.downsample(x) if x.size(3) < 20: l = 4 r = -4 x = x[:, :, l:r, l:r] return x class ResDown(MultiStageFeature): def __init__(self, pretrain=False): super(ResDown, self).__init__() self.features = resnet50(layer3=True, layer4=False) if pretrain: load_pretrain(self.features, 'resnet.model') self.downsample = ResDownS(1024, 256) self.layers = [self.downsample, self.features.layer2, self.features.layer3] self.train_nums = [1, 3] self.change_point = [0, 0.5] self.unfix(0.0) def param_groups(self, start_lr, feature_mult=1): lr = start_lr * feature_mult def _params(module, mult=1): params = list(filter(lambda x:x.requires_grad, module.parameters())) if len(params): return [{'params': params, 'lr': lr * mult}] else: return [] groups = [] groups += _params(self.downsample) groups += _params(self.features, 0.1) return groups def forward(self, x): output = self.features(x) p3 = self.downsample(output[-1]) return p3 def forward_all(self, x): output = self.features(x) p3 = self.downsample(output[-1]) return output, p3 class UP(RPN): def __init__(self, anchor_num=5, feature_in=256, feature_out=256): super(UP, self).__init__() self.anchor_num = anchor_num self.feature_in = feature_in self.feature_out = feature_out self.cls_output = 2 * self.anchor_num self.loc_output = 4 * self.anchor_num self.cls = DepthCorr(feature_in, feature_out, self.cls_output) self.loc = DepthCorr(feature_in, feature_out, self.loc_output) def forward(self, z_f, x_f): cls = self.cls(z_f, x_f) loc = self.loc(z_f, x_f) return cls, loc class MaskCorr(Mask): def __init__(self, oSz=63): super(MaskCorr, self).__init__() self.oSz = oSz self.mask = DepthCorr(256, 256, self.oSz**2) def forward(self, z, x): return self.mask(z, x) class Refine(nn.Module): def __init__(self): super(Refine, self).__init__() self.v0 = nn.Sequential(nn.Conv2d(64, 16, 3, padding=1), nn.ReLU(), nn.Conv2d(16, 4, 3, padding=1),nn.ReLU()) self.v1 = nn.Sequential(nn.Conv2d(256, 64, 3, padding=1), nn.ReLU(), nn.Conv2d(64, 16, 3, padding=1), nn.ReLU()) self.v2 = nn.Sequential(nn.Conv2d(512, 128, 3, padding=1), nn.ReLU(), nn.Conv2d(128, 32, 3, padding=1), nn.ReLU()) self.h2 = nn.Sequential(nn.Conv2d(32, 32, 3, padding=1), nn.ReLU(), nn.Conv2d(32, 32, 3, padding=1), nn.ReLU()) self.h1 = nn.Sequential(nn.Conv2d(16, 16, 3, padding=1), nn.ReLU(), nn.Conv2d(16, 16, 3, padding=1), nn.ReLU()) self.h0 = nn.Sequential(nn.Conv2d(4, 4, 3, padding=1), nn.ReLU(), nn.Conv2d(4, 4, 3, padding=1), nn.ReLU()) self.deconv = nn.ConvTranspose2d(256, 32, 15, 15) self.post0 = nn.Conv2d(32, 16, 3, padding=1) self.post1 = nn.Conv2d(16, 4, 3, padding=1) self.post2 = nn.Conv2d(4, 1, 3, padding=1) for modules in [self.v0, self.v1, self.v2, self.h2, self.h1, self.h0, self.deconv, self.post0, self.post1, self.post2,]: for l in modules.modules(): if isinstance(l, nn.Conv2d): nn.init.kaiming_uniform_(l.weight, a=1) def forward(self, f, corr_feature, pos=None, test=False): if test: p0 = torch.nn.functional.pad(f[0], [16, 16, 16, 16])[:, :, 4*pos[0]:4*pos[0]+61, 4*pos[1]:4*pos[1]+61] p1 = torch.nn.functional.pad(f[1], [8, 8, 8, 8])[:, :, 2 * pos[0]:2 * pos[0] + 31, 2 * pos[1]:2 * pos[1] + 31] p2 = torch.nn.functional.pad(f[2], [4, 4, 4, 4])[:, :, pos[0]:pos[0] + 15, pos[1]:pos[1] + 15] else: p0 = F.unfold(f[0], (61, 61), padding=0, stride=4).permute(0, 2, 1).contiguous().view(-1, 64, 61, 61) if not (pos is None): p0 = torch.index_select(p0, 0, pos) p1 = F.unfold(f[1], (31, 31), padding=0, stride=2).permute(0, 2, 1).contiguous().view(-1, 256, 31, 31) if not (pos is None): p1 = torch.index_select(p1, 0, pos) p2 = F.unfold(f[2], (15, 15), padding=0, stride=1).permute(0, 2, 1).contiguous().view(-1, 512, 15, 15) if not (pos is None): p2 = torch.index_select(p2, 0, pos) if not(pos is None): p3 = corr_feature[:, :, pos[0], pos[1]].view(-1, 256, 1, 1) else: p3 = corr_feature.permute(0, 2, 3, 1).contiguous().view(-1, 256, 1, 1) out = self.deconv(p3) out = self.post0(F.upsample(self.h2(out) + self.v2(p2), size=(31, 31))) out = self.post1(F.upsample(self.h1(out) + self.v1(p1), size=(61, 61))) out = self.post2(F.upsample(self.h0(out) + self.v0(p0), size=(127, 127))) out = out.view(-1, 127*127) return out def param_groups(self, start_lr, feature_mult=1): params = filter(lambda x:x.requires_grad, self.parameters()) params = [{'params': params, 'lr': start_lr * feature_mult}] return params class Custom(SiamMask): def __init__(self, pretrain=False, **kwargs): super(Custom, self).__init__(**kwargs) self.features = ResDown(pretrain=pretrain) self.rpn_model = UP(anchor_num=self.anchor_num, feature_in=256, feature_out=256) self.mask_model = MaskCorr() self.refine_model = Refine() def refine(self, f, pos=None): return self.refine_model(f, pos) def template(self, template): self.zf = self.features(template) def track(self, search): search = self.features(search) rpn_pred_cls, rpn_pred_loc = self.rpn(self.zf, search) return rpn_pred_cls, rpn_pred_loc def track_mask(self, search): self.feature, self.search = self.features.forward_all(search) rpn_pred_cls, rpn_pred_loc = self.rpn(self.zf, self.search) self.corr_feature = self.mask_model.mask.forward_corr(self.zf, self.search) pred_mask = self.mask_model.mask.head(self.corr_feature) return rpn_pred_cls, rpn_pred_loc, pred_mask def track_refine(self, pos): pred_mask = self.refine_model(self.feature, self.corr_feature, pos=pos, test=True) return pred_mask