Spaces:
Starting
on
T4
Starting
on
T4
import torch | |
import torch.nn as nn | |
import torch.nn.functional as F | |
def _make_divisible(v, divisor, min_value=None): | |
if min_value is None: | |
min_value = divisor | |
new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) | |
# Make sure that round down does not go down by more than 10%. | |
if new_v < 0.9 * v: | |
new_v += divisor | |
return new_v | |
class swish(nn.Module): | |
def forward(self, x): | |
return x * torch.sigmoid(x) | |
class h_swish(nn.Module): | |
def __init__(self, inplace=False): | |
super(h_swish, self).__init__() | |
self.inplace = inplace | |
def forward(self, x): | |
return x * F.relu6(x + 3.0, inplace=self.inplace) / 6.0 | |
class h_sigmoid(nn.Module): | |
def __init__(self, inplace=True, h_max=1): | |
super(h_sigmoid, self).__init__() | |
self.relu = nn.ReLU6(inplace=inplace) | |
self.h_max = h_max | |
def forward(self, x): | |
return self.relu(x + 3) * self.h_max / 6 | |
class DYReLU(nn.Module): | |
def __init__( | |
self, | |
inp, | |
oup, | |
reduction=4, | |
lambda_a=1.0, | |
K2=True, | |
use_bias=True, | |
use_spatial=False, | |
init_a=[1.0, 0.0], | |
init_b=[0.0, 0.0], | |
): | |
super(DYReLU, self).__init__() | |
self.oup = oup | |
self.lambda_a = lambda_a * 2 | |
self.K2 = K2 | |
self.avg_pool = nn.AdaptiveAvgPool2d(1) | |
self.use_bias = use_bias | |
if K2: | |
self.exp = 4 if use_bias else 2 | |
else: | |
self.exp = 2 if use_bias else 1 | |
self.init_a = init_a | |
self.init_b = init_b | |
# determine squeeze | |
if reduction == 4: | |
squeeze = inp // reduction | |
else: | |
squeeze = _make_divisible(inp // reduction, 4) | |
# print('reduction: {}, squeeze: {}/{}'.format(reduction, inp, squeeze)) | |
# print('init_a: {}, init_b: {}'.format(self.init_a, self.init_b)) | |
self.fc = nn.Sequential( | |
nn.Linear(inp, squeeze), nn.ReLU(inplace=True), nn.Linear(squeeze, oup * self.exp), h_sigmoid() | |
) | |
if use_spatial: | |
self.spa = nn.Sequential( | |
nn.Conv2d(inp, 1, kernel_size=1), | |
nn.BatchNorm2d(1), | |
) | |
else: | |
self.spa = None | |
def forward(self, x): | |
if isinstance(x, list): | |
x_in = x[0] | |
x_out = x[1] | |
else: | |
x_in = x | |
x_out = x | |
b, c, h, w = x_in.size() | |
y = self.avg_pool(x_in).view(b, c) | |
y = self.fc(y).view(b, self.oup * self.exp, 1, 1) | |
if self.exp == 4: | |
a1, b1, a2, b2 = torch.split(y, self.oup, dim=1) | |
a1 = (a1 - 0.5) * self.lambda_a + self.init_a[0] # 1.0 | |
a2 = (a2 - 0.5) * self.lambda_a + self.init_a[1] | |
b1 = b1 - 0.5 + self.init_b[0] | |
b2 = b2 - 0.5 + self.init_b[1] | |
out = torch.max(x_out * a1 + b1, x_out * a2 + b2) | |
elif self.exp == 2: | |
if self.use_bias: # bias but not PL | |
a1, b1 = torch.split(y, self.oup, dim=1) | |
a1 = (a1 - 0.5) * self.lambda_a + self.init_a[0] # 1.0 | |
b1 = b1 - 0.5 + self.init_b[0] | |
out = x_out * a1 + b1 | |
else: | |
a1, a2 = torch.split(y, self.oup, dim=1) | |
a1 = (a1 - 0.5) * self.lambda_a + self.init_a[0] # 1.0 | |
a2 = (a2 - 0.5) * self.lambda_a + self.init_a[1] | |
out = torch.max(x_out * a1, x_out * a2) | |
elif self.exp == 1: | |
a1 = y | |
a1 = (a1 - 0.5) * self.lambda_a + self.init_a[0] # 1.0 | |
out = x_out * a1 | |
if self.spa: | |
ys = self.spa(x_in).view(b, -1) | |
ys = F.softmax(ys, dim=1).view(b, 1, h, w) * h * w | |
ys = F.hardtanh(ys, 0, 3, inplace=True) / 3 | |
out = out * ys | |
return out | |