Kaggle competition: Severstal: Steel Defect Detection
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True )<feature_engineering>
warnings.filterwarnings('ignore')
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152']
model_urls = {
'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
}
def conv3x3(in_planes, out_planes, stride=1):
'3x3 convolution with padding'
return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False)
 class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self ).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = conv3x3(planes, planes)
 self.bn2 = nn.BatchNorm2d(planes)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class ResNet(nn.Module):
def __init__(self, block, layers, in_channels=3):
self.inplanes = 64
super(ResNet, self ).__init__()
self.conv1 = nn.Conv2d(in_channels, 64, kernel_size=7, stride=2, padding=3, bias=False)
 self.bn1 = nn.BatchNorm2d(64)
 self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0])
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
 self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2.0 / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(
 self.inplanes,
planes * block.expansion,
kernel_size=1,
stride=stride,
bias=False, 
),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, stride, downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 print(x.size())
 x = self.layer1(x)
 print(x.size())
 x = self.layer2(x)
 print(x.size())
 x = self.layer3(x)
 print(x.size())
 x = self.layer4(x)
 print(x.size())
 return x
def resnet18(pretrained=True, **kwargs):

 model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)
 return model
def resnet34(pretrained=True, **kwargs):

 model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs)
 return model
def resnet50(pretrained=False, **kwargs):

 model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs)
 return model
def resnet101(**kwargs):

 model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs)
 return model
def resnet152(**kwargs):

 model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs)
 return model
def conv3x3(in_, out):
return nn.Conv2d(in_, out, 3, padding=1)
 class ConvRelu(nn.Module):
def __init__(self, in_, out):
super().__init__()
self.conv = conv3x3(in_, out)
 self.activation = nn.PReLU()
def forward(self, x):
x = self.conv(x)
 x = self.activation(x)
 return x
class DecoderBlockV2(nn.Module):
def __init__(self, in_channels, middle_channels, out_channels, is_deconv=True):
super(DecoderBlockV2, self ).__init__()
self.in_channels = in_channels
if is_deconv:

 self.block = nn.Sequential(
 ConvRelu(in_channels, middle_channels),
nn.ConvTranspose2d(middle_channels, out_channels, kernel_size=4, stride=2,
padding=1),
nn.PReLU())
 else:
self.block = nn.Sequential(
 nn.Upsample(scale_factor=2, mode='bilinear', align_corners = True),
ConvRelu(in_channels, middle_channels),
ConvRelu(middle_channels, out_channels))
def forward(self, x):
return self.block(x)
 class AlbuNet(nn.Module):

 def __init__(self, encoder_depth, num_classes, num_filters=32, dropout=0.05,
pretrained=False, is_deconv=False):
super().__init__()
self.num_classes = num_classes
self.dropout = dropout
if encoder_depth == 34:
self.encoder = torchvision.models.resnet34(pretrained=pretrained)
 bottom_channel_nr = 512
elif encoder_depth == 50:
self.encoder = torchvision.models.resnet50(pretrained=pretrained)
 bottom_channel_nr = 512
elif encoder_depth == 101:
self.encoder = torchvision.models.resnet101(pretrained=pretrained)
 bottom_channel_nr = 2048
elif encoder_depth == 152:
self.encoder = torchvision.models.resnet152(pretrained=pretrained)
 bottom_channel_nr = 2048
else:
raise NotImplementedError('only 34, 50, 101, 152 version of Resnet are implemented')
 self.pool = nn.MaxPool2d(2, 2)
 self.conv1 = nn.Sequential(self.encoder.conv1, self.encoder.bn1, self.encoder.relu, self.pool)
 self.conv2 = self.encoder.layer1
self.conv3 = self.encoder.layer2
self.conv4 = self.encoder.layer3
self.conv5 = self.encoder.layer4
self.center = DecoderBlockV2(bottom_channel_nr, num_filters * 8 * 2, num_filters * 8, is_deconv)
 self.dec5 = DecoderBlockV2(bottom_channel_nr + num_filters * 8, num_filters * 8 * 2, num_filters * 8, is_deconv)
 self.dec4 = DecoderBlockV2(bottom_channel_nr // 2 + num_filters * 8, num_filters * 8 * 2, num_filters * 8,is_deconv)
 self.dec3 = DecoderBlockV2(bottom_channel_nr // 4 + num_filters * 8, num_filters * 4 * 2, num_filters * 2,is_deconv)
 self.dec2 = DecoderBlockV2(bottom_channel_nr // 8 + num_filters * 2, num_filters * 2 * 2, num_filters * 2 * 2,is_deconv)
 self.dec1 = DecoderBlockV2(num_filters * 2 * 2, num_filters * 2 * 2, num_filters, is_deconv)
 self.dec0 = ConvRelu(num_filters, num_filters)
 self.final_conv = nn.Conv2d(num_filters, self.num_classes , kernel_size=1)
 self.dropout = nn.Dropout(self.dropout)
 def forward(self, x):
conv1 = self.conv1(x)
 conv2 = self.conv2(conv1)
 conv3 = self.conv3(conv2)
 conv4 = self.conv4(conv3)
 conv5 = self.conv5(conv4)
 pool = self.pool(conv5)
 center = self.center(pool)
 center = self.dropout(center)
 dec5 = self.dec5(torch.cat([center, conv5], 1))
dec5 = self.dropout(dec5)
 dec4 = self.dec4(torch.cat([dec5, conv4], 1))
dec4 = self.dropout(dec4)
 dec3 = self.dec3(torch.cat([dec4, conv3], 1))
dec3 = self.dropout(dec3)
 dec2 = self.dec2(torch.cat([dec3, conv2], 1))
dec2 = self.dropout(dec2)
 dec1 = self.dec1(dec2)
 dec1 = self.dropout(dec1)
 dec0 = self.dec0(dec1)
 dec0 = self.dropout(dec0)
 x = self.final_conv(dec0)
 return x
__all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',
'se_resnext50_32x4d', 'se_resnext101_32x4d']
pretrained_settings = {
'senet154': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet50': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet101': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet152': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext50_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext101_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
}
class SEModule(nn.Module):
def __init__(self, channels, reduction, concat=False):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class SCSEModule(nn.Module):
def __init__(self, channels, reduction=16, mode='concat'):
super(SCSEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
self.spatial_se = nn.Sequential(nn.Conv2d(channels, 1, kernel_size=1,
stride=1, padding=0, bias=False),
nn.Sigmoid())
 self.mode = mode
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 chn_se = self.sigmoid(x)
 chn_se = chn_se * module_input
spa_se = self.spatial_se(module_input)
 spa_se = module_input * spa_se
if self.mode == 'concat':
return torch.cat([chn_se, spa_se], dim=1)
 elif self.mode == 'maxout':
return torch.max(chn_se, spa_se)
 else:
return chn_se + spa_se
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SCSEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SCSEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SCSEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SCSEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4, final=False):
super(SCSEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SCSEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):

 super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(4, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
self.pool = nn.MaxPool2d(3, stride=2, ceil_mode=True)
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 self._initialize_weights()
def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def _initialize_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d)or isinstance(m, nn.ConvTranspose2d):
m.weight.data = nn.init.kaiming_normal_(m.weight.data)
 if m.bias is not None:
m.bias.data.zero_()
elif isinstance(m, BatchNorm2d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def features(self, x):
x = self.layer0(x)
 x = self.pool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def initialize_pretrained_model(model, num_classes, settings):
assert num_classes == settings['num_classes'], \
'num_classes should be {}, but is {}'.format(
 settings['num_classes'], num_classes)
 model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
def senet154(num_classes=1000, pretrained='imagenet'):
model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,
dropout_p=0.2, num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['senet154'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def scsenet154(num_classes=1000, pretrained='imagenet'):
print("scsenet154")
 model = SENet(SCSEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,
dropout_p=0.2, num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['senet154'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet50(num_classes=1000, pretrained='None'):
model = SENet(SEResNetBottleneck, [3, 4, 6, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 return model
def se_resnet101(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 4, 23, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 return model
def se_resnet152(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet152'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 return model
def scse_resnext50_32x4d(num_classes=1000, pretrained=None):
model = SENet(SCSEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 return model
def se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 pretrained = None
if pretrained is not None:
settings = pretrained_settings['se_resnext101_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
encoder_params = {
'resnet18':
{
'filters': [64, 64, 128, 256, 512],
'decoder_filters': [64, 128, 256, 256],
'last_upsample': 64,
'init_op': partial(resnet18, in_channels=3),
'url': model_urls['resnet18'],
},
'resnet34':
{
'filters': [64, 64, 128, 256, 512],
'decoder_filters': [64, 128, 256, 256],
'last_upsample': 64,
'init_op': partial(resnet34, in_channels=3),
'url': model_urls['resnet34'],
},
'resnet101':
{
'filters': [64, 256, 512, 1024, 2048],
'decoder_filters': [64, 128, 256, 256],
'last_upsample': 64,
'init_op': partial(resnet101, in_channels=3),
'url': model_urls['resnet101'],
},
'resnet50':
{
'filters': [64, 256, 512, 1024, 2048],
'decoder_filters': [64, 128, 256, 256],
'last_upsample': 64,
'init_op': partial(resnet50, in_channels=3),
'url': model_urls['resnet50'],
},
'seresnext50':
{
'filters': [64, 256, 512, 1024, 2048],
'decoder_filters': [64, 128, 256, 384],
'init_op': se_resnext50_32x4d,
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',
},
'senet154':
{
'filters': [128, 256, 512, 1024, 2048],
'decoder_filters': [64, 128, 256, 384],
'init_op': senet154,
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',
},
'seresnext101':
{
'filters': [64, 256, 512, 1024, 2048],
'decoder_filters': [64, 128, 256, 384],
'last_upsample': 64,
'init_op': se_resnext101_32x4d,
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',
}
}
class AbstractModel(nn.Module):
def _initialize_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d)or isinstance(m, nn.ConvTranspose2d):
m.weight.data = nn.init.kaiming_normal_(m.weight.data)
 if m.bias is not None:
m.bias.data.zero_()
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def initialize_encoder(self, model, model_url, num_channels_changed=False):

 model_dict = model.state_dict()
if num_channels_changed:
model.state_dict() [self.first_layer_params_name +
'.weight'][:, :3,...] = pretrained_dict[self.first_layer_params_name + '.weight'].data
skip_layers = [
self.first_layer_params_name,
self.first_layer_params_name + '.weight',
]
pretrained_dict = {
k: v
for k, v in pretrained_dict.items()
if not any(k.startswith(s)for s in skip_layers)
 }
@property
def first_layer_params_name(self):
return 'conv1'
class EncoderDecoder(AbstractModel):
def __init__(self, num_classes, num_channels=3, encoder_name='resnet34'):
if not hasattr(self, 'first_layer_stride_two'):
self.first_layer_stride_two = False
if not hasattr(self, 'decoder_block'):
self.decoder_block = UnetDecoderBlock
if not hasattr(self, 'bottleneck_type'):
self.bottleneck_type = ConvBottleneck
self.filters = encoder_params[encoder_name]['filters']
self.decoder_filters = encoder_params[encoder_name].get('decoder_filters', self.filters[:-1])
 self.last_upsample_filters = encoder_params[encoder_name].get('last_upsample', self.decoder_filters[0] // 2)
 super().__init__()
self.num_channels = num_channels
self.num_classes = num_classes
self.bottlenecks = nn.ModuleList(
 [
self.bottleneck_type(self.filters[-i - 2] + f, f)
 for i, f in enumerate(reversed(self.decoder_filters[:]))
] 
)
 self.decoder_stages = nn.ModuleList([self.get_decoder(idx)for idx in range(0, len(self.decoder_filters)) ])
 if self.first_layer_stride_two:
self.last_upsample = self.decoder_block(
 self.decoder_filters[0],
self.last_upsample_filters,
self.last_upsample_filters, 
)
 self.final = self.make_final_classifier(
 self.last_upsample_filters if self.first_layer_stride_two else self.decoder_filters[0],
num_classes, 
)
 self._initialize_weights()
encoder = encoder_params[encoder_name]['init_op'](pretrained=False)
 self.encoder_stages = nn.ModuleList([self.get_encoder(encoder, idx)for idx in range(len(self.filters)) ])
 def forward(self, x):
enc_results = []
for stage in self.encoder_stages:
x = stage(x)
 enc_results.append(torch.cat(x, dim=1)if isinstance(x, tuple)else x.clone())
 last_dec_out = enc_results[-1]
x = last_dec_out
for idx, bottleneck in enumerate(self.bottlenecks):
rev_idx = -(idx + 1)
 x = self.decoder_stages[rev_idx](x)
 x = bottleneck(x, enc_results[rev_idx - 1])
 if self.first_layer_stride_two:
x = self.last_upsample(x)
 f = self.final(x)
 return f
def get_decoder(self, layer):
in_channels =(
 self.filters[layer + 1] if layer + 1 == len(self.decoder_filters)else self.decoder_filters[layer + 1] 
)
 return self.decoder_block(
 in_channels,
self.decoder_filters[layer],
self.decoder_filters[max(layer, 0)], 
)
 def make_final_classifier(self, in_filters, num_classes):
return nn.Sequential(nn.Conv2d(in_filters, num_classes, 1, padding=0))
def get_encoder(self, encoder, layer):
raise NotImplementedError
@property
def first_layer_params(self):
return _get_layers_params([self.encoder_stages[0]])
 @property
def layers_except_first_params(self):
layers = get_slice(self.encoder_stages, 1, -1)+ [
self.bottlenecks,
self.decoder_stages,
self.final,
]
return _get_layers_params(layers)
 def _get_layers_params(layers):
return sum(( list(l.parameters())for l in layers), [])
 def get_slice(features, start, end):
if end == -1:
end = len(features)
 return [features[i] for i in range(start, end)]
class ConvBottleneck(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.seq = nn.Sequential(nn.Conv2d(in_channels, out_channels, 3, padding=1), nn.ReLU(inplace=True))
def forward(self, dec, enc):
x = torch.cat([dec, enc], dim=1)
 return self.seq(x)
 class UnetDecoderBlock(nn.Module):
def __init__(self, in_channels, middle_channels, out_channels):
super().__init__()
self.layer = nn.Sequential(
 nn.Upsample(scale_factor=2),
nn.Conv2d(in_channels, out_channels, 3, padding=1),
nn.ReLU(inplace=True), 
)
 def forward(self, x):
return self.layer(x)
 class UnetConvTransposeDecoderBlock(nn.Module):
def __init__(self, in_channels, middle_channels, out_channels):
super().__init__()
self.layer = nn.Sequential(
 nn.ConvTranspose2d(in_channels, in_channels, kernel_size=2, stride=2, padding=0),
nn.Conv2d(in_channels, out_channels, 3, padding=1),
nn.ReLU(inplace=True), 
)
 def forward(self, x):
return self.layer(x)
 class Resnet(EncoderDecoder):
def __init__(self, seg_classes, backbone_arch):
self.first_layer_stride_two = True
super().__init__(seg_classes, 4, backbone_arch)
 def get_encoder(self, encoder, layer):
if layer == 0:
return nn.Sequential(encoder.conv1, encoder.bn1, encoder.relu)
 elif layer == 1:
return nn.Sequential(encoder.maxpool, encoder.layer1)
 elif layer == 2:
return encoder.layer2
elif layer == 3:
return encoder.layer3
elif layer == 4:
return encoder.layer4
class ConvTransposeResnetUnet(EncoderDecoder):
def __init__(self, seg_classes, backbone_arch):
self.first_layer_stride_two = True
self.decoder_block = UnetConvTransposeDecoderBlock
super().__init__(seg_classes, 3, backbone_arch)
 def get_encoder(self, encoder, layer):
if layer == 0:
return nn.Sequential(encoder.conv1, encoder.bn1, encoder.relu)
 elif layer == 1:
return nn.Sequential(encoder.maxpool, encoder.layer1)
 elif layer == 2:
return encoder.layer2
elif layer == 3:
return encoder.layer3
elif layer == 4:
return encoder.layer4
class SEUnet(EncoderDecoder):
def __init__(self, seg_classes, backbone_arch='senet50'):
self.first_layer_stride_two = True
super().__init__(seg_classes, num_channels=3, encoder_name=backbone_arch)
 def get_encoder(self, encoder, layer):
if layer == 0:
return encoder.layer0
elif layer == 1:
return nn.Sequential(encoder.pool, encoder.layer1)
 elif layer == 2:
return encoder.layer2
elif layer == 3:
return encoder.layer3
elif layer == 4:
return encoder.layer4
@property
def first_layer_params_name(self):
return 'layer0.conv1'
class ConvSCSEBottleneckNoBn(nn.Module):
def __init__(self, in_channels, out_channels, reduction=2):
print('bottleneck ', in_channels, out_channels)
 super().__init__()
self.seq = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, padding=1),
nn.ReLU(inplace=True),
SCSEModule(out_channels, reduction=reduction, mode='maxout'), 
)
 def forward(self, dec, enc):
x = torch.cat([dec, enc], dim=1)
 return self.seq(x)
 class SCSEUnet(SEUnet):
def __init__(self, seg_classes, backbone_arch='seresnext50'):
self.bottleneck_type = ConvSCSEBottleneckNoBn
super().__init__(seg_classes, backbone_arch=backbone_arch)
 def conv3x3(in_, out):
return nn.Conv2d(in_, out, 3, padding=1)
 class ConvRelu(nn.Module):
def __init__(self, in_, out):
super().__init__()
self.conv = conv3x3(in_, out)
 self.activation = nn.PReLU()
def forward(self, x):
x = self.conv(x)
 x = self.activation(x)
 return x
class DecoderBlockV2(nn.Module):
def __init__(self, in_channels, middle_channels, out_channels, is_deconv=True):
super(DecoderBlockV2, self ).__init__()
self.in_channels = in_channels
if is_deconv:

 self.block = nn.Sequential(
 ConvRelu(in_channels, middle_channels),
nn.ConvTranspose2d(middle_channels, out_channels, kernel_size=4, stride=2,
padding=1),
nn.PReLU())
 else:
self.block = nn.Sequential(
 nn.Upsample(scale_factor=2, mode='bilinear', align_corners = True),
ConvRelu(in_channels, middle_channels),
ConvRelu(middle_channels, out_channels))
def forward(self, x):
return self.block(x)
 class AlbuNet(nn.Module):

 def __init__(self, encoder_depth, num_classes, num_filters=32, dropout=0.3,
pretrained=False, is_deconv=False):
super().__init__()
self.num_classes = num_classes
self.dropout = dropout
if encoder_depth == 34:
self.encoder = torchvision.models.resnet34(pretrained=pretrained)
 bottom_channel_nr = 512
elif encoder_depth == 50:
self.encoder = torchvision.models.resnet50(pretrained=pretrained)
 bottom_channel_nr = 512
elif encoder_depth == 101:
self.encoder = torchvision.models.resnet101(pretrained=pretrained)
 bottom_channel_nr = 2048
elif encoder_depth == 152:
self.encoder = torchvision.models.resnet152(pretrained=pretrained)
 bottom_channel_nr = 2048
else:
raise NotImplementedError('only 34, 50, 101, 152 version of Resnet are implemented')
 self.pool = nn.MaxPool2d(2, 2)
 self.conv1 = nn.Sequential(self.encoder.conv1, self.encoder.bn1, self.encoder.relu, self.pool)
 self.conv2 = self.encoder.layer1
self.conv3 = self.encoder.layer2
self.conv4 = self.encoder.layer3
self.conv5 = self.encoder.layer4
self.center = DecoderBlockV2(bottom_channel_nr, num_filters * 8 * 2, num_filters * 8, is_deconv)
 self.dec5 = DecoderBlockV2(bottom_channel_nr + num_filters * 8, num_filters * 8 * 2, num_filters * 8, is_deconv)
 self.dec4 = DecoderBlockV2(bottom_channel_nr // 2 + num_filters * 8, num_filters * 8 * 2, num_filters * 8,is_deconv)
 self.dec3 = DecoderBlockV2(bottom_channel_nr // 4 + num_filters * 8, num_filters * 4 * 2, num_filters * 2,is_deconv)
 self.dec2 = DecoderBlockV2(bottom_channel_nr // 8 + num_filters * 2, num_filters * 2 * 2, num_filters * 2 * 2,is_deconv)
 self.dec1 = DecoderBlockV2(num_filters * 2 * 2, num_filters * 2 * 2, num_filters, is_deconv)
 self.dec0 = ConvRelu(num_filters, num_filters)
 self.final_conv = nn.Conv2d(num_filters, self.num_classes , kernel_size=1)
 self.dropout = nn.Dropout(self.dropout)
 def forward(self, x):
conv1 = self.conv1(x)
 conv2 = self.conv2(conv1)
 conv3 = self.conv3(conv2)
 conv4 = self.conv4(conv3)
 conv5 = self.conv5(conv4)
 pool = self.pool(conv5)
 center = self.center(pool)
 center = self.dropout(center)
 dec5 = self.dec5(torch.cat([center, conv5], 1))
dec5 = self.dropout(dec5)
 dec4 = self.dec4(torch.cat([dec5, conv4], 1))
dec4 = self.dropout(dec4)
 dec3 = self.dec3(torch.cat([dec4, conv3], 1))
dec3 = self.dropout(dec3)
 dec2 = self.dec2(torch.cat([dec3, conv2], 1))
dec2 = self.dropout(dec2)
 dec1 = self.dec1(dec2)
 dec1 = self.dropout(dec1)
 dec0 = self.dec0(dec1)
 dec0 = self.dropout(dec0)
 x = self.final_conv(dec0)
 return x
class ConvBn2d(nn.Module):
def __init__(self, in_channel, out_channel, kernel_size=3, padding=1, stride=1):
super(ConvBn2d, self ).__init__()
self.conv = nn.Conv2d(in_channel, out_channel, kernel_size=kernel_size, padding=padding, stride=stride, bias=False)
 self.bn = nn.BatchNorm2d(out_channel, eps=1e-5)
 def forward(self,x):
x = self.conv(x)
 x = self.bn(x)
 return x
class BasicBlock_R(nn.Module):
def __init__(self, in_channel, channel, out_channel, stride=1, is_shortcut=False):
super(BasicBlock_R, self ).__init__()
self.is_shortcut = is_shortcut
self.conv_bn1 = ConvBn2d(in_channel, channel, kernel_size=3, padding=1, stride=stride)
 self.conv_bn2 = ConvBn2d(channel,out_channel, kernel_size=3, padding=1, stride=1)
 if is_shortcut:
self.shortcut = ConvBn2d(in_channel, out_channel, kernel_size=1, padding=0, stride=stride)
 def forward(self, x):
z = F.relu(self.conv_bn1(x),inplace=True)
 z = self.conv_bn2(z)
 if self.is_shortcut:
x = self.shortcut(x)
 z += x
z = F.relu(z,inplace=True)
 return z
class ResNet34(nn.Module):
def __init__(self, num_class=4):
super(ResNet34, self ).__init__()
self.block0 = nn.Sequential(
 nn.Conv2d(3, 64, kernel_size=7, padding=3, stride=2, bias=False),
BatchNorm2d(64),
nn.ReLU(inplace=True), 
)
 self.block1 = nn.Sequential(
 nn.MaxPool2d(kernel_size=3, padding=1, stride=2),
BasicBlock_R(64, 64, 64, stride=1, is_shortcut=False,),
* [BasicBlock_R(64, 64, 64, stride=1, is_shortcut=False,)for i in range(1,3)], 
)
 self.block2 = nn.Sequential(
 BasicBlock_R(64,128,128, stride=2, is_shortcut=True,),
* [BasicBlock_R(128,128,128, stride=1, is_shortcut=False,)for i in range(1,4)], 
)
 self.block3 = nn.Sequential(
 BasicBlock_R(128,256,256, stride=2, is_shortcut=True,),
* [BasicBlock_R(256,256,256, stride=1, is_shortcut=False,)for i in range(1,6)], 
)
 self.block4 = nn.Sequential(
 BasicBlock_R(256,512,512, stride=2, is_shortcut=True,),
* [BasicBlock_R(512,512,512, stride=1, is_shortcut=False,)for i in range(1,3)], 
)
 self.logit = nn.Linear(512,num_class)
 def forward(self, x):
batch_size = len(x)
 x = self.block0(x)
 x = self.block1(x)
 x = self.block2(x)
 x = self.block3(x)
 x = self.block4(x)
 x = F.adaptive_avg_pool2d(x,1 ).reshape(batch_size,-1)
 logit = self.logit(x)
 return logit
CONVERSION=[
'block0.0.weight',	(64, 3, 7, 7),	 'conv1.weight',	(64, 3, 7, 7),
'block0.1.weight',	(64,),	 'bn1.weight',	(64,),
'block0.1.bias',	(64,),	 'bn1.bias',	(64,),
'block0.1.running_mean',	(64,),	 'bn1.running_mean',	(64,),
'block0.1.running_var',	(64,),	 'bn1.running_var',	(64,),
'block1.1.conv_bn1.conv.weight',	(64, 64, 3, 3),	 'layer1.0.conv1.weight',	(64, 64, 3, 3),
'block1.1.conv_bn1.bn.weight',	(64,),	 'layer1.0.bn1.weight',	(64,),
'block1.1.conv_bn1.bn.bias',	(64,),	 'layer1.0.bn1.bias',	(64,),
'block1.1.conv_bn1.bn.running_mean',	(64,),	 'layer1.0.bn1.running_mean',	(64,),
'block1.1.conv_bn1.bn.running_var',	(64,),	 'layer1.0.bn1.running_var',	(64,),
'block1.1.conv_bn2.conv.weight',	(64, 64, 3, 3),	 'layer1.0.conv2.weight',	(64, 64, 3, 3),
'block1.1.conv_bn2.bn.weight',	(64,),	 'layer1.0.bn2.weight',	(64,),
'block1.1.conv_bn2.bn.bias',	(64,),	 'layer1.0.bn2.bias',	(64,),
'block1.1.conv_bn2.bn.running_mean',	(64,),	 'layer1.0.bn2.running_mean',	(64,),
'block1.1.conv_bn2.bn.running_var',	(64,),	 'layer1.0.bn2.running_var',	(64,),
'block1.2.conv_bn1.conv.weight',	(64, 64, 3, 3),	 'layer1.1.conv1.weight',	(64, 64, 3, 3),
'block1.2.conv_bn1.bn.weight',	(64,),	 'layer1.1.bn1.weight',	(64,),
'block1.2.conv_bn1.bn.bias',	(64,),	 'layer1.1.bn1.bias',	(64,),
'block1.2.conv_bn1.bn.running_mean',	(64,),	 'layer1.1.bn1.running_mean',	(64,),
'block1.2.conv_bn1.bn.running_var',	(64,),	 'layer1.1.bn1.running_var',	(64,),
'block1.2.conv_bn2.conv.weight',	(64, 64, 3, 3),	 'layer1.1.conv2.weight',	(64, 64, 3, 3),
'block1.2.conv_bn2.bn.weight',	(64,),	 'layer1.1.bn2.weight',	(64,),
'block1.2.conv_bn2.bn.bias',	(64,),	 'layer1.1.bn2.bias',	(64,),
'block1.2.conv_bn2.bn.running_mean',	(64,),	 'layer1.1.bn2.running_mean',	(64,),
'block1.2.conv_bn2.bn.running_var',	(64,),	 'layer1.1.bn2.running_var',	(64,),
'block1.3.conv_bn1.conv.weight',	(64, 64, 3, 3),	 'layer1.2.conv1.weight',	(64, 64, 3, 3),
'block1.3.conv_bn1.bn.weight',	(64,),	 'layer1.2.bn1.weight',	(64,),
'block1.3.conv_bn1.bn.bias',	(64,),	 'layer1.2.bn1.bias',	(64,),
'block1.3.conv_bn1.bn.running_mean',	(64,),	 'layer1.2.bn1.running_mean',	(64,),
'block1.3.conv_bn1.bn.running_var',	(64,),	 'layer1.2.bn1.running_var',	(64,),
'block1.3.conv_bn2.conv.weight',	(64, 64, 3, 3),	 'layer1.2.conv2.weight',	(64, 64, 3, 3),
'block1.3.conv_bn2.bn.weight',	(64,),	 'layer1.2.bn2.weight',	(64,),
'block1.3.conv_bn2.bn.bias',	(64,),	 'layer1.2.bn2.bias',	(64,),
'block1.3.conv_bn2.bn.running_mean',	(64,),	 'layer1.2.bn2.running_mean',	(64,),
'block1.3.conv_bn2.bn.running_var',	(64,),	 'layer1.2.bn2.running_var',	(64,),
'block2.0.conv_bn1.conv.weight',	(128, 64, 3, 3),	 'layer2.0.conv1.weight',	(128, 64, 3, 3),
'block2.0.conv_bn1.bn.weight',	(128,),	 'layer2.0.bn1.weight',	(128,),
'block2.0.conv_bn1.bn.bias',	(128,),	 'layer2.0.bn1.bias',	(128,),
'block2.0.conv_bn1.bn.running_mean',	(128,),	 'layer2.0.bn1.running_mean',	(128,),
'block2.0.conv_bn1.bn.running_var',	(128,),	 'layer2.0.bn1.running_var',	(128,),
'block2.0.conv_bn2.conv.weight',	(128, 128, 3, 3),	 'layer2.0.conv2.weight',	(128, 128, 3, 3),
'block2.0.conv_bn2.bn.weight',	(128,),	 'layer2.0.bn2.weight',	(128,),
'block2.0.conv_bn2.bn.bias',	(128,),	 'layer2.0.bn2.bias',	(128,),
'block2.0.conv_bn2.bn.running_mean',	(128,),	 'layer2.0.bn2.running_mean',	(128,),
'block2.0.conv_bn2.bn.running_var',	(128,),	 'layer2.0.bn2.running_var',	(128,),
'block2.0.shortcut.conv.weight',	(128, 64, 1, 1),	 'layer2.0.downsample.0.weight',	(128, 64, 1, 1),
'block2.0.shortcut.bn.weight',	(128,),	 'layer2.0.downsample.1.weight',	(128,),
'block2.0.shortcut.bn.bias',	(128,),	 'layer2.0.downsample.1.bias',	(128,),
'block2.0.shortcut.bn.running_mean',	(128,),	 'layer2.0.downsample.1.running_mean',	(128,),
'block2.0.shortcut.bn.running_var',	(128,),	 'layer2.0.downsample.1.running_var',	(128,),
'block2.1.conv_bn1.conv.weight',	(128, 128, 3, 3),	 'layer2.1.conv1.weight',	(128, 128, 3, 3),
'block2.1.conv_bn1.bn.weight',	(128,),	 'layer2.1.bn1.weight',	(128,),
'block2.1.conv_bn1.bn.bias',	(128,),	 'layer2.1.bn1.bias',	(128,),
'block2.1.conv_bn1.bn.running_mean',	(128,),	 'layer2.1.bn1.running_mean',	(128,),
'block2.1.conv_bn1.bn.running_var',	(128,),	 'layer2.1.bn1.running_var',	(128,),
'block2.1.conv_bn2.conv.weight',	(128, 128, 3, 3),	 'layer2.1.conv2.weight',	(128, 128, 3, 3),
'block2.1.conv_bn2.bn.weight',	(128,),	 'layer2.1.bn2.weight',	(128,),
'block2.1.conv_bn2.bn.bias',	(128,),	 'layer2.1.bn2.bias',	(128,),
'block2.1.conv_bn2.bn.running_mean',	(128,),	 'layer2.1.bn2.running_mean',	(128,),
'block2.1.conv_bn2.bn.running_var',	(128,),	 'layer2.1.bn2.running_var',	(128,),
'block2.2.conv_bn1.conv.weight',	(128, 128, 3, 3),	 'layer2.2.conv1.weight',	(128, 128, 3, 3),
'block2.2.conv_bn1.bn.weight',	(128,),	 'layer2.2.bn1.weight',	(128,),
'block2.2.conv_bn1.bn.bias',	(128,),	 'layer2.2.bn1.bias',	(128,),
'block2.2.conv_bn1.bn.running_mean',	(128,),	 'layer2.2.bn1.running_mean',	(128,),
'block2.2.conv_bn1.bn.running_var',	(128,),	 'layer2.2.bn1.running_var',	(128,),
'block2.2.conv_bn2.conv.weight',	(128, 128, 3, 3),	 'layer2.2.conv2.weight',	(128, 128, 3, 3),
'block2.2.conv_bn2.bn.weight',	(128,),	 'layer2.2.bn2.weight',	(128,),
'block2.2.conv_bn2.bn.bias',	(128,),	 'layer2.2.bn2.bias',	(128,),
'block2.2.conv_bn2.bn.running_mean',	(128,),	 'layer2.2.bn2.running_mean',	(128,),
'block2.2.conv_bn2.bn.running_var',	(128,),	 'layer2.2.bn2.running_var',	(128,),
'block2.3.conv_bn1.conv.weight',	(128, 128, 3, 3),	 'layer2.3.conv1.weight',	(128, 128, 3, 3),
'block2.3.conv_bn1.bn.weight',	(128,),	 'layer2.3.bn1.weight',	(128,),
'block2.3.conv_bn1.bn.bias',	(128,),	 'layer2.3.bn1.bias',	(128,),
'block2.3.conv_bn1.bn.running_mean',	(128,),	 'layer2.3.bn1.running_mean',	(128,),
'block2.3.conv_bn1.bn.running_var',	(128,),	 'layer2.3.bn1.running_var',	(128,),
'block2.3.conv_bn2.conv.weight',	(128, 128, 3, 3),	 'layer2.3.conv2.weight',	(128, 128, 3, 3),
'block2.3.conv_bn2.bn.weight',	(128,),	 'layer2.3.bn2.weight',	(128,),
'block2.3.conv_bn2.bn.bias',	(128,),	 'layer2.3.bn2.bias',	(128,),
'block2.3.conv_bn2.bn.running_mean',	(128,),	 'layer2.3.bn2.running_mean',	(128,),
'block2.3.conv_bn2.bn.running_var',	(128,),	 'layer2.3.bn2.running_var',	(128,),
'block3.0.conv_bn1.conv.weight',	(256, 128, 3, 3),	 'layer3.0.conv1.weight',	(256, 128, 3, 3),
'block3.0.conv_bn1.bn.weight',	(256,),	 'layer3.0.bn1.weight',	(256,),
'block3.0.conv_bn1.bn.bias',	(256,),	 'layer3.0.bn1.bias',	(256,),
'block3.0.conv_bn1.bn.running_mean',	(256,),	 'layer3.0.bn1.running_mean',	(256,),
'block3.0.conv_bn1.bn.running_var',	(256,),	 'layer3.0.bn1.running_var',	(256,),
'block3.0.conv_bn2.conv.weight',	(256, 256, 3, 3),	 'layer3.0.conv2.weight',	(256, 256, 3, 3),
'block3.0.conv_bn2.bn.weight',	(256,),	 'layer3.0.bn2.weight',	(256,),
'block3.0.conv_bn2.bn.bias',	(256,),	 'layer3.0.bn2.bias',	(256,),
'block3.0.conv_bn2.bn.running_mean',	(256,),	 'layer3.0.bn2.running_mean',	(256,),
'block3.0.conv_bn2.bn.running_var',	(256,),	 'layer3.0.bn2.running_var',	(256,),
'block3.0.shortcut.conv.weight',	(256, 128, 1, 1),	 'layer3.0.downsample.0.weight',	(256, 128, 1, 1),
'block3.0.shortcut.bn.weight',	(256,),	 'layer3.0.downsample.1.weight',	(256,),
'block3.0.shortcut.bn.bias',	(256,),	 'layer3.0.downsample.1.bias',	(256,),
'block3.0.shortcut.bn.running_mean',	(256,),	 'layer3.0.downsample.1.running_mean',	(256,),
'block3.0.shortcut.bn.running_var',	(256,),	 'layer3.0.downsample.1.running_var',	(256,),
'block3.1.conv_bn1.conv.weight',	(256, 256, 3, 3),	 'layer3.1.conv1.weight',	(256, 256, 3, 3),
'block3.1.conv_bn1.bn.weight',	(256,),	 'layer3.1.bn1.weight',	(256,),
'block3.1.conv_bn1.bn.bias',	(256,),	 'layer3.1.bn1.bias',	(256,),
'block3.1.conv_bn1.bn.running_mean',	(256,),	 'layer3.1.bn1.running_mean',	(256,),
'block3.1.conv_bn1.bn.running_var',	(256,),	 'layer3.1.bn1.running_var',	(256,),
'block3.1.conv_bn2.conv.weight',	(256, 256, 3, 3),	 'layer3.1.conv2.weight',	(256, 256, 3, 3),
'block3.1.conv_bn2.bn.weight',	(256,),	 'layer3.1.bn2.weight',	(256,),
'block3.1.conv_bn2.bn.bias',	(256,),	 'layer3.1.bn2.bias',	(256,),
'block3.1.conv_bn2.bn.running_mean',	(256,),	 'layer3.1.bn2.running_mean',	(256,),
'block3.1.conv_bn2.bn.running_var',	(256,),	 'layer3.1.bn2.running_var',	(256,),
'block3.2.conv_bn1.conv.weight',	(256, 256, 3, 3),	 'layer3.2.conv1.weight',	(256, 256, 3, 3),
'block3.2.conv_bn1.bn.weight',	(256,),	 'layer3.2.bn1.weight',	(256,),
'block3.2.conv_bn1.bn.bias',	(256,),	 'layer3.2.bn1.bias',	(256,),
'block3.2.conv_bn1.bn.running_mean',	(256,),	 'layer3.2.bn1.running_mean',	(256,),
'block3.2.conv_bn1.bn.running_var',	(256,),	 'layer3.2.bn1.running_var',	(256,),
'block3.2.conv_bn2.conv.weight',	(256, 256, 3, 3),	 'layer3.2.conv2.weight',	(256, 256, 3, 3),
'block3.2.conv_bn2.bn.weight',	(256,),	 'layer3.2.bn2.weight',	(256,),
'block3.2.conv_bn2.bn.bias',	(256,),	 'layer3.2.bn2.bias',	(256,),
'block3.2.conv_bn2.bn.running_mean',	(256,),	 'layer3.2.bn2.running_mean',	(256,),
'block3.2.conv_bn2.bn.running_var',	(256,),	 'layer3.2.bn2.running_var',	(256,),
'block3.3.conv_bn1.conv.weight',	(256, 256, 3, 3),	 'layer3.3.conv1.weight',	(256, 256, 3, 3),
'block3.3.conv_bn1.bn.weight',	(256,),	 'layer3.3.bn1.weight',	(256,),
'block3.3.conv_bn1.bn.bias',	(256,),	 'layer3.3.bn1.bias',	(256,),
'block3.3.conv_bn1.bn.running_mean',	(256,),	 'layer3.3.bn1.running_mean',	(256,),
'block3.3.conv_bn1.bn.running_var',	(256,),	 'layer3.3.bn1.running_var',	(256,),
'block3.3.conv_bn2.conv.weight',	(256, 256, 3, 3),	 'layer3.3.conv2.weight',	(256, 256, 3, 3),
'block3.3.conv_bn2.bn.weight',	(256,),	 'layer3.3.bn2.weight',	(256,),
'block3.3.conv_bn2.bn.bias',	(256,),	 'layer3.3.bn2.bias',	(256,),
'block3.3.conv_bn2.bn.running_mean',	(256,),	 'layer3.3.bn2.running_mean',	(256,),
'block3.3.conv_bn2.bn.running_var',	(256,),	 'layer3.3.bn2.running_var',	(256,),
'block3.4.conv_bn1.conv.weight',	(256, 256, 3, 3),	 'layer3.4.conv1.weight',	(256, 256, 3, 3),
'block3.4.conv_bn1.bn.weight',	(256,),	 'layer3.4.bn1.weight',	(256,),
'block3.4.conv_bn1.bn.bias',	(256,),	 'layer3.4.bn1.bias',	(256,),
'block3.4.conv_bn1.bn.running_mean',	(256,),	 'layer3.4.bn1.running_mean',	(256,),
'block3.4.conv_bn1.bn.running_var',	(256,),	 'layer3.4.bn1.running_var',	(256,),
'block3.4.conv_bn2.conv.weight',	(256, 256, 3, 3),	 'layer3.4.conv2.weight',	(256, 256, 3, 3),
'block3.4.conv_bn2.bn.weight',	(256,),	 'layer3.4.bn2.weight',	(256,),
'block3.4.conv_bn2.bn.bias',	(256,),	 'layer3.4.bn2.bias',	(256,),
'block3.4.conv_bn2.bn.running_mean',	(256,),	 'layer3.4.bn2.running_mean',	(256,),
'block3.4.conv_bn2.bn.running_var',	(256,),	 'layer3.4.bn2.running_var',	(256,),
'block3.5.conv_bn1.conv.weight',	(256, 256, 3, 3),	 'layer3.5.conv1.weight',	(256, 256, 3, 3),
'block3.5.conv_bn1.bn.weight',	(256,),	 'layer3.5.bn1.weight',	(256,),
'block3.5.conv_bn1.bn.bias',	(256,),	 'layer3.5.bn1.bias',	(256,),
'block3.5.conv_bn1.bn.running_mean',	(256,),	 'layer3.5.bn1.running_mean',	(256,),
'block3.5.conv_bn1.bn.running_var',	(256,),	 'layer3.5.bn1.running_var',	(256,),
'block3.5.conv_bn2.conv.weight',	(256, 256, 3, 3),	 'layer3.5.conv2.weight',	(256, 256, 3, 3),
'block3.5.conv_bn2.bn.weight',	(256,),	 'layer3.5.bn2.weight',	(256,),
'block3.5.conv_bn2.bn.bias',	(256,),	 'layer3.5.bn2.bias',	(256,),
'block3.5.conv_bn2.bn.running_mean',	(256,),	 'layer3.5.bn2.running_mean',	(256,),
'block3.5.conv_bn2.bn.running_var',	(256,),	 'layer3.5.bn2.running_var',	(256,),
'block4.0.conv_bn1.conv.weight',	(512, 256, 3, 3),	 'layer4.0.conv1.weight',	(512, 256, 3, 3),
'block4.0.conv_bn1.bn.weight',	(512,),	 'layer4.0.bn1.weight',	(512,),
'block4.0.conv_bn1.bn.bias',	(512,),	 'layer4.0.bn1.bias',	(512,),
'block4.0.conv_bn1.bn.running_mean',	(512,),	 'layer4.0.bn1.running_mean',	(512,),
'block4.0.conv_bn1.bn.running_var',	(512,),	 'layer4.0.bn1.running_var',	(512,),
'block4.0.conv_bn2.conv.weight',	(512, 512, 3, 3),	 'layer4.0.conv2.weight',	(512, 512, 3, 3),
'block4.0.conv_bn2.bn.weight',	(512,),	 'layer4.0.bn2.weight',	(512,),
'block4.0.conv_bn2.bn.bias',	(512,),	 'layer4.0.bn2.bias',	(512,),
'block4.0.conv_bn2.bn.running_mean',	(512,),	 'layer4.0.bn2.running_mean',	(512,),
'block4.0.conv_bn2.bn.running_var',	(512,),	 'layer4.0.bn2.running_var',	(512,),
'block4.0.shortcut.conv.weight',	(512, 256, 1, 1),	 'layer4.0.downsample.0.weight',	(512, 256, 1, 1),
'block4.0.shortcut.bn.weight',	(512,),	 'layer4.0.downsample.1.weight',	(512,),
'block4.0.shortcut.bn.bias',	(512,),	 'layer4.0.downsample.1.bias',	(512,),
'block4.0.shortcut.bn.running_mean',	(512,),	 'layer4.0.downsample.1.running_mean',	(512,),
'block4.0.shortcut.bn.running_var',	(512,),	 'layer4.0.downsample.1.running_var',	(512,),
'block4.1.conv_bn1.conv.weight',	(512, 512, 3, 3),	 'layer4.1.conv1.weight',	(512, 512, 3, 3),
'block4.1.conv_bn1.bn.weight',	(512,),	 'layer4.1.bn1.weight',	(512,),
'block4.1.conv_bn1.bn.bias',	(512,),	 'layer4.1.bn1.bias',	(512,),
'block4.1.conv_bn1.bn.running_mean',	(512,),	 'layer4.1.bn1.running_mean',	(512,),
'block4.1.conv_bn1.bn.running_var',	(512,),	 'layer4.1.bn1.running_var',	(512,),
'block4.1.conv_bn2.conv.weight',	(512, 512, 3, 3),	 'layer4.1.conv2.weight',	(512, 512, 3, 3),
'block4.1.conv_bn2.bn.weight',	(512,),	 'layer4.1.bn2.weight',	(512,),
'block4.1.conv_bn2.bn.bias',	(512,),	 'layer4.1.bn2.bias',	(512,),
'block4.1.conv_bn2.bn.running_mean',	(512,),	 'layer4.1.bn2.running_mean',	(512,),
'block4.1.conv_bn2.bn.running_var',	(512,),	 'layer4.1.bn2.running_var',	(512,),
'block4.2.conv_bn1.conv.weight',	(512, 512, 3, 3),	 'layer4.2.conv1.weight',	(512, 512, 3, 3),
'block4.2.conv_bn1.bn.weight',	(512,),	 'layer4.2.bn1.weight',	(512,),
'block4.2.conv_bn1.bn.bias',	(512,),	 'layer4.2.bn1.bias',	(512,),
'block4.2.conv_bn1.bn.running_mean',	(512,),	 'layer4.2.bn1.running_mean',	(512,),
'block4.2.conv_bn1.bn.running_var',	(512,),	 'layer4.2.bn1.running_var',	(512,),
'block4.2.conv_bn2.conv.weight',	(512, 512, 3, 3),	 'layer4.2.conv2.weight',	(512, 512, 3, 3),
'block4.2.conv_bn2.bn.weight',	(512,),	 'layer4.2.bn2.weight',	(512,),
'block4.2.conv_bn2.bn.bias',	(512,),	 'layer4.2.bn2.bias',	(512,),
'block4.2.conv_bn2.bn.running_mean',	(512,),	 'layer4.2.bn2.running_mean',	(512,),
'block4.2.conv_bn2.bn.running_var',	(512,),	 'layer4.2.bn2.running_var',	(512,),
'logit.weight',	(1000, 512),	 'fc.weight',	(1000, 512),
'logit.bias',	(1000,),	 'fc.bias',	(1000,),
]
def load_pretrain_(net, pretrain_file, conversion = CONVERSION, skip=[], is_print=True):
print('\tload pretrain_file: %s'%pretrain_file)
 pretrain_state_dict = torch.load(pretrain_file, map_location=lambda storage, loc: storage)
 state_dict = net.state_dict()
i = 0
conversion = np.array(conversion ).reshape(-1,4)
 for key,_,pretrain_key,_ in conversion:
if any(s in key for s in
['.num_batches_tracked',]+skip):
continue
if is_print:
print('\t\t','%-48s %-24s <--- %-32s %-24s'%(
 key, str(state_dict[key].shape),
pretrain_key, str(pretrain_state_dict[pretrain_key].shape), 
))
i = i+1
state_dict[key] = pretrain_state_dict[key]
net.load_state_dict(state_dict)
 print('')
 print('len(pretrain_state_dict.keys())= %d'%len(pretrain_state_dict.keys()))
print('len(state_dict.keys())= %d'%len(state_dict.keys()))
print('loaded = %d'%i)
 print('')
 class Net(nn.Module):
def load_pretrain(self, skip, is_print=True):
conversion=copy.copy(CONVERSION)
 for i in range(0,len(conversion)-8,4):
conversion[i] = 'block.' + conversion[i][5:]
load_pretrain_(self, pretrain_file=PRETRAIN_FILE, conversion=conversion, is_print=False)
 def __init__(self, num_class=4, drop_connect_rate=0.2):
super(Net, self ).__init__()
e = ResNet34()
self.block = nn.ModuleList([
e.block0,
e.block1,
e.block2,
e.block3,
e.block4,
])
 e = None
self.feature = nn.Conv2d(512,32, kernel_size=1)
 self.logit = nn.Conv2d(32,num_class, kernel_size=1)
 def forward(self, x):
batch_size,C,H,W = x.shape
for i in range(len(self.block)) :
x = self.block[i](x)
 x = F.dropout(x,0.5,training=self.training)
 x = F.adaptive_avg_pool2d(x, 1)
 x = self.feature(x)
 logit = self.logit(x)
 return logit
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))<data_type_conversions>
resnet34_classify = load('/kaggle/input/severstalmodels/resnet34_classify.pth' ).cuda()
unet_mobilenet2 = load('/kaggle/input/severstalmodels/unet_mobilenet2.pth' ).cuda()
ures34 = load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda()
urese = load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda()
device = torch.device("cuda")
 seresnext50 = SCSEUnet(seg_classes = 4, backbone_arch='seresnext50')
 seresnext50 = nn.DataParallel(seresnext50)
 state = torch.load(".. /input/steelfaultmodels0/SEResNext50_combo.pth", map_location=lambda storage, loc: storage)
 seresnext50.load_state_dict(state["state_dict"])
 seresnext50.eval()
seresnext50.to(device)
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 3
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)],
[A.VerticalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True )<data_type_conversions>
thresholds = [0.5, 0.5, 0.5, 0.5]
min_area = [600, 600, 1000, 2000]
thresh_lab = [0.3,0.3,0.3,0.3]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
classes = []
image_file = []
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
cls = torch.sigmoid(resnet34_classify(features))
p = datasets[i].inverse(p)
 preds.append(p)
 classes.append(cls)
 image_file = batch['image_file']
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
classes = torch.stack(classes)
 classes = torch.mean(classes, dim=0)
 classes = classes.detach().cpu().numpy()
for p, cls, file in zip(preds, classes, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
cls_channel = cls[i]
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i] or cls_channel < thresh_lab[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 })
 df = pd.DataFrame(res)
 df.to_csv('submission.csv', index=False )
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8 )<merge>
df = pd.DataFrame(res)
 df = df.fillna('')
 df.to_csv('submission.csv', index=False )
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<statistical_test>
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()
<merge><EOS> 
%matplotlib inline
df = pd.read_csv('submission.csv')[:100]
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 for row in df.itertuples() :
img_path = os.path.join(img_folder, row.Image)
 img = cv2.imread(img_path)
 mask = rle2mask(row.EncodedPixels,(1600, 256)) \
if isinstance(row.EncodedPixels, str)else np.zeros(( 256, 1600))
if mask.sum() == 0:
continue
fig, axes = plt.subplots(1, 2, figsize=(20, 60))
axes[0].imshow(img/255)
 axes[1].imshow(mask*60)
 axes[0].set_title(row.Image)
 axes[1].set_title(row.Class)
 plt.show()