Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<string_transform>
import mxnet as mx
from mxnet.gluon import data, HybridBlock, nn
import pandas as pd
import cv2
import os
import numpy as np
from mxnet.gluon.data.vision import transforms
from mxnet.gluon.model_zoo import vision
import glob
from mxnet import nd as F, gluon
from gluoncv import model_zoo as gm
def clean_name(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x<feature_engineering>
%pylab inline
items["name1"], items["name2"] = items.item_name.str.split("[", 1 ).str
items["name1"], items["name3"] = items.item_name.str.split("(", 1 ).str
items["name2"] = items.name2.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items["name3"] = items.name3.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items = items.fillna('0')
 items["item_name"] = items["item_name"].apply(lambda x: clean_name(x))
items.name2 = items.name2.apply(lambda x: x[:-1] if x !="0" else "0" )<feature_engineering>
ls.. /input/mxnet-gluon-baseline/model/
items["type"] = items.name2.apply(lambda x: x[0:8] if x.split(" ")[0] == "xbox" else x.split(" ")[0])
 items.loc[(items.type == "x360")|(items.type == "xbox360")|(items.type == "xbox 360"),"type"] = "xbox 360"
items.loc[ items.type == "", "type"] = "mac"
items.type = items.type.apply(lambda x: x.replace(" ", ""))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == "pc"), "type" ] = "pc"
items.loc[ items.type == 'рs3' , "type"] = "ps3"<groupby>
class ResNetBackbone(mx.gluon.HybridBlock):
def __init__(self, backbone='resnet50', pretrained_base=True,dilated=True, **kwargs):
super(ResNetBackbone, self ).__init__()
with self.name_scope() :
if backbone == 'resnet50':
pretrained = resnet50_v1s(pretrained=pretrained_base, dilated=dilated, **kwargs)
 elif backbone == 'resnet101':
pretrained = resnet101_v1s(pretrained=pretrained_base, dilated=dilated, **kwargs)
 elif backbone == 'resnet152':
pretrained = resnet152_v1s(pretrained=pretrained_base, dilated=dilated, **kwargs)
 else:
raise RuntimeError(f'unknown backbone: {backbone}')
 self.conv1 = pretrained.conv1
self.bn1 = pretrained.bn1
self.relu = pretrained.relu
self.maxpool = pretrained.maxpool
self.layer1 = pretrained.layer1
self.layer2 = pretrained.layer2
self.layer3 = pretrained.layer3
self.layer4 = pretrained.layer4
def hybrid_forward(self, F, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 c1 = self.layer1(x)
 c2 = self.layer2(c1)
 c3 = self.layer3(c2)
 c4 = self.layer4(c3)
 return c1, c2, c3, c4
group_sum = items.groupby(["type"], as_index=False ).agg({"item_id": "count"})
 group_sum<filter>
class ResNetFPN(mx.gluon.HybridBlock):
def __init__(self, backbone= 'resnet50', backbone_lr_mult=0.1, **kwargs):
super(ResNetFPN, self ).__init__()
self.backbone_name = backbone
self.backbone_lr_mult = backbone_lr_mult
self._kwargs = kwargs
with self.name_scope() :
self.backbone = ResNetBackbone(backbone=self.backbone_name, pretrained_base=False, dilated=False, **kwargs)
 self.head = _FPNHead(output_channels=256, **kwargs)
 def load_pretrained_weights(self):
pretrained = ResNetBackbone(backbone=self.backbone_name, pretrained_base=True, dilated=False, **self._kwargs)
 backbone_params = self.backbone.collect_params()
pretrained_weights = pretrained.collect_params()
for k, v in pretrained_weights.items() :
param_name = backbone_params.prefix + k[len(pretrained_weights.prefix):]
backbone_params[param_name].set_data(v.data())
 self.backbone.collect_params().setattr('lr_mult', self.backbone_lr_mult)
 def hybrid_forward(self,F, x):
c1, c2, c3, c4 = self.backbone(x)
 p1, p2, p3, p4 = self.head(c1, c2, c3, c4)
 return p1, p2, p3, p4
class ResNetUnet(mx.gluon.HybridBlock):
def __init__(self, backbone= 'resnet50', backbone_lr_mult=0.1, cls_branch=False, **kwargs):
super(ResNetUnet, self ).__init__()
self.backbone_name = backbone
self.backbone_lr_mult = backbone_lr_mult
self.cls_branch = cls_branch
self._kwargs = kwargs
with self.name_scope() :
self.backbone = ResNetBackbone(backbone=self.backbone_name, pretrained_base=False, dilated=False, **kwargs)
 self.head = _UnetHead(**kwargs)
 def load_pretrained_weights(self):
pretrained = ResNetBackbone(backbone=self.backbone_name, pretrained_base=True, dilated=False, **self._kwargs)
 backbone_params = self.backbone.collect_params()
pretrained_weights = pretrained.collect_params()
for k, v in pretrained_weights.items() :
param_name = backbone_params.prefix + k[len(pretrained_weights.prefix):]
backbone_params[param_name].set_data(v.data())
 self.backbone.collect_params().setattr('lr_mult', self.backbone_lr_mult)
 def hybrid_forward(self,F, x):
c1, c2, c3, c4 = self.backbone(x)
 out = self.head(c1, c2, c3, c4)
 if self.cls_branch:
logits = F.max(F.mean(out, axis=1), axis=(1, 2))
return out, logits
return out
class _DecoderBlock(HybridBlock):
def __init__(self, output_channels, norm_layer=nn.BatchNorm):
super(_DecoderBlock, self ).__init__()
with self.name_scope() :
self.block = nn.HybridSequential()
self.block.add(ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer))
self.block.add(ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer))
def hybrid_forward(self, F, x, y=None):
if y is not None:
x = F.contrib.BilinearResize2D(x, scale_height=2, scale_width=2)
 x = F.concat(x, y, dim=1)
 out = self.block(x)
 return out
class _UnetHead(HybridBlock):
def __init__(self, num_classes, output_channels=[256, 128, 64, 32], scale=4, norm_layer=nn.BatchNorm):
super(_UnetHead, self ).__init__()
self.scale = scale
with self.name_scope() :
self.block4 = _DecoderBlock(output_channels[0], norm_layer=norm_layer)
 self.block3 = _DecoderBlock(output_channels[1], norm_layer=norm_layer)
 self.block2 = _DecoderBlock(output_channels[2], norm_layer=norm_layer)
 self.block1 = _DecoderBlock(output_channels[3], norm_layer=norm_layer)
 self.postprocess_block = nn.Conv2D(num_classes, kernel_size=1)
 def hybrid_forward(self, F, c1, c2, c3, c4):
p4 = self.block4(c4)
 p3 = self.block3(p4, c3)
 p2 = self.block2(p3, c2)
 p1 = self.block1(p2, c1)
 if self.scale > 1:
p1 = F.contrib.BilinearResize2D(p1, scale_height=self.scale, scale_width=self.scale)
 out = self.postprocess_block(p1)
 return out
class _FPNHead(HybridBlock):
def __init__(self, output_channels=256, norm_layer=nn.BatchNorm):
super(_FPNHead, self ).__init__()
self._hdsize = {}
with self.name_scope() :
self.block4 = ConvBlock(output_channels, kernel_size=1, norm_layer=norm_layer)
 self.block3 = ConvBlock(output_channels, kernel_size=1, norm_layer=norm_layer)
 self.block2 = ConvBlock(output_channels, kernel_size=1, norm_layer=norm_layer)
 self.block1 = ConvBlock(output_channels, kernel_size=1, norm_layer=norm_layer)
 def hybrid_forward(self, F, c1, c2, c3, c4):
p4 = self.block4(c4)
 p3 = self._resize_as(F, 'id_1', p4, c3)+ self.block3(c3)
 p2 = self._resize_as(F, 'id_2', p3, c2)+ self.block2(c2)
 p1 = self._resize_as(F, 'id_3', p2, c1)+ self.block1(c1)
 return p1, p2, p3, p4
def _resize_as(self, F, name, x, y):
h_key = name + '_h'
w_key = name + '_w'
if hasattr(y, 'shape'):
_, _, h, w = y.shape
_, _, h2, w2 = x.shape
if h == h2 and w == w2:
h = 0
w = 0
self._hdsize[h_key] = h
self._hdsize[w_key] = w
else:
h, w = self._hdsize[h_key], self._hdsize[w_key]
if h == 0 and w == 0:
return x
else:
return F.contrib.BilinearResize2D(x, height=h, width=w)
 class SemanticFPNHead(HybridBlock):
def __init__(self, num_classes, output_channels=128, norm_layer=nn.BatchNorm):
super(SemanticFPNHead, self ).__init__()
self._hdsize = {}
with self.name_scope() :
self.block4_1 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block4_2 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block4_3 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block3_1 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block3_2 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block2 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block1 = ConvBlock(output_channels, kernel_size=1, norm_layer=norm_layer)
 self.postprocess_block = nn.Conv2D(num_classes, kernel_size=1)
 def hybrid_forward(self, F, c1, c2, c3, c4):
out4 = self._resize_as(F, 'id_1', self.block4_1(c4), c3)
 out4 = self._resize_as(F, 'id_2', self.block4_2(out4), c2)
 out4 = self._resize_as(F, 'id_3', self.block4_3(out4), c1)
 out3 = self._resize_as(F, 'id_4', self.block3_1(c3), c2)
 out3 = self._resize_as(F, 'id_5', self.block3_2(out3), c1)
 out2 = self._resize_as(F, 'id_6', self.block2(c2), c1)
 out1 = self.block1(c1)
 out = out1 + out2 + out3 + out4
out = self.postprocess_block(out)
 out = F.contrib.BilinearResize2D(out,scale_height=4,scale_width=4)
 return out
def _resize_as(self, F,name, x, y):
h_key = name + '_h'
w_key = name + '_w'
if hasattr(y, 'shape'):
_, _, h, w = y.shape
_, _, h2, w2 = x.shape
if h == h2 and w == w2:
h = 0
w = 0
self._hdsize[h_key]=h
self._hdsize[w_key]=w
else:
h, w = self._hdsize[h_key], self._hdsize[w_key]
if h == 0 and w == 0:
return x
else:
return F.contrib.BilinearResize2D(x,height=h,width=w)
 class ConvBlock(HybridBlock):
def __init__(self, output_channels, kernel_size, padding=0, activation='relu', norm_layer=nn.BatchNorm):
super().__init__()
self.body = nn.HybridSequential()
self.body.add(
 nn.Conv2D(output_channels, kernel_size=kernel_size, padding=padding, activation=activation),
norm_layer(in_channels=output_channels)
)
 def hybrid_forward(self, F, x):
return self.body(x)
drop_cols = group_sum.loc[group_sum['item_id']<40,'type'].values.tolist()<drop_column>
class ResNetSteel(mx.gluon.HybridBlock):
def __init__(self, backbone= 'resnet50', num_classes=4, backbone_lr_mult=0.1, **kwargs):
super(ResNetSteel, self ).__init__()
self.backbone_name = backbone
self.backbone_lr_mult = backbone_lr_mult
self._kwargs = kwargs
with self.name_scope() :
self.backbone = ResNetBackbone(backbone=self.backbone_name, pretrained_base=False, dilated=False, **kwargs)
 self.head = Classification_head(output_channels=256, num_classes=num_classes)
 def load_pretrained_weights(self):
pretrained = ResNetBackbone(backbone=self.backbone_name, pretrained_base=True, dilated=False, **self._kwargs)
 backbone_params = self.backbone.collect_params()
pretrained_weights = pretrained.collect_params()
for k, v in pretrained_weights.items() :
param_name = backbone_params.prefix + k[len(pretrained_weights.prefix):]
backbone_params[param_name].set_data(v.data())
 self.backbone.collect_params().setattr('lr_mult', self.backbone_lr_mult)
 def hybrid_forward(self,F, x):
c1, c2, c3, c4 = self.backbone(x)
 logits = self.head(c4)
 return logits
class Classification_head(HybridBlock):
def __init__(self, output_channels=256, num_classes=4):
super(Classification_head, self ).__init__()
with self.name_scope() :
self.cls_head = nn.HybridSequential()
self.cls_head.add(ConvBlock(output_channels, kernel_size=1))
self.cls_head.add(nn.GlobalAvgPool2D())
 self.cls_head.add(nn.Conv2D(num_classes, kernel_size=1))
def hybrid_forward(self, F, x):
logits = self.cls_head(x)
 return F.squeeze(logits)
items.name2 = items.name2.apply(lambda x: "etc" if(x in drop_cols)else x)
 items = items.drop(["type"], axis = 1 )<categorify>
ctx = mx.gpu()
cls_net = ResNetSteel(num_classes=4)
 cls_net.collect_params().initialize()
cls_net.load_parameters('.. /input/mxnet-gluon-classification/unet_4_0.0.params')
 cls_net.collect_params().reset_ctx(ctx )
items.name2 = LabelEncoder().fit_transform(items.name2)
 items.name3 = LabelEncoder().fit_transform(items.name3)
 items.drop(["item_name", "name1"],axis = 1, inplace= True)
 items.head()<data_type_conversions>
unet = ResNetUnet(output_channels=[256, 128, 64, 32], num_classes=5)
 unet.load_parameters('.. /input/mxnet-gluon-baseline/unet_34_-1.params')
 unet.collect_params().reset_ctx(ctx )
ts = time.time()
matrix = []
cols = ["date_block_num", "shop_id", "item_id"]
for i in range(34):
sales = train[train.date_block_num == i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype = np.int16))
matrix = pd.DataFrame(np.vstack(matrix), columns = cols)
 matrix["date_block_num"] = matrix["date_block_num"].astype(np.int8)
 matrix["shop_id"] = matrix["shop_id"].astype(np.int8)
 matrix["item_id"] = matrix["item_id"].astype(np.int16)
 matrix.sort_values(cols, inplace = True)
 time.time() - ts<feature_engineering>
def mask2rle(mask):
if np.sum(mask)== 0: return ''
ar = mask.flatten(order='F')
 EncodedPixel = ''
l = 0
for i in range(len(ar)) :
if ar[i] == 0:
if l > 0:
if EncodedPixel != '': EncodedPixel += ' '
EncodedPixel += str(st+1)+' '+str(l)
 l = 0
else:
if l == 0: st = i
l += 1
return EncodedPixel
train["revenue"] = train["item_cnt_day"] * train["item_price"]<merge>
def remove_small_one(predict, min_size):
H,W = predict.shape
num_component, component = cv2.connectedComponents(predict.astype(np.uint8))
predict = np.zeros(( H,W), np.bool)
 for c in range(1,num_component):
p =(component==c)
 if p.sum() >min_size:
predict[p] = True
return predict
<data_type_conversions><EOS> 
submission = pd.read_csv(".. /input/severstal-steel-defect-detection/sample_submission.csv")
 print(len(ImageId),len(submission['ImageId_ClassId']))
for i, encoded in zip(ImageId_ClassIds,EncodedPixels):
submission.loc[submission['ImageId_ClassId']==i,["EncodedPixels"]] = encoded
submission.to_csv('submission.csv',index=False )