Kaggle competition: Severstal: Steel Defect Detection
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<feature_engineering>
warnings.filterwarnings('ignore')
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS; GCS_PATH3 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES[:FOLDS]):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
GCS_PATH3[i] = KaggleDatasets().get_gcs_path('malignant-v2-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )<normalization>
eff_net = load('.. /input/severstaleffnet/effnetb7_combined.pth' ).cuda()
ROT_ = 180.0; SHR_ = 2.0
HZOOM_ = 8.0; WZOOM_ = 8.0
HSHIFT_ = 8.0; WSHIFT_ = 8.0
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
__all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',
'se_resnext50_32x4d', 'se_resnext101_32x4d']
pretrained_settings = {
'senet154': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet50': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet101': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet152': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext50_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext101_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
}
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):

 super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def initialize_pretrained_model(model, num_classes, settings):
assert num_classes == settings['num_classes'], \
'num_classes should be {}, but is {}'.format(
 settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
def senet154(num_classes=1000, pretrained='imagenet'):
model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,
dropout_p=0.2, num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['senet154'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet50(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 4, 6, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet50'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet101(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 4, 23, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet101'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet152(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet152'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext50_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext101_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
class SENetEncoder(SENet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
del self.last_linear
del self.avg_pool
def forward(self, x):
for module in self.layer0[:-1]:
x = module(x)
 x0 = x
x = self.layer0[-1](x)
 x1 = self.layer1(x)
 x2 = self.layer2(x1)
 x3 = self.layer3(x2)
 x4 = self.layer4(x3)
 features = [x4, x3, x2, x1, x0]
return features
def load_state_dict(self, state_dict, **kwargs):
state_dict.pop('last_linear.bias')
 state_dict.pop('last_linear.weight')
 super().load_state_dict(state_dict, **kwargs)
 senet_encoders = {
'senet154': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 128),
'params': {
'block': SEBottleneck,
'dropout_p': 0.2,
'groups': 64,
'layers': [3, 8, 36, 3],
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet50': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 4, 6, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet101': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 4, 23, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnet152': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNetBottleneck,
'layers': [3, 8, 36, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 1,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnext50_32x4d': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNeXtBottleneck,
'layers': [3, 4, 6, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 32,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
'se_resnext101_32x4d': {
'encoder': SENetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': SEResNeXtBottleneck,
'layers': [3, 4, 23, 3],
'downsample_kernel_size': 1,
'downsample_padding': 0,
'dropout_p': None,
'groups': 32,
'inplanes': 64,
'input_3x3': False,
'num_classes': 1000,
'reduction': 16
},
},
}
class ResNetEncoder(ResNet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
del self.fc
def forward(self, x):
x0 = self.conv1(x)
 x0 = self.bn1(x0)
 x0 = self.relu(x0)
 x1 = self.maxpool(x0)
 x1 = self.layer1(x1)
 x2 = self.layer2(x1)
 x3 = self.layer3(x2)
 x4 = self.layer4(x3)
 return [x4, x3, x2, x1, x0]
def load_state_dict(self, state_dict, **kwargs):
state_dict.pop('fc.bias')
 state_dict.pop('fc.weight')
 super().load_state_dict(state_dict, **kwargs)
 resnet_encoders = {
'resnet18': {
'encoder': ResNetEncoder,
'out_shapes':(512, 256, 128, 64, 64),
'params': {
'block': BasicBlock,
'layers': [2, 2, 2, 2],
},
},
'resnet34': {
'encoder': ResNetEncoder,
'out_shapes':(512, 256, 128, 64, 64),
'params': {
'block': BasicBlock,
'layers': [3, 4, 6, 3],
},
},
'resnet50': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 6, 3],
},
},
'resnet101': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 4, 23, 3],
},
},
'resnet152': {
'encoder': ResNetEncoder,
'out_shapes':(2048, 1024, 512, 256, 64),
'params': {
'block': Bottleneck,
'layers': [3, 8, 36, 3],
},
},
}
class DenseNetEncoder(DenseNet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 self.pretrained = False
del self.classifier
self.initialize()
@staticmethod
def _transition(x, transition_block):
for module in transition_block:
x = module(x)
 if isinstance(module, nn.ReLU):
skip = x
return x, skip
def initialize(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 def forward(self, x):
x = self.features.conv0(x)
 x = self.features.norm0(x)
 x = self.features.relu0(x)
 x0 = x
x = self.features.pool0(x)
 x = self.features.denseblock1(x)
 x, x1 = self._transition(x, self.features.transition1)
 x = self.features.denseblock2(x)
 x, x2 = self._transition(x, self.features.transition2)
 x = self.features.denseblock3(x)
 x, x3 = self._transition(x, self.features.transition3)
 x = self.features.denseblock4(x)
 x4 = self.features.norm5(x)
 features = [x4, x3, x2, x1, x0]
return features
def load_state_dict(self, state_dict):
pattern = re.compile(
 r'^ (.*denselayer\d+\.( ?:norm|relu|conv)) \.((?:[12])\.( ?:weight|bias|running_mean|running_var)) $')
 for key in list(state_dict.keys()):
res = pattern.match(key)
 if res:
new_key = res.group(1)+ res.group(2)
 state_dict[new_key] = state_dict[key]
del state_dict[key]
state_dict.pop('classifier.bias')
 state_dict.pop('classifier.weight')
 super().load_state_dict(state_dict)
 densenet_encoders = {
'densenet121': {
'encoder': DenseNetEncoder,
'out_shapes':(1024, 1024, 512, 256, 64),
'params': {
'num_init_features': 64,
'growth_rate': 32,
'block_config':(6, 12, 24, 16),
}
},
'densenet169': {
'encoder': DenseNetEncoder,
'out_shapes':(1664, 1280, 512, 256, 64),
'params': {
'num_init_features': 64,
'growth_rate': 32,
'block_config':(6, 12, 32, 32),
}
},
'densenet201': {
'encoder': DenseNetEncoder,
'out_shapes':(1920, 1792, 512, 256, 64),
'params': {
'num_init_features': 64,
'growth_rate': 32,
'block_config':(6, 12, 48, 32),
}
},
'densenet161': {
'encoder': DenseNetEncoder,
'out_shapes':(2208, 2112, 768, 384, 96),
'params': {
'num_init_features': 96,
'growth_rate': 48,
'block_config':(6, 12, 36, 24),
}
},
}
encoders = {}
encoders.update(resnet_encoders)
 encoders.update(senet_encoders)
 encoders.update(densenet_encoders)
 def get_encoder(name, encoder_weights=None):
Encoder = encoders[name]['encoder']
encoder = Encoder(**encoders[name]['params'])
 encoder.out_shapes = encoders[name]['out_shapes']
if encoder_weights is not None:
settings = encoders[name]['pretrained_settings'][encoder_weights]
encoder.load_state_dict(model_zoo.load_url(settings['url']))
return encoder
def get_encoder_names() :
return list(encoders.keys())
 def get_preprocessing_params(encoder_name, pretrained='imagenet'):
settings = encoders[encoder_name]['pretrained_settings']
if pretrained not in settings.keys() :
raise ValueError('Avaliable pretrained options {}'.format(settings.keys()))
formatted_settings = {}
formatted_settings['input_space'] = settings[pretrained].get('input_space')
 formatted_settings['input_range'] = settings[pretrained].get('input_range')
 formatted_settings['mean'] = settings[pretrained].get('mean')
 formatted_settings['std'] = settings[pretrained].get('std')
 return formatted_settings
def get_preprocessing_fn(encoder_name, pretrained='imagenet'):
params = get_preprocessing_params(encoder_name, pretrained=pretrained)
 return functools.partial(preprocess_input, **params)
 class Model(nn.Module):
def __init__(self):
super().__init__()
def initialize(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 class EncoderDecoder(Model):
def __init__(self, encoder, decoder, activation):
super().__init__()
self.encoder = encoder
self.decoder = decoder
if callable(activation)or activation is None:
self.activation = activation
elif activation == 'softmax':
self.activation = nn.Softmax(dim=1)
 elif activation == 'sigmoid':
self.activation = nn.Sigmoid()
else:
raise ValueError('Activation should be "sigmoid"/"softmax"/callable/None')
 def forward(self, x):

 x = self.encoder(x)
 x = self.decoder(x)
 return x
def predict(self, x):

 if self.training:
self.eval()
with torch.no_grad() :
x = self.forward(x)
 if self.activation:
x = self.activation(x)
 return x
class DecoderBlock(nn.Module):
def __init__(self, in_channels, out_channels, use_batchnorm=True, attention_type=None):
super().__init__()
if attention_type is None:
self.attention1 = nn.Identity()
self.attention2 = nn.Identity()
elif attention_type == 'scse':
self.attention1 = SCSEModule(in_channels)
 self.attention2 = SCSEModule(out_channels)
 self.block = nn.Sequential(
 Conv2dReLU(in_channels, out_channels, kernel_size=3, padding=1, use_batchnorm=use_batchnorm),
Conv2dReLU(out_channels, out_channels, kernel_size=3, padding=1, use_batchnorm=use_batchnorm), 
)
 def forward(self, x):
x, skip = x
x = F.interpolate(x, scale_factor=2, mode='nearest')
 if skip is not None:
x = torch.cat([x, skip], dim=1)
 x = self.attention1(x)
 x = self.block(x)
 x = self.attention2(x)
 return x
class CenterBlock(DecoderBlock):
def forward(self, x):
return self.block(x)
 class UnetDecoder(Model):
def __init__(
 self,
encoder_channels,
decoder_channels=(256, 128, 64, 32, 16),
final_channels=1,
use_batchnorm=True,
center=False,
attention_type=None 
):
super().__init__()
if center:
channels = encoder_channels[0]
self.center = CenterBlock(channels, channels, use_batchnorm=use_batchnorm)
 else:
self.center = None
in_channels = self.compute_channels(encoder_channels, decoder_channels)
 out_channels = decoder_channels
self.layer1 = DecoderBlock(in_channels[0], out_channels[0],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer2 = DecoderBlock(in_channels[1], out_channels[1],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer3 = DecoderBlock(in_channels[2], out_channels[2],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer4 = DecoderBlock(in_channels[3], out_channels[3],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.layer5 = DecoderBlock(in_channels[4], out_channels[4],
use_batchnorm=use_batchnorm, attention_type=attention_type)
 self.final_conv = nn.Conv2d(out_channels[4], final_channels, kernel_size=(1, 1))
self.initialize()
def compute_channels(self, encoder_channels, decoder_channels):
channels = [
encoder_channels[0] + encoder_channels[1],
encoder_channels[2] + decoder_channels[0],
encoder_channels[3] + decoder_channels[1],
encoder_channels[4] + decoder_channels[2],
0 + decoder_channels[3],
]
return channels
def forward(self, x):
encoder_head = x[0]
skips = x[1:]
if self.center:
encoder_head = self.center(encoder_head)
 x = self.layer1([encoder_head, skips[0]])
 x = self.layer2([x, skips[1]])
 x = self.layer3([x, skips[2]])
 x = self.layer4([x, skips[3]])
 x = self.layer5([x, None])
 x = self.final_conv(x)
 return x
class Conv2dReLU(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size, padding=0,
stride=1, use_batchnorm=True, **batchnorm_params):
super().__init__()
layers = [
nn.Conv2d(in_channels, out_channels, kernel_size,
stride=stride, padding=padding, bias=not(use_batchnorm)) ,
nn.ReLU(inplace=True),
]
if use_batchnorm:
layers.insert(1, nn.BatchNorm2d(out_channels, **batchnorm_params))
self.block = nn.Sequential(*layers)
 def forward(self, x):
return self.block(x)
 class Unet(EncoderDecoder):

 def __init__(
 self,
encoder_name='resnet34',
encoder_weights='imagenet',
decoder_use_batchnorm=True,
decoder_channels=(256, 128, 64, 32, 16),
classes=1,
activation='sigmoid',
center=False,
attention_type=None 
):
encoder = get_encoder(
 encoder_name,
encoder_weights=encoder_weights 
)
 decoder = UnetDecoder(
 encoder_channels=encoder.out_shapes,
decoder_channels=decoder_channels,
final_channels=classes,
use_batchnorm=decoder_use_batchnorm,
center=center,
attention_type=attention_type 
)
 super().__init__(encoder, decoder, activation)
 self.name = 'u-{}'.format(encoder_name )
def dropout(image, DIM=256, PROBABILITY = 0.75, CT = 8, SZ = 0.2):
P = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(P==0)|(CT==0)|(SZ==0): return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image<prepare_x_and_y>
unet2 = Unet('se_resnext50_32x4d', classes=4, encoder_weights=None, activation='softmax')
 ckpt_path = '.. /input/weight-segmentation/se_resnext50_32x4d_Unet_checkpoint_185.pth'
device = torch.device('cuda')
 unet2 = unet2.to(device)
 unet2.eval()
print()
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name=True):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256, droprate=0, dropct=0, dropsize=0):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 if(droprate!=0)&(dropct!=0)&(dropsize!=0):
img = dropout(img, DIM=dim, PROBABILITY=droprate, CT=dropct, SZ=dropsize)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<create_dataframe>
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = eff_net
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256,
droprate=0, dropct=0, dropsize=0):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*2)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(
 prepare_image(img, augment=augment, dim=dim,
droprate=droprate, dropct=dropct, dropsize=dropsize),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<prepare_x_and_y>
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 1
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)],
[A.VerticalFlip(p=1)],
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
nn = target_or_imgid.numpy().decode("utf-8")
 display(mosaic)
 return nn<choose_model_class>
df = pd.DataFrame(res)
 df = df.fillna('')
 df.to_csv('submission.csv', index=False )
<init_hyperparams><EOS> 
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()