Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<drop_column>
import os
from datetime import datetime
import math
import numpy as np
import random
import PIL
import cv2
import torch
from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
from torch.utils.data.sampler import *
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.nn.parallel.data_parallel import data_parallel
from torch.nn.utils.rnn import *
import collections
import copy
import numbers
import inspect
import shutil
from timeit import default_timer as timer
import itertools
from collections import OrderedDict
from multiprocessing import Pool
import multiprocessing as mp
import json
import zipfile
import csv
import pandas as pd
import pickle
import glob
import sys
import time
del bundle_agg
del part_agg
del tags1_agg
gc.collect()<train_model>
DATA_DIR = '/kaggle/input/severstal-steel-defect-detection/test_images'
class Struct(object):
def __init__(self, is_copy=False, **kwargs):
self.add(is_copy, **kwargs)
 def add(self, is_copy=False, **kwargs):
if is_copy == False:
for key, value in kwargs.items() :
setattr(self, key, value)
 else:
for key, value in kwargs.items() :
try:
setattr(self, key, copy.deepcopy(value))
except Exception:
setattr(self, key, value)
 def __str__(self):
text =''
for k,v in self.__dict__.items() :
text += '\t%s : %s
'%(k, str(v))
return text
class SteelDataset(Dataset):
def __init__(self, augment=None):
self.augment = augment
self.uid = os.listdir(DATA_DIR)
 def __len__(self):
return len(self.uid)
 def __getitem__(self, index):
image_id = self.uid[index]
image = cv2.imread(DATA_DIR + '/%s'%(image_id), cv2.IMREAD_COLOR)
 infor = Struct(
 index = index,
image_id = image_id, 
)
 if self.augment is None:
return image, infor
else:
return self.augment(image, infor)
 def null_collate(batch):
batch_size = len(batch)
 input = []
infor = []
for b in range(batch_size):
input.append(batch[b][0])
 infor.append(batch[b][1])
 input = np.stack(input ).astype(np.float32)/255
input = input.transpose(0,3,1,2)
 input = torch.from_numpy(input ).float()
return input, infor
train_df['user_correctness'].fillna(1, inplace=True)
 train_df['attempt_no'].fillna(1, inplace=True)
 train_df.fillna(0, inplace=True )<define_variables>
class _BatchNorm(nn.Module):

 def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True,
track_running_stats=True):
super(_BatchNorm, self ).__init__()
self.num_features = num_features
self.eps = eps
self.momentum = momentum
self.affine = affine
self.track_running_stats = track_running_stats
self.freezed = False
if self.affine:
self.weight = Parameter(torch.Tensor(num_features))
self.bias = Parameter(torch.Tensor(num_features))
else:
self.register_parameter('weight', None)
 self.register_parameter('bias', None)
 if self.track_running_stats:
self.register_buffer('running_mean', torch.zeros(num_features))
self.register_buffer('running_var', torch.ones(num_features))
else:
self.register_parameter('running_mean', None)
 self.register_parameter('running_var', None)
 self.reset_parameters()
def reset_parameters(self):
if self.track_running_stats:
self.running_mean.zero_()
self.running_var.fill_(1)
 if self.affine:
self.weight.data.uniform_()
self.bias.data.zero_()
def _check_input_dim(self, input):
return NotImplemented
def forward(self, input):
self._check_input_dim(input)
 compute_stats = not self.freezed and \
self.training and self.track_running_stats
ret = F.batch_norm(input, self.running_mean, self.running_var,
self.weight, self.bias, compute_stats,
self.momentum, self.eps)
 return ret
def extra_repr(self):
return '{num_features}, eps={eps}, momentum={momentum}, '\
'affine={affine}, ' \
'track_running_stats={track_running_stats}'.format(
 **self.__dict__)
 class BatchNorm2dNoSync(_BatchNorm):

 def _check_input_dim(self, input):
if input.dim() != 4:
raise ValueError('expected 4D input(got {}D input)' 
.format(input.dim()))
class BatchNorm2dSync(BatchNorm2dNoSync):

 def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True,
track_running_stats=True):
super(BatchNorm2dSync, self ).__init__(
 num_features, eps=eps, momentum=momentum, affine=affine,
track_running_stats=track_running_stats)
 self.sync_enabled = True
self.devices = list(range(torch.cuda.device_count()))
if len(self.devices)> 1:
self.worker_ids = self.devices[1:]
self.master_queue = Queue(len(self.worker_ids))
self.worker_queues = [Queue(1)for _ in self.worker_ids]
def forward(self, x):
compute_stats = not self.freezed and \
self.training and self.track_running_stats
if self.sync_enabled and compute_stats and len(self.devices)> 1:
if x.get_device() == self.devices[0]:
extra = {
"is_master": True,
"master_queue": self.master_queue,
"worker_queues": self.worker_queues,
"worker_ids": self.worker_ids
}
else:
extra = {
"is_master": False,
"master_queue": self.master_queue,
"worker_queue": self.worker_queues[
self.worker_ids.index(x.get_device())]
}
return batchnorm2d_sync(x, self.weight, self.bias,
self.running_mean, self.running_var,
extra, compute_stats, self.momentum,
self.eps)
 return super(BatchNorm2dSync, self ).forward(x)
 def __repr__(self):

 rep = '{name}({num_features}, eps={eps}, momentum={momentum},' \
'affine={affine}, ' \
'track_running_stats={track_running_stats},' \
'devices={devices})'
return rep.format(name=self.__class__.__name__, **self.__dict__ )
features = [
'timestamp',
'lagtime',
'lagtime_mean',
'content_id',
'task_container_id',
'user_lecture_cumsum',
'user_lecture_lv',
'prior_question_elapsed_time',
'delta_prior_question_elapsed_time',
'user_correctness',
'user_correct_cumcount',
'user_correct_cumsum',
'content_correctness',
'content_correctness_std',
'content_count',
'content_sum',
'task_container_correctness',
'task_container_std',
'task_container_sum',
'bundle_correctness',
'attempt_no',
'part',
'part_correctness_mean',
'part_correctness_std',
'tags1',
'tags1_correctness_mean',
'tags1_correctness_std',
'tags2',
'tags3',
'tags4',
'tags5',
'tags6',
'bundle_id',
'part_bundle_id',
'explanation_mean',
'explanation_cumsum',
'prior_question_had_explanation',
]
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'tags6',
'bundle_id',
'part_bundle_id',
'prior_question_had_explanation',
] 
<init_hyperparams>
BatchNorm2d = BatchNorm2dSync
class ConvBn2d(nn.Module):
def __init__(self, in_channel, out_channel, kernel_size=3, padding=1, stride=1):
super(ConvBn2d, self ).__init__()
self.conv = nn.Conv2d(in_channel, out_channel, kernel_size=kernel_size, padding=padding, stride=stride, bias=False)
 self.bn = nn.BatchNorm2d(out_channel, eps=1e-5)
 def forward(self,x):
x = self.conv(x)
 x = self.bn(x)
 return x
class BasicBlock(nn.Module):
def __init__(self, in_channel, channel, out_channel, stride=1, is_shortcut=False):
super(BasicBlock, self ).__init__()
self.is_shortcut = is_shortcut
self.conv_bn1 = ConvBn2d(in_channel, channel, kernel_size=3, padding=1, stride=stride)
 self.conv_bn2 = ConvBn2d(channel,out_channel, kernel_size=3, padding=1, stride=1)
 if is_shortcut:
self.shortcut = ConvBn2d(in_channel, out_channel, kernel_size=1, padding=0, stride=stride)
 def forward(self, x):
z = F.relu(self.conv_bn1(x),inplace=True)
 z = self.conv_bn2(z)
 if self.is_shortcut:
x = self.shortcut(x)
 z += x
z = F.relu(z,inplace=True)
 return z
class ResNet18(nn.Module):
def __init__(self, num_class=1000):
super(ResNet18, self ).__init__()
self.block0 = nn.Sequential(
 nn.Conv2d(3, 64, kernel_size=7, padding=3, stride=2, bias=False),
BatchNorm2d(64),
nn.ReLU(inplace=True), 
)
 self.block1 = nn.Sequential(
 nn.MaxPool2d(kernel_size=3, padding=1, stride=2),
BasicBlock(64, 64, 64, stride=1, is_shortcut=False,),
* [BasicBlock(64, 64, 64, stride=1, is_shortcut=False,)for i in range(1,2)], 
)
 self.block2 = nn.Sequential(
 BasicBlock(64,128,128, stride=2, is_shortcut=True,),
* [BasicBlock(128,128,128, stride=1, is_shortcut=False,)for i in range(1,2)], 
)
 self.block3 = nn.Sequential(
 BasicBlock(128,256,256, stride=2, is_shortcut=True,),
* [BasicBlock(256,256,256, stride=1, is_shortcut=False,)for i in range(1,2)], 
)
 self.block4 = nn.Sequential(
 BasicBlock(256,512,512, stride=2, is_shortcut=True,),
* [BasicBlock(512,512,512, stride=1, is_shortcut=False,)for i in range(1,2)], 
)
 self.logit = nn.Linear(512,num_class)
 def forward(self, x):
batch_size = len(x)
 x = self.block0(x)
 x = F.max_pool2d(x,kernel_size=3, padding=1, stride=2, ceil_mode=False)
 x = self.block1(x)
 x = self.block2(x)
 x = self.block3(x)
 x = self.block4(x)
 x = F.adaptive_avg_pool2d(x,1 ).reshape(batch_size,-1)
 logit = self.logit(x)
 return logit
def upsize(x,scale_factor=2):
x = F.interpolate(x, scale_factor=scale_factor, mode='nearest')
 return x
class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 class Decode(nn.Module):
def __init__(self, in_channel, out_channel):
super(Decode, self ).__init__()
self.top = nn.Sequential(
 nn.Conv2d(in_channel, out_channel//2, kernel_size=3, stride=1, padding=1, bias=False),
BatchNorm2d(out_channel//2),
nn.ReLU(inplace=True),
nn.Conv2d(out_channel//2, out_channel//2, kernel_size=3, stride=1, padding=1, bias=False),
BatchNorm2d(out_channel//2),
nn.ReLU(inplace=True),
nn.Conv2d(out_channel//2, out_channel, kernel_size=1, stride=1, padding=0, bias=False),
BatchNorm2d(out_channel),
nn.ReLU(inplace=True), 
)
 def forward(self, x):
x = self.top(torch.cat(x, 1))
return x
class Net(nn.Module):
def load_pretrain(self, skip, is_print=True):
conversion=copy.copy(CONVERSION)
 for i in range(0,len(conversion)-8,4):
conversion[i] = 'block.' + conversion[i][5:]
load_pretrain(self, skip, pretrain_file=PRETRAIN_FILE, conversion=conversion, is_print=is_print)
 def __init__(self, num_class=5, drop_connect_rate=0.2):
super(Net, self ).__init__()
e = ResNet18()
self.block = nn.ModuleList([
e.block0,
e.block1,
e.block2,
e.block3,
e.block4
])
 e = None
self.decode1 = Decode(512, 128)
 self.decode2 = Decode(128+256, 128)
 self.decode3 = Decode(128+128, 128)
 self.decode4 = Decode(128+ 64, 128)
 self.decode5 = Decode(128+ 64, 128)
 self.logit = nn.Conv2d(128,num_class, kernel_size=1)
 def forward(self, x):
batch_size,C,H,W = x.shape
backbone = []
for i in range(len(self.block)) :
x = self.block[i](x)
 if i in [0,1,2,3,4]:
backbone.append(x)
 x = self.decode1([backbone[-1], ])
 x = self.decode2([backbone[-2], upsize(x)])
 x = self.decode3([backbone[-3], upsize(x)])
 x = self.decode4([backbone[-4], upsize(x)])
 x = self.decode5([backbone[-5], upsize(x)])
 logit = self.logit(x)
 logit = F.interpolate(logit, size=(H,W), mode='bilinear', align_corners=False)
 return logit
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 350,
'max_bin':700,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.58,
'bagging_fraction': 0.58,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.05,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df.sample(n=1200*10000)
 print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.025)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 valid_df=train_df_clf.sample(frac=0.15)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('valid_df length：',len(valid_df)) 
<drop_column>
def one_hot_encode_truth(truth, num_class=4):
one_hot = truth.repeat(1,num_class,1,1)
 arange = torch.arange(1,num_class+1 ).view(1,num_class,1,1 ).to(truth.device)
 one_hot =(one_hot == arange ).float()
return one_hot
def one_hot_encode_predict(predict, num_class=4):
value, index = torch.max(predict, 1, keepdim=True)
 value = value.repeat(1,num_class,1,1)
 index = index.repeat(1,num_class,1,1)
 arange = torch.arange(1,num_class+1 ).view(1,num_class,1,1 ).to(predict.device)
 one_hot =(index == arange ).float()
value = value*one_hot
return value
def remove_small_one(predict, min_size):
H,W = predict.shape
num_component, component = cv2.connectedComponents(predict.astype(np.uint8))
predict = np.zeros(( H,W), np.bool)
 for c in range(1,num_component):
p =(component==c)
 if p.sum() >min_size:
predict[p] = True
return predict
def remove_small(predict, min_size):
for b in range(len(predict)) :
for c in range(4):
predict[b,c] = remove_small_one(predict[b,c], min_size[c])
 return predict
def run_length_encode(mask):
m = mask.T.flatten()
if m.sum() ==0:
rle=''
else:
m = np.concatenate([[0], m, [0]])
 run = np.where(m[1:] != m[:-1])[0] + 1
run[1::2] -= run[::2]
rle = ' '.join(str(r)for r in run)
 return rle
def time_to_str(t, mode='min'):
if mode=='min':
t = int(t)/60
hr = t//60
min = t%60
return '%2d hr %02d min'%(hr,min)
 elif mode=='sec':
t = int(t)
 min = t//60
sec = t%60
return '%2d min %02d sec'%(min,sec)
 else:
raise NotImplementedError
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
def do_evaluate_save(mode, net, test_dataset, augment=[], out_dir=None):
csv_file = 'submission.csv'
image_id_class_id = []
encoded_pixel = []
threshold_pixel = [0.50,0.50,0.50,0.50,]
threshold_size = [200,1500,1500,2000,]
test_loader = DataLoader(
 test_dataset,
sampler = SequentialSampler(test_dataset),
batch_size = 8,
drop_last = False,
num_workers = 0,
pin_memory = True,
collate_fn = null_collate 
)
 def sharpen(p,t=0):
if t!=0:
return p**t
else:
return p
test_num = 0
start = timer()
for t,(input, infor)in enumerate(test_loader):
batch_size,C,H,W = input.shape
input = input.cuda()
with torch.no_grad() :
net.eval()
num_augment = 0
if 1:
logit = data_parallel(net,input)
 probability = torch.softmax(logit,1)
 probability_mask = sharpen(probability,0)
 num_augment+=1
if 'flip_lr' in augment:
logit = data_parallel(net,torch.flip(input,dims=[3]))
probability = torch.softmax(torch.flip(logit,dims=[3]),1)
 probability_mask += sharpen(probability)
 num_augment+=1
if 'flip_ud' in augment:
logit = data_parallel(net,torch.flip(input,dims=[2]))
probability = torch.softmax(torch.flip(logit,dims=[2]),1)
 probability_mask += sharpen(probability)
 num_augment+=1
probability_mask = probability_mask/num_augment
probability = probability_mask.clone()
probability_mask = one_hot_encode_predict(probability_mask)
 batch_size = len(infor)
 probability_mask = probability_mask.data.cpu().numpy()
probability = probability.data.cpu().numpy()
predict_mask = probability_mask>np.array(threshold_pixel ).reshape(1,4,1,1)
 predict_mask = remove_small(predict_mask, threshold_size)
 predict_label =(( predict_mask.sum(-1 ).sum(-1)) >0 ).astype(np.int32)
 for b in range(len(infor)) :
for c in range(4):
image_id_class_id.append(infor[b].image_id+'_%d'%(c+1))
if predict_label[b,c]==0:
rle=''
else:
rle = run_length_encode(predict_mask[b,c])
 encoded_pixel.append(rle)
 test_num += batch_size
print('\r %4d / %4d %s'%(
 test_num, len(test_loader.dataset), time_to_str(( timer() - start),'min')
),end='',flush=True)
 assert(test_num == len(test_loader.dataset))
print('')
 df = pd.DataFrame(zip(image_id_class_id, encoded_pixel), columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv(csv_file, index=False)
 print('>>>csv saved!' )
<install_modules><EOS> 
def run_submit() :
out_dir = \
'/kaggle/input'
initial_checkpoint = \
'/kaggle/input/resnetmodel/00114000_model.pth'
mode = 'test'
augment = ['null', 'flip_lr','flip_ud']
test_dataset = SteelDataset()
net = Net().cuda()
net.load_state_dict(torch.load(initial_checkpoint, map_location=lambda storage, loc: storage), strict=False)
 do_evaluate_save(mode, net, test_dataset, augment,out_dir)
 def test4() :
csv_data = pd.read_csv("/kaggle/input/uploadcsv/resnet18-softmax-tta-0.50.csv")
 csv_data.to_csv('submission.csv', index=False)
 if __name__ == "__main__":
run_submit()
print('>>>Done!' )