Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<merge>
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/unetmodelscript'
sys.path.append(package_path )
test_df=test_df.join(shops_df.set_index("shop_id"),on="shop_id",how="left" ).join(
 items_df.set_index("item_id"),on="item_id",how="left" ).join(item_categories_df.set_index("item_category_id")
 ,on="item_category_id",how="left")
 test_df.head().T<filter>
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.torch import ToTensor
import torch.utils.data as data
import torchvision.models as models
import torch.nn as nn
from torch.nn import functional as F
from model import Unet
sales_train_df[sales_train_df.item_cnt_day>2000]<filter>
class Resnet34_classification(nn.Module):
def __init__(self,num_class=4):
super(Resnet34_classification, self ).__init__()
e = ResNet34()
self.block = nn.ModuleList([
e.block0,
e.block1,
e.block2,
e.block3,
e.block4,
])
 e = None
self.feature = nn.Conv2d(512,32, kernel_size=1)
 self.logit = nn.Conv2d(32,num_class, kernel_size=1)
 def forward(self, x):
batch_size,C,H,W = x.shape
for i in range(len(self.block)) :
x = self.block[i](x)
 x = F.dropout(x,0.5,training=self.training)
 x = F.adaptive_avg_pool2d(x, 1)
 x = self.feature(x)
 logit = self.logit(x)
 return logit
sales_train_df[sales_train_df.item_category_id==9]["item_cnt_day"].mean()<filter>
model_classification = Resnet34_classification()
model_classification.load_state_dict(torch.load('.. /input/clsification/00007500_model.pth', map_location=lambda storage, loc: storage), strict=True )
sales_train_df[(sales_train_df.shop_id==12)&(sales_train_df.item_id==11373)]["item_cnt_day"].mean()<filter>
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_cnt_day>2000])]<filter>
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"
sales_train_df[sales_train_df.item_price>50000]<filter>
batch_size = 1
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225 )
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_price>50000])]<filter>
df = pd.read_csv(sample_submission_path )
sales_train_df[sales_train_df.item_cnt_day>800]<filter>
testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=0,
pin_memory=True
)
sales_train_df[(sales_train_df.shop_id==12)&(sales_train_df.item_id==20949)]["item_cnt_day"].mean()<filter>
threshold_label = [0.50,0.50,0.50,0.50,]
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_cnt_day>800])]<filter>
augment = ['null']
test_unique_items=test_df[~test_df["item_id"].isin(item_id_uniq)]
test_unique_items.head()<data_type_conversions>
probability_label, image_id = get_classification_preds(model_classification, testset)
 predict_label = probability_label>np.array(threshold_label ).reshape(1,4,1,1)
 image_id_class_id = []
encoded_pixel = []
for b in range(len(image_id)) :
for c in range(4):
image_id_class_id.append(image_id[b]+'_%d'%(c+1))
if predict_label[b,c]==0:
rle=''
else:
rle ='1 1'
encoded_pixel.append(rle)
 df_classification = pd.DataFrame(zip(image_id_class_id, encoded_pixel), columns=['ImageId_ClassId', 'EncodedPixels'] )
%%time
train=[]
col=["date_block_num","shop_id","item_id"]
for i in range(34):
sales=sales_train_df[sales_train_df.date_block_num==i]
train.append(np.array(list(product([i],sales.shop_id.unique() ,sales.item_id.unique())) ,dtype="int16"))
train=pd.DataFrame(np.vstack(train),columns=col)
 train["date_block_num"]=train["date_block_num"].astype(np.int8)
 train["shop_id"]=train["shop_id"].astype(np.int8)
 train["item_id"]=train["item_id"].astype(np.int16 )<groupby>
ckpt_path = ".. /input/res18fullsize/model_swa.pth"
device = torch.device("cuda")
 model_segmentation = Unet("resnet18", encoder_weights=None, classes=4, activation=None)
 model_segmentation.to(device)
 model_segmentation.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model_segmentation.load_state_dict(state["state_dict"] )
month_df=sales_train_df[["date_block_num","shop_id","item_id","item_cnt_day"]].groupby(["date_block_num","shop_id","item_id"] ).sum()<groupby>
models = [model_segmentation]
date_block_total=month_df.reset_index()
date_block_total_group=date_block_total[["date_block_num","item_cnt_day"]].groupby("date_block_num",as_index=False ).sum()<feature_engineering>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
date_block_total["year"]=date_block_total.date_block_num.apply(lambda x:(( x//12)+2013))
date_block_total["month"]=date_block_total.date_block_num.apply(lambda x:(x%12))<groupby>
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num
total_month=date_block_total[["month","item_cnt_day"]].groupby("month",as_index=False ).sum()<groupby>
threshold_pixel = [0.5,0.5,0.5,0.5,]
min_size = [200,1500,1500,2000]
total_year=date_block_total[["year","item_cnt_day"]].groupby("year",as_index=False ).sum()<data_type_conversions>
def null_augment(input): return input
def flip_lr_augment(input): return torch.flip(input, dims=[2])
 def flip_ud_augment(input): return torch.flip(input, dims=[3])
 def null_inverse_augment(logit): return logit
def flip_lr_inverse_augment(logit): return torch.flip(logit, dims=[2])
 def flip_ud_inverse_augment(logit): return torch.flip(logit, dims=[3])
 augment =(
(null_augment, null_inverse_augment), 
(flip_lr_augment,flip_lr_inverse_augment), 
(flip_ud_augment,flip_ud_inverse_augment),
)
train["year"]=train.date_block_num.apply(lambda x:(( x//12)+2013))
train["month"]=train.date_block_num.apply(lambda x:(x%12))
train["year"]=train["year"].astype(np.int16)
 train["month"]=train["month"].astype(np.int8)
 train.head()<data_type_conversions>
predictions = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
images = images.cuda()
batch_preds = 0
probabilities = []
for model in models:
model = model.cuda()
for k,(a, inv_a)in enumerate(augment):
logit = model(a(images))
p = inv_a(torch.sigmoid(logit))
if k ==0:
probability = p**0.5
else:
probability += p**0.5
probability = probability/len(augment)
 probabilities.append(probability)
 batch_preds+=probability
batch_preds = batch_preds.data.cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, threshold_pixel[cls], min_size[cls])
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df_segmentation = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'] )
test_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 test_df["date_block_num"]=34
test_df["year"]=(( 34//12)+2013)
 test_df["month"]=(34%12)
 test_df["date_block_num"]=test_df["date_block_num"].astype(np.int8)
 test_df["shop_id"]=test_df["shop_id"].astype(np.int8)
 test_df["item_id"]=test_df["item_id"].astype(np.int16)
 test_df["year"]=test_df["year"].astype(np.int16)
 test_df["month"]=test_df["month"].astype(np.int8)
 test_df.head()<define_variables>
df= df_segmentation.copy()
categorical=["year","month"]<data_type_conversions>
if 1:
df['Class'] = df['ImageId_ClassId'].str[-1].astype(np.int32)
 df['Label'] =(df['EncodedPixels']!='' ).astype(np.int32)
 pos1 =(( df['Class']==1)&(df['Label']==1)).sum()
pos2 =(( df['Class']==2)&(df['Label']==1)).sum()
pos3 =(( df['Class']==3)&(df['Label']==1)).sum()
pos4 =(( df['Class']==4)&(df['Label']==1)).sum()
num_image = len(df)//4
num = len(df)
 pos =(df['Label']==1 ).sum()
neg = num-pos
print('')
 print('\t\tnum_image = %5d(1801)'%num_image)
 print('\t\tnum = %5d(7204)'%num)
 print('\t\tneg = %5d(6172)%0.3f'%(neg,neg/num))
print('\t\tpos = %5d(1032)%0.3f'%(pos,pos/num))
print('\t\tpos1 = %5d(128)%0.3f %0.3f'%(pos1,pos1/num_image,pos1/pos))
print('\t\tpos2 = %5d(43)%0.3f %0.3f'%(pos2,pos2/num_image,pos2/pos))
print('\t\tpos3 = %5d(741)%0.3f %0.3f'%(pos3,pos3/num_image,pos3/pos))
print('\t\tpos4 = %5d(120)%0.3f %0.3f'%(pos4,pos4/num_image,pos4/pos))
train=train.join(month_df,on=["date_block_num","shop_id","item_id"],how="left")
 train["item_cnt_day"]=train["item_cnt_day"].fillna(0 ).clip(0,20 ).astype(np.float16 )<concatenate>
df_mask = df_segmentation.copy()
df_label = df_classification.copy()
train=pd.concat([train,test_df],ignore_index=True,sort=False,keys=col)
 train.fillna(0,inplace=True )<merge>
assert(np.all(df_mask['ImageId_ClassId'].values == df_label['ImageId_ClassId'].values))
print(( df_mask.loc[df_label['EncodedPixels']=='','EncodedPixels'] != '' ).sum())
 df_mask.loc[df_label['EncodedPixels']=='','EncodedPixels']=''
def create_lag_features(df,feature):
numeric_features=[]
for i in range(1,7):
lagged=df.copy()
lagged=lagged[["date_block_num","shop_id","item_id",feature]]
lagged.columns=["date_block_num","shop_id","item_id",feature+"_lag_"+str(i)]
numeric_features.append(feature+"_lag_"+str(i))
lagged["date_block_num"]+=i
df=df.join(lagged.set_index(["date_block_num","shop_id","item_id"]),on=["date_block_num","shop_id","item_id"],how="left")
 return df,numeric_features<feature_engineering>
df_mask.to_csv("submission.csv", index=False )
<groupby><EOS> 
if 1:
df_mask['Class'] = df_mask['ImageId_ClassId'].str[-1].astype(np.int32)
 df_mask['Label'] =(df_mask['EncodedPixels']!='' ).astype(np.int32)
 pos1 =(( df_mask['Class']==1)&(df_mask['Label']==1)).sum()
pos2 =(( df_mask['Class']==2)&(df_mask['Label']==1)).sum()
pos3 =(( df_mask['Class']==3)&(df_mask['Label']==1)).sum()
pos4 =(( df_mask['Class']==4)&(df_mask['Label']==1)).sum()
num_image = len(df_mask)//4
num = len(df_mask)
 pos =(df_mask['Label']==1 ).sum()
neg = num-pos
print('')
 print('\t\tnum_image = %5d(1801)'%num_image)
 print('\t\tnum = %5d(7204)'%num)
 print('\t\tneg = %5d(6172)%0.3f'%(neg,neg/num))
print('\t\tpos = %5d(1032)%0.3f'%(pos,pos/num))
print('\t\tpos1 = %5d(128)%0.3f %0.3f'%(pos1,pos1/num_image,pos1/pos))
print('\t\tpos2 = %5d(43)%0.3f %0.3f'%(pos2,pos2/num_image,pos2/pos))
print('\t\tpos3 = %5d(741)%0.3f %0.3f'%(pos3,pos3/num_image,pos3/pos))
print('\t\tpos4 = %5d(120)%0.3f %0.3f'%(pos4,pos4/num_image,pos4/pos))