Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<data_type_conversions>
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/unetmodelscript'
sys.path.append(package_path )
train=train.join(items_df.set_index(["item_id"]),on=["item_id"],how="left")
 train=train.join(sum_cnt_for_item_category[["item_category_id","category","subcategory"]].set_index(["item_category_id"]),on=["item_category_id"],how="left")
 train.fillna(0,inplace=True)
 train["category"]=train["category"].astype(str)
 train["subcategory"]=train["subcategory"].astype(str )<define_variables>
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.torch import ToTensor
import torch.utils.data as data
from model import Unet
categorical=categorical+["category","subcategory"]<merge>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
train=train.join(total_cnt_for_item_categories,on=["date_block_num","item_category_id"],how="left")
 train.fillna(0,inplace=True)
 train["total_cnt_for_item_categories"]=train["total_cnt_for_item_categories"].astype(np.float16 )<drop_column>
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples
%%time
train,n_col=create_lag_features(train,"total_cnt_for_item_categories")
 train=train.fillna(0)
 train=train.drop(["total_cnt_for_item_categories"],axis=1)
 numeric=numeric+n_col<groupby>
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num
mean_price=sales_train_df[["date_block_num","shop_id","item_id","item_price"]].groupby(["date_block_num","shop_id","item_id"] ).mean()
mean_price.columns=["mean_price"]<data_type_conversions>
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"
train=train.join(mean_price,on=["date_block_num","shop_id","item_id"],how="left")
 train.fillna(0,inplace=True)
 train["mean_price"]=train["mean_price"].astype(np.float16 )<drop_column>
best_threshold = 0.5
num_workers = 2
batch_size = 4
print('best_threshold', best_threshold)
 min_size = 3500
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)
%%time
train,n_col=create_lag_features(train,"mean_price")
 train=train.fillna(0)
 train=train.drop(["mean_price"],axis=1)
 numeric=numeric+n_col<groupby>
ckpt_path = ".. /input/unet-starter-kernel-pytorch-lb-0-88/model.pth"
device = torch.device("cuda")
 model = Unet("resnet18", encoder_weights=None, classes=4, activation=None)
 model.to(device)
 model.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"] )
<merge><EOS> 
predictions = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
batch_preds = torch.sigmoid(model(images.to(device)))
 batch_preds = batch_preds.detach().cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, best_threshold, min_size)
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv("submission.csv", index=False )