Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<filter>
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/unetmodelscript'
sys.path.append(package_path )
train[train['item_price']>=100000]<filter>
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.torch import ToTensor
import torch.utils.data as data
from model import Unet
items[items['item_id']==6066]<filter>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
train[train['item_id']==6066]<filter>
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples
train = train[train['item_cnt_day']<1000]
train = train[train['item_price']<100000]
train.reset_index(drop=True, inplace=True )<import_modules>
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num
import holidays<feature_engineering>
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"
ru_holidays = holidays.Russia()<feature_engineering>
best_threshold = 0.5
num_workers = 2
batch_size = 4
print('best_threshold', best_threshold)
 min_size = 3500
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)
train['new_date'] = train['date'].apply(lambda x: x.split('.')[1]+'/'+x.split('.')[0]+'/'+x.split('.')[2] )<feature_engineering>
ckpt_path = ".. /input/unetstartermodelfile/model.pth"
device = torch.device("cuda")
 model = Unet("resnet18", encoder_weights=None, classes=4, activation=None)
 model.to(device)
 model.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"] )
<count_unique_values><EOS> 
predictions = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
batch_preds = torch.sigmoid(model(images.to(device)))
 batch_preds = batch_preds.detach().cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, best_threshold, min_size)
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv("submission.csv", index=False )