Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<set_options>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from pathlib import Path
import cv2
import os
del cum
gc.collect()<data_type_conversions>
train_df = pd.read_csv('.. /input/severstal-steel-defect-detection/train.csv')
 train_df['ImageId'] = train_df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 train_df['ClassId'] = train_df['ImageId_ClassId'].apply(lambda x: x.split('_')[1] ).astype('int8')
 train_df['hasMask'] = ~ train_df['EncodedPixels'].isna()
train_df.drop(['ImageId_ClassId'], axis=1, inplace=True)
 train_df.fillna(0, inplace=True)
 train_df.head()
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()<data_type_conversions>
DEFECT_TYPES = [1, 2, 3, 4]
for x in DEFECT_TYPES:
train_df.loc[train_df.ClassId==x, 'pixel_count'] = train_df.loc[train_df.ClassId==x, 'EncodedPixels'].apply(pixel_count )
explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
<groupby>
THRESHOLDS = []
for x in DEFECT_TYPES:
pixels = train_df.loc[train_df['ClassId']==x, 'pixel_count']
threshold = np.nanquantile(pixels, 0.1)
 THRESHOLDS.append(int(threshold))
print(f'Quintile threshold for defect type {x} = {int(threshold)}' )
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
<data_type_conversions>
THRESHOLDS[3] = 3500
user_agg=user_agg.astype('int16')
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )<categorify>
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/unetmodelscript'
sys.path.append(package_path )
attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'])
 attempt_no_agg=attempt_no_agg.astype('int8')
<data_type_conversions>
import pdb
import cv2
import torch
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.pytorch import ToTensor
import torch.utils.data as data
from model import Unet
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_sum'] = train_df['content_id'].map(content_agg['sum'] ).astype('int32')
 train_df['content_correctness'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 train_df.content_correctness=train_df.content_correctness.astype('float16')
 train_df['task_container_sum'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )<load_from_csv>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'} 
)
 questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
 tag = questions_df["tags"].str.split(" ", n = 10, expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
tag.fillna(0, inplace=True)
 tag = tag.astype('int16')
 questions_df = pd.concat([questions_df,tag],axis=1 ).drop(['tags'],axis=1 )<rename_columns>
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"
best_threshold = 0.5
num_workers = 2
batch_size = 4
min_size = 0
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True 
)
 ckpt_path = ".. /input/unetstartermodelfile/model.pth"
device = torch.device("cuda")
 model = Unet("resnet18", encoder_weights=None, classes=4, activation=None)
 model.to(device)
 model.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"])
 predictions = []
for i, batch in enumerate(testset):
fnames, images = batch
batch_preds = torch.sigmoid(model(images.to(device)))
 batch_preds = batch_preds.detach().cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, best_threshold, min_size)
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 sub_init = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'] )
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )<data_type_conversions>
INIT_THRESHOLD = np.full(4, 3500 )
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16' )<categorify>
sub_1 = sub_init.copy()
sub_2 = sub_init.copy()
for i in range(len(THRESHOLDS)) :
threshold_init = INIT_THRESHOLD[i]
threshold_new = THRESHOLDS[i]
defect = DEFECT_TYPES[i]
sub_1.loc[(sub_1['ClassId']==defect)&(sub_1.pixel_count < threshold_init), 'EncodedPixels'] = ''
sub_2.loc[(sub_2['ClassId']==defect)&(sub_2.pixel_count < threshold_new), 'EncodedPixels'] = ''
sub_1.drop(['ClassId', 'pixel_count'], axis=1, inplace=True)
 sub_2.drop(['ClassId', 'pixel_count'], axis=1, inplace=True )
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )<categorify>
initial_sub_len = len(sub_1.loc[sub_1['EncodedPixels'] != '', :])
 modified_sub_len = len(sub_2.loc[sub_2['EncodedPixels'] != '', :])
 print(f'Positive instance case count with 3500 pixel threshold: {initial_sub_len}')
 print(f'Positive instance case count with new pixel thresholds: {modified_sub_len}' )
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness=questions_df.bundle_correctness.astype('float16' )<categorify>
sub_2.to_csv('submission.csv', index=False )
<drop_column><EOS> 
def pixel_postprocess(sub, thresholds):

 def pixel_count(rle):
if np.logical_or(rle==0, rle==''):
return(np.nan)
 array = np.asarray([int(x)for x in rle.split() ])
 lengths = array[1::2]
return sum(lengths)
 DEFECT_TYPES = [1, 2, 3, 4]
sub['ClassId'] = sub['ImageId_ClassId'].apply(lambda x: x.split('_')[1] ).astype('int8')
 for x in DEFECT_TYPES:
sub.loc[sub.ClassId==x, 'pixel_count'] = sub.loc[sub.ClassId==x, 'EncodedPixels'].apply(pixel_count)
 for i in range(len(thresholds)) :
threshold = thresholds[i]
defect = DEFECT_TYPES[i]
sub.loc[(sub['ClassId']==defect)&(sub.pixel_count < threshold), 'EncodedPixels'] = ''
sub.drop(['ClassId', 'pixel_count'], axis=1, inplace=True)
 return(sub )