Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<categorify>
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/unet-model'
sys.path.append(package_path )
df_sales = add_feature(df_sales,['month'],'month_target')
 df_sales = shift_feature(df_sales,[1,2,3,12],'month_target')
 df_sales = add_feature(df_sales,['month','item_id'],'month_item_target')
 df_sales = shift_feature(df_sales,[1,2,3],'month_item_target')
 df_sales = add_feature(df_sales,['month','shop_id'],'month_shop_target')
 df_sales = shift_feature(df_sales,[1,2,3],'month_shop_target')
 df_sales = add_feature(df_sales,['month','item_category_id'],'month_cat_target')
 df_sales = shift_feature(df_sales,[1,2,3,12],'month_cat_target')
 df_sales.fillna(0,inplace = True)
 df_sales = optimize_memory(df_sales)
 gc.collect()<categorify>
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.torch import ToTensor
import torch.utils.data as data
from model import Unet
df_sales['date_item_day'] = df_sales['item_cnt_month'] / df_sales['days']
df_sales = shift_feature(df_sales,[1,2,3,12],'date_item_day')
 df_sales = add_feature(df_sales,['date_block_num','item_id','city_code'],'date_item_city_target')
 df_sales = shift_feature(df_sales,[1,2,3,12],'date_item_city_target')
 gb = df_sales.groupby(['shop_id','item_id'],as_index = False ).agg({ 'avg_item_price':'mean'})
 gb.rename(columns ={'avg_item_price':'avg_pair_price'},inplace = True)
 df_sales = pd.merge(df_sales,gb,how='left',on=['shop_id','item_id'] ).fillna(0)
 df_sales['delta_item_price'] =(df_sales['avg_pair_price'] - df_sales['avg_item_price'])/df_sales['avg_item_price']
df_sales = shift_feature(df_sales,[1,2,3,6],'avg_item_price')
 df_sales = shift_feature(df_sales,[1,2,3,6],'avg_pair_price')
 df_sales = shift_feature(df_sales,[1,2,3,6],'delta_item_price')
 df_sales.fillna(0,inplace = True)
 df_sales = optimize_memory(df_sales)
 del gb
gc.collect()<set_options>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
start_mem = df_sales.memory_usage().sum() / 1024**2
df_sales = df_sales[df_sales['date_block_num'] > 11]
end_mem = df_sales.memory_usage().sum() / 1024**2
print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
gc.collect()<categorify>
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples
df_sales.to_pickle('df_sales.pkl' )<load_pretrained>
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num
<prepare_x_and_y>
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"
X_train = df_sales[df_sales['date_block_num'] < 33].drop('item_cnt_month',axis = 1)
 y_train = df_sales[df_sales['date_block_num'] < 33].item_cnt_month
X_val = df_sales[df_sales['date_block_num'] == 33].drop('item_cnt_month',axis = 1)
 y_val = df_sales[df_sales['date_block_num'] == 33].item_cnt_month
X_test = df_sales[df_sales['date_block_num'] == 34].drop('item_cnt_month',axis = 1)
 gc.collect()<train_model>
best_threshold = 0.5
num_workers = 2
batch_size = 4
print('best_threshold', best_threshold)
 min_size = 3500
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)
ts = time.time()
xgb = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
subsample=0.8,
colsample_bytree=0.8,
eta = 0.3,
seed=42)
 xgb.fit(
 X_train,
y_train,
eval_metric="rmse",
eval_set=[(X_train, y_train),(X_val, y_val)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts<load_pretrained>
torch.cuda.is_available()
pickle.dump(xgb, open("xgb.dat", "wb")) 
<compute_train_metric>
ckpt_path = ".. /input/unetstartermodelfile/model.pth"
device = torch.device("cuda")
 model = Unet("resnet18", encoder_weights=None, classes=4, activation=None)
 model.to(device)
 model.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"] )
<save_to_csv><EOS> 
predictions = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
batch_preds = torch.sigmoid(model(images.to(device)))
 batch_preds = batch_preds.detach().cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, best_threshold, min_size)
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv("submission.csv", index=False )