Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<prepare_x_and_y>
package_path = '.. /input/underscripts/'
sys.path.append(package_path )
X_train = df_sales[df_sales['date_block_num'] < 34].drop('item_cnt_month',axis = 1)
 y_train = df_sales[df_sales['date_block_num'] < 34].item_cnt_month<train_on_grid>
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(HorizontalFlip, ShiftScaleRotate, Normalize, Resize, Compose, GaussNoise)
 from albumentations.torch import ToTensor
import torch.utils.data as data
from model import Unet
param_search = {"max_iter": [1500]}
elanet = ElasticNet()
tscv = TimeSeriesSplit(n_splits = 10 ).split(X_train)
 gsearch = GridSearchCV(elanet, cv=tscv,param_grid=param_search,scoring = 'neg_root_mean_squared_error',n_jobs = 2,verbose = 3)
 gsearch.fit(X_train,y_train)
 gc.collect()<predict_on_test>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
dates_train = X_train['date_block_num']
dates_train_level2 = dates_train[dates_train.isin([28,29,30,31,32,33])]
pred_lr = gsearch.predict(X_test ).clip(0,20)
 pred_xgb = xgb.predict(X_test ).clip(0,20)
 X_test_level2 = np.c_[pred_lr,pred_xgb]
y_train_level2 = y_train[dates_train.isin([28,29,30,31,32,33])]
X_train_level2 = np.zeros([y_train_level2.shape[0], 2] )<train_on_grid>
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std,TTA=False):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 if TTA==True:
self.transform = Compose(
 [
HorizontalFlip() ,
Normalize(mean=mean,std=std,p=1),
ToTensor() ,
] 
)
 else:
self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples
del gsearch,pred_lr,pred_xgb
gc.collect()
ts = time.time()
lr = ElasticNet(max_iter = 1500)
 for cur_block_num in [28,29,30,31,32,33]:
print(cur_block_num)
 X_train_part = X_train[dates_train < cur_block_num]
y_train_part = y_train[dates_train < cur_block_num]
X_test_part = X_train[dates_train == cur_block_num]
y_test_part = y_train[dates_train == cur_block_num]
print('Linear Regression Model')
 lr.fit(X_train_part,y_train_part)
 pred_lr = lr.predict(X_test_part)
 print('Fitting XGBoost Model')
 xgb.fit(X_train_part,y_train_part,eval_metric="rmse", eval_set=[(X_train_part, y_train_part),(X_test_part, y_test_part)],
verbose=True, early_stopping_rounds = 10)
 pred_xgb = xgb.predict(X_test_part)
 X_train_level2[dates_train_level2 == cur_block_num,0] = pred_lr
X_train_level2[dates_train_level2 == cur_block_num,1] = pred_xgb
time.time() - ts<load_pretrained>
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num
pickle.dump(X_train_level2, open("X_train_level2.dat", "wb"))<compute_test_metric>
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"
alphas_to_try = np.linspace(0, 1, 5000)
 rmse_scores = np.array([mean_squared_error(y_train_level2,np.dot(X_train_level2,[alpha,1-alpha]),squared = False)for alpha in alphas_to_try])
 best_alpha = alphas_to_try[rmse_scores.argmin() ]
rmse_train_simple_mix = rmse_scores.min()
print('Best alpha: %f; Corresponding rmse score on train: %f' %(best_alpha, rmse_train_simple_mix))<save_to_csv>
best_threshold = 0.5
num_workers = 2
batch_size = 1
print('best_threshold', best_threshold)
 min_size = 3500
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True 
)
 testset_TTA = DataLoader(
 TestDataset(test_data_folder,df,mean,std,TTA=True),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)
y_test = np.dot(X_test_level2,[best_alpha,1-best_alpha] ).clip(0,20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": y_test
})
 submission.to_csv('submission_2.csv', index=False )<import_modules>
predictions = []
model_name = "resnet18"
models = []
for i in range(4):
ckpt_path = ".. /input/ures184fold/u-resnet18_fold_{}.pth".format(i)
 device = torch.device("cuda")
 model = Unet(encoder_type="resnet",encoder_name="resnet18",classes=4,activation=None)
 model.to(device)
 model.eval()
state = torch.load(ckpt_path,map_location=lambda storage,loc:storage)
 model.load_state_dict(state["state_dict"])
 models.append(model )
<import_modules><EOS> 
predictions = []
for i,(batch,batch_TTA)in enumerate(tqdm(zip(testset,testset_TTA))):
fnames, images = batch
fnames, images_TTA = batch
batch_preds = 0
for model in models:
pred = torch.sigmoid(model(images.to(device)) ).detach().cpu().numpy()
pred_TTA = torch.sigmoid(model(images_TTA.to(device)) ).detach().cpu().numpy()
batch_preds +=(pred+pred_TTA[:,:,:,::-1])/2
batch_preds+=pred
batch_preds/= len(models)
 for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, best_threshold, min_size)
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv("submission.csv", index=False )