Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<drop_column>
import os
import json
import gc
import cv2
import keras
from keras import backend as K
from keras import layers
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Model, load_model
from keras.layers import Input
from keras.layers.convolutional import Conv2D, Conv2DTranspose
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import concatenate
from keras.optimizers import Adam
from keras.callbacks import Callback, ModelCheckpoint
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from tqdm import tqdm
from sklearn.model_selection import train_test_split
%%time
train,n_col=create_lag_features(train,"mean_price_for_items")
 train=train.fillna(0)
 train=train.drop(["mean_price_for_items"],axis=1)
 numeric=numeric+n_col<groupby>
INPUT_PATH = ".. /input"
BOOTSTRAP = False
DF_TRAIN_PATH = os.path.join(INPUT_PATH, "severstal-steel-defect-detection/train.csv")
 DF_TEST_PATH = os.path.join(INPUT_PATH, "severstal-steel-defect-detection/sample_submission.csv")
 TRAIN_IMAGE_PATH = os.path.join(INPUT_PATH, "severstal-steel-defect-detection/train_images")
 TEST_IMAGE_PATH = os.path.join(INPUT_PATH, "severstal-steel-defect-detection/test_images")
 DATA_PATH = os.path.join(INPUT_PATH, "severstal-steel-defect-detection-data-files")
 GENERATE_WEIGHTS = False
USE_CALLBACK = True
EPOCHS = 16
if BOOTSTRAP:
EPOCHS = 1
CHANNELS = 3
MODEL_NAME = "model_single_segmentation.h5"
data_dir_path = ".. /input/severstal-steel-defect-detection-data-files"
if os.path.exists(data_dir_path):
for fname in os.listdir(data_dir_path):
filepath = os.path.join(data_dir_path, fname)
 print(filepath)
 if os.path.isfile(filepath):
if GENERATE_WEIGHTS == True:
if fname.find("h5")> 0:
continue
if fname.find("json")> 0:
continue
destfilepath = os.path.join("./", fname)
 print("copy file ", filepath, " to ", destfilepath)
 shutil.copy(filepath, destfilepath)
mean_price_for_item_categories=sales_train_df[["date_block_num","item_category_id","item_price"]].groupby(["date_block_num","item_category_id"] ).mean()
mean_price_for_item_categories.columns=["mean_price_for_item_categories"]<merge>
train_df = pd.read_csv(DF_TRAIN_PATH)

train_df['ImageId'] = train_df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 train_df['ClassId'] = train_df['ImageId_ClassId'].apply(lambda x: x.split('_')[1])
 train_df['hasMask'] = ~ train_df['EncodedPixels'].isna()
train_df.head()
train=train.join(mean_price_for_item_categories,on=["date_block_num","item_category_id"],how="left")
 train.fillna(0,inplace=True)
 train["mean_price_for_item_categories"]=train["mean_price_for_item_categories"].astype(np.float16 )<drop_column>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(rle, input_shape):

 height, width = input_shape[:2]
mask= np.zeros(width * height ).astype(np.uint8)

array = np.asarray([int(x)for x in rle.split() ])
 starts = array[0::2]
lengths = array[1::2]
for index, start in enumerate(starts):
begin = int(start - 1)
 end = int(begin + lengths[index])
 mask[begin : end] = 1

 return mask.reshape(width, height ).T
def mask2contour(mask, width=3):
w = mask.shape[1]
h = mask.shape[0]
mask2 = np.concatenate([mask[:,width:],np.zeros(( h,width)) ],axis=1)
 mask2 = np.logical_xor(mask,mask2)
 mask3 = np.concatenate([mask[width:,:],np.zeros(( width,w)) ],axis=0)
 mask3 = np.logical_xor(mask,mask3)
 return np.logical_or(mask2,mask3)
 def mask2pad(mask, pad=2):
w = mask.shape[1]
h = mask.shape[0]
for k in range(1,pad,2):
temp = np.concatenate([mask[k:,:],np.zeros(( k,w)) ],axis=0)
 mask = np.logical_or(mask,temp)
 for k in range(1,pad,2):
temp = np.concatenate([np.zeros(( k,w)) ,mask[:-k,:]],axis=0)
 mask = np.logical_or(mask,temp)
 for k in range(1,pad,2):
temp = np.concatenate([mask[:,k:],np.zeros(( h,k)) ],axis=1)
 mask = np.logical_or(mask,temp)
 for k in range(1,pad,2):
temp = np.concatenate([np.zeros(( h,k)) ,mask[:,:-k]],axis=1)
 mask = np.logical_or(mask,temp)
 return mask
def build_masks(rles, input_shape):
depth = 5
masks = np.zeros(( *input_shape, depth))
assert len(rles)== 4
for i, rle in enumerate(rles):
if type(rle)is str:
masks[:, :, i] = rle2mask(rle, input_shape)
 m2 = np.sum(masks, axis=-1 ).astype('bool')
 m2 = np.logical_not(m2 ).astype('int')
 m2 = m2.reshape(input_shape[0], input_shape[1], 1)
 masks[:, :, 4] = m2.reshape(input_shape)
 return masks
def build_rles(masks):
width, height, depth = masks.shape
rles = [mask2rle(masks[:, :, i])
 for i in range(depth)]
return rles
%%time
train,n_col=create_lag_features(train,"mean_price_for_item_categories")
 train=train.fillna(0)
 train=train.drop(["mean_price_for_item_categories"],axis=1)
 numeric=numeric+n_col<feature_engineering>
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=TRAIN_IMAGE_PATH,
batch_size=32, dim=(256, 1600), n_channels=CHANNELS,
n_classes=4, random_state=2019, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
ret = 0
if(len(self.list_IDs)% self.batch_size)> 0:
ret = int(len(self.list_IDs)/ self.batch_size)+ 1
else:
ret = int(np.floor(len(self.list_IDs)/ self.batch_size))
return ret
def __getitem__(self, index):
'Generate one batch of data'

 start = index * self.batch_size
end = min(len(self.list_IDs),(index + 1)* self.batch_size)
 indexes = self.indexes[start : end]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.seed(self.random_state)
 np.random.shuffle(self.indexes)
 def __generate_X(self, list_IDs_batch):
'Generates data containing batch_size samples'
X = np.empty(( len(list_IDs_batch), *self.dim, self.n_channels))
for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].loc[ID]
img_path = f"{self.base_path}/{im_name}"
img = self.__load_rgb(img_path)
 X[i,] = img
return X
def __generate_y(self, list_IDs_batch):
y = np.empty(( len(list_IDs_batch), *self.dim, self.n_classes), dtype=int)
 for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].loc[ID]
image_df = self.target_df[self.target_df['ImageId'] == im_name]
rles = image_df['EncodedPixels'].values
masks = build_masks(rles, input_shape=self.dim)
 y[i, ] = masks
return y
def __load_grayscale(self, img_path):

 img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
 img = img.astype(np.float32)/ 255.
 img = np.expand_dims(img, axis=-1)
 return img
def __load_rgb(self, img_path):
img = cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = img.astype(np.float32)/ 255.
 return img
sales_train_df["revenue"]=sales_train_df["item_price"]*sales_train_df["item_cnt_day"]
total_revenue=sales_train_df[["date_block_num","shop_id","revenue"]].groupby(["date_block_num","shop_id"] ).sum()
total_revenue.columns=["total_shop_revenue"]<data_type_conversions>
def test_DataGenerator(index = 0):
BATCH_SIZE = 16
rows = BATCH_SIZE
columns = 1
train_idx = non_missing_train_idx.index
fig = plt.figure(figsize=(16, 4 * BATCH_SIZE))
gen = DataGenerator(
 train_idx,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=5)
 X, y = gen.__getitem__(index)
 print("X.shape :", X.shape)
 print("y.shape :", y.shape)
 ax_idx = 1
for i in range(BATCH_SIZE):
if ax_idx > rows * columns * 2:
break
img = X[i]
img = img * 255
img = img.astype(int)
 fig.add_subplot(rows * 2, columns, ax_idx ).set_title(str(i))
for mask_index in range(4):
mask = y[i, :, :, mask_index]
mask = mask2pad(mask,pad=3)
 mask = mask2contour(mask,width=4)
 y[i, :, :, mask_index] = mask
k = y[i, :, :, mask_index]
img[k == 1, mask_index % 3] = 255
plt.imshow(img)
 ax_idx += 1
fig.add_subplot(rows * 2, columns, ax_idx ).set_title(str(i))
img = X[i]
img = img * 255
img = img.astype(int)
 no_defect_mask = y[i, :, :, 4]
img[no_defect_mask > 0, 0] = 255
plt.imshow(img)
 ax_idx += 1
plt.show()
test_DataGenerator(TEST_INDEX )
train=train.join(total_revenue,on=["date_block_num","shop_id"],how="left")
 train["total_shop_revenue"]=train["total_shop_revenue"].astype(np.float32)
 train.fillna(0,inplace=True )<drop_column>
BATCH_SIZE = 8
train_idx, val_idx = train_test_split(
 mask_count_df.index,
random_state=2019,
test_size=0.15 
)
 train_generator = DataGenerator(
 train_idx,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=5)
 val_generator = DataGenerator(
 val_idx,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=5 )
%%time
train,n_col=create_lag_features(train,"total_shop_revenue")
 train=train.fillna(0.0)
 train=train.drop(["total_shop_revenue"],axis=1)
 numeric=numeric+n_col<groupby>
def dice_coef(y_true, y_pred, smooth=1):
print(y_true.shape, y_pred.shape)
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth )
groups = train.groupby(train.date_block_num ).groups
sorted_groups = [value for(key, value)in sorted(groups.items())]
cv=[(np.concatenate(sorted_groups[:8]),np.concatenate(sorted_groups[8:])) , 
(np.concatenate(sorted_groups[:16]),np.concatenate(sorted_groups[16:])) , 
(np.concatenate(sorted_groups[:24]),np.concatenate(sorted_groups[24:])) ]<prepare_x_and_y>
y_train=train["item_cnt_day"]<data_type_conversions>
X_train_categorical=train[categorical]
X_test_categorical=test_df[categorical]
X_train_categorical["subcategory"]=X_train_categorical["subcategory"].astype(str)
 X_test_categorical["subcategory"]=X_test_categorical["subcategory"].astype(str)
 X_test_categorical["year"]=X_test_categorical["year"].astype(np.int16)
 X_test_categorical["month"]=X_test_categorical["month"].astype(np.int8)
 X_test_categorical.loc[X_test_categorical.category=="PC ","category"]="Игры PC "
X_test_categorical.loc[X_test_categorical.subcategory==" Гарнитуры/Наушники","subcategory"]=" Аксессуары для игр"<categorify>
def get_pretrained_model() :

 model_json_file_name = MODEL_NAME.split('.')[0] + ".json"
json_file = open(model_json_file_name, "r")
 loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
 loaded_model.load_weights(MODEL_NAME)
 loaded_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc'])
 print("Loaded Model Input => Output : ", loaded_model.input_shape, " ======> ", loaded_model.output_shape)
 return loaded_model
for feature in categorical:
le=LabelEncoder()
print(feature)
 X_train_categorical[feature]=le.fit_transform(X_train_categorical[feature])
 X_test_categorical[feature]=le.transform(X_test_categorical[feature] )<prepare_x_and_y>
model = get_pretrained_model()
check_df = mask_count_df.sample(10)
 gen = DataGenerator(
 check_df.index,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=5,
shuffle=False)
 predict = model.predict_generator(gen )
X_train_numeric=train[numeric]
X_test_numeric=test_df[numeric]<concatenate>
def get_one_hot(targets, nb_classes):
res = np.eye(nb_classes)[np.array(targets ).reshape(-1)]
return res.reshape(list(targets.shape)+[nb_classes])
 def pred_to_onehot(pred):
retval = np.argmax(pred, axis=-1)
 retval = get_one_hot(retval.reshape(-1), 5)
 retval = retval.reshape(pred.shape)
 return retval
label_cat_not_num_train=pd.concat([X_train_categorical,X_train_numeric],axis=1)
 label_cat_not_num_train.head().T<concatenate>
def show_test_prediction_head(n=60):
def clear_memory() :
clear_session()
for i in range(20):
gc.collect()
clear_memory()
TEST_BATCH_SIZE = n
test_df = get_test_df()
test_images_df = get_test_imgs_df()
batch_idx = list(range(0, min(test_images_df.shape[0], TEST_BATCH_SIZE)))
 print("running: ", 0, " - ", min(test_images_df.shape[0], TEST_BATCH_SIZE))
model = get_pretrained_model()
test_generator = DataGenerator(
 batch_idx,
df=test_images_df,
base_path = TEST_IMAGE_PATH,
target_df=test_df,
mode = 'predict',
batch_size=TEST_BATCH_SIZE,
shuffle=False,
n_classes=5)
 src_generator = DataGenerator(
 batch_idx,
df=test_images_df,
base_path = TEST_IMAGE_PATH,
target_df=test_df,
batch_size=TEST_BATCH_SIZE,
shuffle=False,
n_classes=5)
 X, _ = src_generator.__getitem__(0)
 predict = model.predict_generator(test_generator)
 pred = pred_to_onehot(predict)
 columns = 1
rows = TEST_BATCH_SIZE
fig = plt.figure(figsize=(12, 3 * rows))
ax_idx = 1
for i in range(rows):
if ax_idx > rows * columns:
break
fig.add_subplot(rows, columns, ax_idx ).set_title(str(i))
img = X[i]
img = img * 255
img = img.astype(int)
 for mask_index in range(4):
k = pred[i, :, :, mask_index]
img[k == 1, mask_index % 3] = 255
plt.imshow(img)
 ax_idx += 1
plt.show()
label_cat_not_num_test=pd.concat([X_test_categorical,X_test_numeric],axis=1)
 label_cat_not_num_test.head().T<data_type_conversions>
def clear_memory() :
clear_session()
for i in range(20):
gc.collect()
clear_memory()
TEST_BATCH_SIZE = 100
df_submit = []
MIN_MASK_PIXEL_THRESHOLD = 3500
test_df = get_test_df()
test_images_df = get_test_imgs_df()
for batch_start in range(0, test_images_df.shape[0], TEST_BATCH_SIZE):
batch_idx = list(range(batch_start, min(test_images_df.shape[0], batch_start + TEST_BATCH_SIZE)))
 print("running: ", batch_start, " - ", min(test_images_df.shape[0], batch_start + TEST_BATCH_SIZE))
model = get_pretrained_model()
test_generator = DataGenerator(
 batch_idx,
df=test_images_df,
base_path = TEST_IMAGE_PATH,
target_df=test_df,
mode = 'predict',
batch_size=BATCH_SIZE,
shuffle=False,
n_classes=5)
 predict = model.predict_generator(test_generator)
 pred = pred_to_onehot(predict)
 for index, bindex, in enumerate(batch_idx):
fname = test_images_df['ImageId'].loc[bindex]
image_df = test_df[test_df['ImageId'] == fname]
pred_masks = pred[index, ]
for mask_index in range(4):
pixelcnt = np.count_nonzero(pred_masks[:,:,mask_index])
 if pixelcnt < MIN_MASK_PIXEL_THRESHOLD:
pred_masks[:,:,mask_index] = 0
pred_masks = pred_masks[:, :, :-1]
pred_rles = build_rles(pred_masks)
 image_df['EncodedPixels'] = pred_rles
df_submit.append(image_df)
 clear_memory()
df_submit = pd.concat(df_submit)
 print(df_submit.shape[0])
 df_submit.head()
<compute_test_metric><EOS> 
df_submit[['ImageId_ClassId', 'EncodedPixels']].to_csv('submission.csv', index=False )