Kaggle competition: Carvana Image Masking Challenge
test.drop(['Name'],axis=1,inplace=True )<count_missing_values>
%matplotlib inline
K.set_image_dim_ordering('th')
train.isnull().sum()<count_missing_values>
INPUT_PATH='.. /input/'
print(os.getcwd())
 dims=[128,128]
img_rows=dims[0]
img_cols=dims[1]
train=sorted(glob.glob(INPUT_PATH+'train/*.jpg'))
masks=sorted(glob.glob(INPUT_PATH+'train_masks/*.gif'))
test=sorted(glob.glob(INPUT_PATH+'test/*.jpg'))
print('Number of training images: ', len(train), 'Number of corresponding masks: ', len(masks), 'Number of test images: ', len(test))
meta=pd.read_csv(INPUT_PATH+'metadata.csv')
 mask_df=pd.read_csv(INPUT_PATH+'train_masks.csv')
 ids_train=mask_df['img'].map(lambda s: s.split('_')[0] ).unique()
print('Length of ids_train ', len(ids_train))
train.isnull().sum()<count_missing_values>
image = cv2.imread(INPUT_PATH+"train/00087a6bd4dc_01.jpg")
 plt.imshow(image)
 plt.show()
test.isnull().sum()<count_missing_values>
masked_img=cv2.bitwise_and(image,img2mask)
 plt.imshow(masked_img )
test.isnull().sum()<drop_column>
smooth = 1.
 def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection=K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def dice_coef_mp(y_true, y_pred):
y_true_f=y_true.flatten()
y_pred_f=y_pred.flatten()
intersection = np.sum(y_true_f*y_pred_f)
 return(2.* intersection + smooth)/(np.sum(y_pred_f)+ mp.sum(y_pred_f)+ smooth)
 def dice_coef_loss(y_true, y_pred):
return -dice_coef(y_true, y_pred )
train.drop(['Cabin'],axis=1,inplace=True)
 test.drop(['Cabin'],axis=1,inplace=True)
 test.drop(['Ticket'],axis=1,inplace=True)
 train.drop(['Ticket'],axis=1,inplace=True)
 train.drop(['PassengerId'],axis=1,inplace=True)
 test.drop(['PassengerId'],axis=1,inplace=True )<count_missing_values>
def get_layer(inputs, pixel, pool=True):
conv = Conv2D(pixel,(3, 3), padding="same", activation='relu' )(inputs)
 conv = Conv2D(pixel,(3 ,3), padding="same", activation='relu' )(conv)
 if pool:
pool = MaxPooling2D(pool_size=(2, 2))(conv)
 else:
pool = None
return conv, pool
train.isnull().sum()<count_missing_values>
def get_unet() :
inputs = Input(( 3,img_rows, img_cols))
conv1 = Conv2D(32,(3, 3), padding="same", activation='relu' )(inputs)
 conv1 = Conv2D(32,(3, 3), padding="same", activation='relu' )(conv1)
 pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
 conv2 = Conv2D(64,(3, 3), padding="same", activation='relu' )(pool1)
 conv2 = Conv2D(64,(3, 3), padding="same", activation='relu' )(conv2)
 pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
 conv3 = Conv2D(128,(3, 3), padding="same", activation='relu' )(pool2)
 conv3 = Conv2D(128,(3, 3), padding="same", activation='relu' )(conv3)
 pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
 conv4 = Conv2D(256,(3, 3), padding="same", activation='relu' )(pool3)
 conv4 = Conv2D(256,(3, 3), padding="same", activation='relu' )(conv4)
 pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
 conv5 = Conv2D(512,(3, 3), padding="same", activation='relu' )(pool4)
 conv5 = Conv2D(512,(3, 3), padding="same", activation='relu' )(conv5)
 up6 = Concatenate(axis=1 )([UpSampling2D(size=(2, 2))(conv5), conv4])
 conv6 = Conv2D(256,(3, 3), padding="same", activation='relu' )(up6)
 conv6 = Conv2D(256,(3, 3), padding="same", activation='relu' )(conv6)
 up7 = Concatenate(axis=1 )([UpSampling2D(size=(2, 2))(conv6), conv3])
 conv7 = Conv2D(128,(3, 3), padding="same", activation='relu' )(up7)
 conv7 = Conv2D(128,(3, 3), padding="same", activation='relu' )(conv7)
 up8 = Concatenate(axis=1 )([UpSampling2D(size=(2, 2))(conv7), conv2])
 conv8 = Conv2D(64,(3, 3), padding="same", activation='relu' )(up8)
 conv8 = Conv2D(64,(3, 3), padding="same", activation='relu' )(conv8)
 up9 = Concatenate(axis=1 )([UpSampling2D(size=(2, 2))(conv8), conv1])
 conv9 = Conv2D(32,(3, 3), padding="same", activation='relu' )(up9)
 conv9 = Conv2D(32,(3, 3), padding="same", activation='relu' )(conv9)
 conv10 = Conv2D(1,(1, 1), activation='sigmoid' )(conv9)
 model = Model(inputs=inputs, outputs=conv10)
 model.compile(optimizer=Adam(5e-4), loss='binary_crossentropy', metrics=[dice_coef])
 return model
train.isnull().sum()<count_missing_values>
train_images, validation_images = train_test_split(train, train_size=0.8, test_size=0.2)
 print('Split into training set with ', len(train_images), ' images anf validation set with ', len(validation_images), ' images' )
test.isnull().sum()<count_missing_values>
def grey2rgb(img):
new_img = []
for i in range(img.shape[0]):
for j in range(img.shape[1]):
new_img.append(list(img[i][j])*3)
 new_img = np.array(new_img ).reshape(img.shape[0], img.shape[1], 3)
 return new_img
def data_gen_small(data_dir, masks, images, batch_size, dims):

 while True:
if batch_size==1:
ix=np.array([0])
 else:
ix=np.random.choice(np.arange(len(images)) , batch_size)
 imgs = []
labels = []
for i in ix:
if batch_size==1:
original_img = cv2.imread(images)
 else:
original_img = cv2.imread(images[i])
 resized_img = imresize(original_img, dims + [3])
 array_img = resized_img/255
array_img = array_img.swapaxes(0, 2)
 imgs.append(array_img)
 try:
mask_filename = basename(images[i])
 no_extension = os.path.splitext(mask_filename)[0]
correct_mask = INPUT_PATH + 'train_masks/' + no_extension + '_mask.gif'
original_mask = Image.open(correct_mask ).convert('L')
 data = np.asarray(original_mask, dtype="int32")
 resized_mask = imresize(original_mask, dims+[3])
 array_mask = resized_mask / 255
labels.append(array_mask)
 except Exception as e:
labels=None
imgs = np.array(imgs)
 labels = np.array(labels)
 try:
relabel = labels.reshape(-1, dims[0], dims[1], 1)
 relabel = relabel.swapaxes(1, 3)
 except Exception as e:
relabel=labels
yield imgs, relabel
test.isnull().sum()<data_type_conversions>
train_gen = data_gen_small(INPUT_PATH + 'train/', masks, train_images, 2, dims)
 img, msk = next(train_gen)
 print('Size of batch: ', len(img))
print('shape of img ', img.shape, 'number dimensions: ', img[0].ndim)
 print('shape of msk ', msk.shape, 'number dimensions: ', msk[0].ndim)
 newshape = img[0].swapaxes(0,2)
 plt.imshow(newshape)
 plt.show()
resized_img = imresize(img[0], [1280, 1918]+[3])
 print('resized up: ', resized_img.shape)
 newshape = resized_img.swapaxes(0,1)
 print('resized swapaxes: ', newshape.shape)
 print('resized swapaxes shape[-1]: ', newshape.shape[-1])
 plt.imshow(newshape)
 plt.show()
newshape = msk.swapaxes(1,3)
 print(newshape.shape)
 plt.imshow(grey2rgb(newshape[0]), alpha=0.5)
 plt.show()
train['Age'].fillna(( train['Age'].mean()), inplace=True )<correct_missing_values>
validation_gen = data_gen_small(INPUT_PATH + 'train/', masks, validation_images, 4, dims )
test['Age'].fillna(( test['Age'].mean()), inplace=True )<count_missing_values>
model.fit_generator(train_gen, steps_per_epoch=50, epochs=35, validation_data=validation_gen, validation_steps=50 )
test['Fare'].fillna(( test['Fare'].mean()), inplace=True )<correct_missing_values>
img, msk = next(validation_gen)
 print(img.shape)
 predicted_mask = model.predict(img)
 predicted_mask.shape
train.dropna()<count_missing_values>
validation_test = data_gen_small(INPUT_PATH + 'test/', masks, test, 4, dims)
 img_tst, msk_tst = next(validation_test)
 print(img_tst.shape)
 predicted_mask_tst = model.predict(img_tst)
 predicted_mask_tst.shape
test.isnull().sum()<count_missing_values>
newshape_tst = predicted_mask_tst.swapaxes(1,3)
 print('newshape shape ', newshape_tst.shape)
 grey_tst = grey2rgb(newshape_tst[3])
 print('grey shape ', grey_tst.shape)
 plt.imshow(grey_tst, alpha = 0.5)
 plt.show()
test.isnull().sum()<categorify>
newshape_tst = img_tst[3].swapaxes(0,2)
 print(newshape_tst.shape)
 plt.imshow(newshape_tst)
 plt.show()
Pclass=pd.get_dummies(train['Pclass'],drop_first=True)
 Pclass1=pd.get_dummies(test['Pclass'],drop_first=True)
 Sex=pd.get_dummies(train['Sex'],drop_first=True)
 Sex1=pd.get_dummies(test['Sex'],drop_first=True)
 Embarked=pd.get_dummies(train['Embarked'],drop_first=True)
 Embarked1=pd.get_dummies(test['Embarked'],drop_first=True )<concatenate>
newshape_tst = predicted_mask_tst.swapaxes(1,3)
 print('newshape shape ', newshape_tst.shape)
 grey_tst = grey2rgb(newshape_tst[3])
 print('grey shape ', grey_tst.shape)
 plt.imshow(grey_tst, alpha = 0.5)
 plt.show()
train=pd.concat([train,Pclass,Sex,Embarked],axis=1)
 test=pd.concat([test,Pclass1,Sex1,Embarked1],axis=1 )<drop_column>
from tqdm import tqdm_notebook as tqdm
train.drop(['Sex','Embarked','Pclass'],axis=1,inplace=True)
 test.drop(['Sex','Embarked','Pclass'],axis=1,inplace=True )<import_modules>
df=pd.DataFrame(columns=['img','rle_mask'] )
from sklearn.model_selection import train_test_split<import_modules>
<prepare_x_and_y><EOS> 

 def rle_encoding(x):

 dots = np.where(x.T.flatten() ==1)[0]
run_lengths = []
prev = -2
for b in dots:
if(b>prev+1): run_lengths.extend(( b+1, 0))
run_lengths[-1] += 1
prev = b
return run_lengths