Kaggle competition: 2018 Data Science Bowl
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: data-science-bowl-2018<predict_on_test>
def make_df(train_path, test_path, img_size):
train_ids = next(os.walk(train_path)) [1]
test_ids = next(os.walk(test_path)) [1]
X_train = np.zeros(( len(train_ids), img_size, img_size, 3), dtype=np.uint8)
 Y_train = np.zeros(( len(train_ids), img_size, img_size, 1), dtype=np.bool)
 for i, id_ in enumerate(train_ids):
path = train_path + id_
img = cv2.imread(path + '/images/' + id_ + '.png')
 img = cv2.resize(img,(img_size, img_size))
X_train[i] = img
mask = np.zeros(( img_size, img_size, 1), dtype=np.bool)
 for mask_file in next(os.walk(path + '/masks/')) [2]:
mask_ = cv2.imread(path + '/masks/' + mask_file, 0)
 mask_ = cv2.resize(mask_,(img_size, img_size))
mask_ = mask_[:, :, np.newaxis]
mask = np.maximum(mask, mask_)
 Y_train[i] = mask
X_test = np.zeros(( len(test_ids), img_size, img_size, 3), dtype=np.uint8)
 sizes_test = []
for i, id_ in enumerate(test_ids):
path = test_path + id_
img = cv2.imread(path + '/images/' + id_ + '.png')
 sizes_test.append([img.shape[0], img.shape[1]])
 img = cv2.resize(img,(img_size, img_size))
X_test[i] = img
return X_train, Y_train, X_test, sizes_test
opt_th = 0.505
tmp = np.zeros(len(features))
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt[0, :] = fast_fillna(x_tt[0, :], tmp)
 tmp = x_tt[0, :]
d_tt = xgb.DMatrix(x_tt)
 pred = 0.
 for clf in models:
pred += clf.predict(d_tt)/ len(models)
 pred_df.action = np.where(pred >= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<import_modules>
def Unet(img_size):
inputs = Input(( img_size, img_size, 3))
s = Lambda(lambda x: x / 255 )(inputs)
 c1 = Conv2D(16,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(s)
 c1 = Dropout(0.1 )(c1)
 c1 = Conv2D(16,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c1)
 p1 = MaxPooling2D(( 2, 2))(c1)
 print('c1')
 c2 = Conv2D(32,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(p1)
 c2 = Dropout(0.1 )(c2)
 c2 = Conv2D(32,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c2)
 p2 = MaxPooling2D(( 2, 2))(c2)
 print('c2')
 c3 = Conv2D(64,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(p2)
 c3 = Dropout(0.2 )(c3)
 c3 = Conv2D(64,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c3)
 p3 = MaxPooling2D(( 2, 2))(c3)
 print('c3')
 c4 = Conv2D(128,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(p3)
 c4 = Dropout(0.2 )(c4)
 c4 = Conv2D(128,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c4)
 p4 = MaxPooling2D(pool_size=(2, 2))(c4)
 print('c4')
 c5 = Conv2D(256,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(p4)
 c5 = Dropout(0.3 )(c5)
 c5 = Conv2D(256,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c5)
 print('c5')
 u6 = Conv2DTranspose(128,(2, 2), strides=(2, 2), padding='same' )(c5)
 u6 = concatenate([u6, c4])
 c6 = Conv2D(128,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(u6)
 c6 = Dropout(0.2 )(c6)
 c6 = Conv2D(128,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c6)
 print('c6')
 u7 = Conv2DTranspose(64,(2, 2), strides=(2, 2), padding='same' )(c6)
 u7 = concatenate([u7, c3])
 c7 = Conv2D(64,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(u7)
 c7 = Dropout(0.2 )(c7)
 c7 = Conv2D(64,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c7)
 print('c7')
 u8 = Conv2DTranspose(32,(2, 2), strides=(2, 2), padding='same' )(c7)
 u8 = concatenate([u8, c2])
 c8 = Conv2D(32,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(u8)
 c8 = Dropout(0.1 )(c8)
 c8 = Conv2D(32,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c8)
 print('c8')
 u9 = Conv2DTranspose(16,(2, 2), strides=(2, 2), padding='same' )(c8)
 u9 = concatenate([u9, c1], axis=3)
 c9 = Conv2D(16,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(u9)
 c9 = Dropout(0.1 )(c9)
 c9 = Conv2D(16,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c9)
 print('c9')
 outputs = Conv2D(1,(1, 1), activation='sigmoid' )(c9)
 model = Model(inputs=[inputs], outputs=[outputs])
 return model
import numpy as np
import pandas as pd<import_modules>
def mean_iou(y_true, y_pred):
prec = []
for t in np.arange(0.5, 1.0, 0.05):
y_pred_ = tf.to_int32(y_pred > t)
 score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2, y_true)
 K.get_session().run(tf.local_variables_initializer())
 with tf.control_dependencies([up_opt]):
score = tf.identity(score)
 prec.append(score)
 return K.mean(K.stack(prec), axis=0)
 def dice_coef(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def bce_dice_loss(y_true, y_pred):
return 0.5 * binary_crossentropy(y_true, y_pred)- dice_coef(y_true, y_pred )
from catboost import CatBoostRegressor, Pool<load_from_csv>
def rle_encoding(x):
dots = np.where(x.T.flatten() == 1)[0]
run_lengths = []
prev = -2
for b in dots:
if(b>prev+1): run_lengths.extend(( b + 1, 0))
run_lengths[-1] += 1
prev = b
return run_lengths
def prob_to_rles(x, cutoff=0.5):
lab_img = label(x > cutoff)
 for i in range(1, lab_img.max() + 1):
yield rle_encoding(lab_img == i )
train = pd.read_csv('.. /input/train.csv', iterator=True, chunksize=150_000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64} )<prepare_x_and_y>
if __name__ == "__main__":
img_size = 256
batch_size = 32
train_path = '.. /input/stage1_train/'
test_path = '.. /input/stage2_test_final/'
X_train, Y_train, X_test, sizes_test = make_df(train_path, test_path, img_size)
 xtr, xval, ytr, yval = train_test_split(X_train, Y_train, test_size=0.1, random_state=7)
 train_generator, val_generator = generator(xtr, xval, ytr, yval, batch_size)
 print('imported')
 print(xtr)
 model = Unet(img_size)
 model.compile(optimizer='adam', loss=bce_dice_loss, metrics=[mean_iou])
 print('going to model fit generator now')
 model.fit_generator(train_generator, steps_per_epoch=25, epochs=20,
validation_data=val_generator, validation_steps=len(xval)/batch_size)
 preds_test = model.predict(X_test, verbose=1)
 preds_test_upsampled = []
for i in range(len(preds_test)) :
preds_test_upsampled.append(cv2.resize(preds_test[i], 
(sizes_test[i][1], sizes_test[i][0])))
 test_ids = next(os.walk(test_path)) [1]
new_test_ids = []
rles = []
for n, id_ in enumerate(test_ids):
rle = list(prob_to_rles(preds_test_upsampled[n]))
rles.extend(rle)
 new_test_ids.extend([id_] * len(rle))
sub = pd.DataFrame()
sub['ImageId'] = new_test_ids
sub['EncodedPixels'] = pd.Series(rles ).apply(lambda x: ' '.join(str(y)for y in x))
sub.to_csv('submission_relu.csv', index=False )