Kaggle competition: Titanic - Machine Learning from Disaster
im_width = 101
im_height = 101
im_chan = 1
basicpath = '.. /input/'
path_train = basicpath + 'train/'
path_test = basicpath + 'test/'
path_train_images = path_train + 'images/'
path_train_masks = path_train + 'masks/'
path_test_images = path_test + 'images/' 
<train_model>
!pip install autofeat
img_size_ori = 101
img_size_target = 101
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
<load_from_csv>
warnings.filterwarnings("ignore")
 pd.set_option('max_columns', 100 )
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
len(train_df )<feature_engineering>
traindf = pd.read_csv('.. /input/titanic/train.csv' ).set_index('PassengerId')
 testdf = pd.read_csv('.. /input/titanic/test.csv' ).set_index('PassengerId')
 df = pd.concat([traindf, testdf], axis=0, sort=False )
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
df = pd.concat([traindf, testdf], axis=0, sort=False)
 df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['IsWomanOrBoy'] =(( df.Title == 'Master')|(df.Sex == 'female'))
df['LastName'] = df.Name.str.split(',' ).str[0]
family = df.groupby(df.LastName ).Survived
df['WomanOrBoyCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).count())
 df['WomanOrBoyCount'] = df.mask(df.IsWomanOrBoy, df.WomanOrBoyCount - 1, axis=0)
 df['FamilySurvivedCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).sum())
 df['FamilySurvivedCount'] = df.mask(df.IsWomanOrBoy, df.FamilySurvivedCount - \
df.Survived.fillna(0), axis=0)
 df['WomanOrBoySurvived'] = df.FamilySurvivedCount / df.WomanOrBoyCount.replace(0, np.nan)
 df.WomanOrBoyCount = df.WomanOrBoyCount.replace(np.nan, 0)
 df['Alone'] =(df.WomanOrBoyCount == 0)
 df['Title'] = df['Title'].replace('Ms','Miss')
 df['Title'] = df['Title'].replace('Mlle','Miss')
 df['Title'] = df['Title'].replace('Mme','Mrs')
 df['Embarked'] = df['Embarked'].fillna('S')
 df['Deck'] = df['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'M')
 df.loc[(df['Deck'] == 'T'), 'Deck'] = 'A'
med_fare = df.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
df['Fare'] = df['Fare'].fillna(med_fare)
 df['Age'] = df.groupby(['Sex', 'Pclass', 'Title'])['Age'].apply(lambda x: x.fillna(x.median()))
df['Family_Size'] = df['SibSp'] + df['Parch'] + 1
cols_to_drop = ['Name','Ticket','Cabin']
df = df.drop(cols_to_drop, axis=1)
 df.WomanOrBoySurvived = df.WomanOrBoySurvived.fillna(0)
 df.WomanOrBoyCount = df.WomanOrBoyCount.fillna(0)
 df.FamilySurvivedCount = df.FamilySurvivedCount.fillna(0)
 df.Alone = df.Alone.fillna(0)
 df.Alone = df.Alone*1
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<categorify>
target = df.Survived.loc[traindf.index]
df = df.drop(['Survived'], axis=1)
 train, test = df.loc[traindf.index], df.loc[testdf.index]
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
<split>
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
categorical_columns = []
features = train.columns.values.tolist()
for col in features:
if train[col].dtype in numerics: continue
categorical_columns.append(col)
 categorical_columns
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state= 1234 )<define_search_model>
for col in categorical_columns:
if col in train.columns:
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 x = BatchNormalization()(x)
 if activation == True:
x = Activation('relu' )(x)
 return x
def residual_block(blockInput, num_filters=16):
x = Activation('relu' )(blockInput)
 x = BatchNormalization()(x)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 return x 
<choose_model_class>
model = AutoFeatRegressor()
model
def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
conv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = Activation('relu' )(conv1)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = Activation('relu' )(conv2)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = Activation('relu' )(conv3)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = Activation('relu' )(conv4)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16)
 convm = Activation('relu' )(convm)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = Activation('relu' )(uconv4)
 deconv3 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = Activation('relu' )(uconv3)
 deconv2 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = Activation('relu' )(uconv2)
 deconv1 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = Activation('relu' )(uconv1)
 uconv1 = Dropout(DropoutRatio/2 )(uconv1)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv1)
 return output_layer<concatenate>
X_train_feature_creation = model.fit_transform(train, target)
 X_test_feature_creation = model.transform(test)
 X_train_feature_creation.head()
x_train2 = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train2 = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train2.shape)
 print(y_valid.shape )<choose_model_class>
print('Number of new features -',X_train_feature_creation.shape[1] - train.shape[1] )
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16,0.5)
 model = Model(input_layer, output_layer)
 model.compile(loss="binary_crossentropy", optimizer="adam", metrics=[my_iou_metric])
 model.summary()<train_model>
df2 = pd.concat([df.WomanOrBoySurvived.fillna(0), df.Alone, df.Sex.replace({'male': 0, 'female': 1})], axis=1)
 train2, test2 = df2.loc[traindf.index], df2.loc[testdf.index]
early_stopping = EarlyStopping(monitor='val_my_iou_metric', mode = 'max',patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint("./unet_best1.model",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric', mode = 'max',factor=0.2, patience=5, min_lr=0.00001, verbose=1)
 epochs = 200
batch_size = 32
history = model.fit(x_train2, y_train2,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[early_stopping, model_checkpoint, reduce_lr],
verbose=2 )<load_pretrained>
X_train_feature_creation2 = model.fit_transform(train2.to_numpy() , target.to_numpy().flatten())
 X_test_feature_creation2 = model.transform(test2.to_numpy())
 X_train_feature_creation2.head()
model = load_model("./unet_best1.model",custom_objects={'my_iou_metric': my_iou_metric})
<predict_on_test>
print('Number of new features -',X_train_feature_creation2.shape[1] - train2.shape[1] )
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test1 = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test2 = np.array([ np.fliplr(x)for x in preds_test2_refect])
 preds_avg =(preds_test1 +preds_test2)/2
return preds_avg<predict_on_test>
test_x = df2.loc[testdf.index]
test_x['Survived'] =(((test_x.WomanOrBoySurvived <= 0.238)&(test_x.Sex > 0.5)&(test_x.Alone > 0.5)) | \ 
(( test_x.WomanOrBoySurvived > 0.238)& \
~(( test_x.WomanOrBoySurvived > 0.55)&(test_x.WomanOrBoySurvived <= 0.633))))
pd.DataFrame({'Survived': test_x['Survived'].astype(int)}, \
index=testdf.index ).reset_index().to_csv('survived.csv', index=False )
preds_valid = predict_result(model,x_valid,img_size_target)
 preds_valid2 = np.array([downsample(x)for x in preds_valid])
 y_valid2 = np.array([downsample(x)for x in y_valid] )<compute_train_metric>
LB_simple_rule = 0.83253
thresholds = np.linspace(0.3, 0.7, 31)
 ious = np.array([iou_metric_batch(y_valid2, np.int32(preds_valid2 > threshold)) for threshold in tqdm_notebook(thresholds)])
<categorify>
model_LR = LinearRegression().fit(train,target.to_numpy().flatten())
 model_Autofeat = LinearRegression().fit(X_train_feature_creation, target.to_numpy().flatten() )
def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<predict_on_test>
test['Survived_LR'] = np.clip(model_LR.predict(test),0,1)
 test['Survived_AF'] = np.clip(model_Autofeat.predict(X_test_feature_creation),0,1 )
preds_test = predict_result(model,x_test,img_size_target )<compute_test_metric>
def hyperopt_gb_score(params):
clf = GradientBoostingClassifier(**params)
 current_score = cross_val_score(clf, X_train_feature_creation, target, cv=10 ).mean()
print(current_score, params)
 return current_score
space_gb = {
'n_estimators': hp.choice('n_estimators', range(100, 1000)) ,
'max_depth': hp.choice('max_depth', np.arange(2, 10, dtype=int))
}
best = fmin(fn=hyperopt_gb_score, space=space_gb, algo=tpe.suggest, max_evals=10)
 print('best:')
 print(best )
t1 = time.time()
pred_dict = {idx: rle_encode(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
t2 = time.time()
print(f"Usedtime = {t2-t1} s" )<save_to_csv>
params = space_eval(space_gb, best)
 params
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission4.csv' )<categorify>
gradient_boosting = GradientBoostingClassifier(**params)
 gradient_boosting.fit(X_train_feature_creation, target)
 Y_pred = gradient_boosting.predict(X_test_feature_creation ).astype(int)
 gradient_boosting.score(X_train_feature_creation, target)
 acc_gradient_boosting = round(gradient_boosting.score(X_train_feature_creation, target)* 100, 2)
 acc_gradient_boosting
def rle_encode(im):

 pixels = im.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle_decode(rle_mask):

 s = rle_mask.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(101*101, dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(101,101 )<merge>
pd.DataFrame({'Survived': test['Survived_LR'].astype(int)}, \
index=testdf.index ).reset_index().to_csv('survived_LR16.csv', index=False)
 pd.DataFrame({'Survived': test['Survived_AF'].astype(int)}, \
index=testdf.index ).reset_index().to_csv('survived_Autofeat16.csv', index=False)
 pd.DataFrame({'Survived': Y_pred}, \
index=testdf.index ).reset_index().to_csv('survived_GBC16.csv', index=False )
def merge_dataframes(dfs, merge_keys):
dfs_merged = reduce(lambda left,right: pd.merge(left, right, on=merge_keys), dfs)
 return dfs_merged<load_from_csv>
LB_LR16 = 0.69377
LB_Autofeat16 = 0.67942
LB_GBC16 = 0.82296
df1 = pd.read_csv('.. /input/submission/crf_c_1.csv' ).rename(columns={'rle_mask': 'rle_mask'})
 df2 = pd.read_csv('.. /input/submission/crf_c.csv' ).rename(columns={'rle_mask': 'rle_mask1'})
 df3 = pd.read_csv('.. /input/submission/crf_c_2.csv' ).rename(columns={'rle_mask': 'rle_mask2'})
 df4 = pd.read_csv('.. /input/submission/sub.csv' ).rename(columns={'rle_mask': 'rle_mask3'})
<merge>
model_LR2 = LinearRegression().fit(train2,target.to_numpy().flatten())
 model_Autofeat2 = LinearRegression().fit(X_train_feature_creation2, target.to_numpy().flatten() )
dfs = [df1,df2,df3,df4]
merge_keys=['id']
df = merge_dataframes(dfs, merge_keys=merge_keys)
 df.head()<categorify>
test2['Survived_LR'] = np.clip(model_LR2.predict(test2),0,1)
 test2['Survived_AF'] = np.clip(model_Autofeat2.predict(X_test_feature_creation2),0,1 )
res = df1.copy()
i = 0
while True:
if i == 18000:
break
if(str(df.loc[i,'rle_mask'])!=str(np.nan)) and(str(df.loc[i,'rle_mask1'])!=str(np.nan)) and(str(df.loc[i,'rle_mask2'])!=str(np.nan)) and(str(df.loc[i,'rle_mask3'])!=str(np.nan)) :
decoded_mask1 = rle_decode(df.loc[i,'rle_mask'])
 decoded_mask2 = rle_decode(df.loc[i,'rle_mask1'])
 decoded_mask3 = rle_decode(df.loc[i,'rle_mask2'])
 decoded_mask4 = rle_decode(df.loc[i,'rle_mask3'])
 decoded_mask_all1 = decoded_mask1 + decoded_mask2 + decoded_mask3 + decoded_mask4
decoded_mask_all1[decoded_mask_all1<=2] = 0
decoded_mask_all1[decoded_mask_all1 >2] = 1
mask = rle_encode(decoded_mask_all1)
 res.loc[i,'rle_mask'] = mask
else:
res.loc[i,'rle_mask'] = df1.loc[i,'rle_mask']
i = i + 1 
<save_to_csv>
def hyperopt_gb_score(params):
clf = GradientBoostingClassifier(**params)
 current_score = cross_val_score(clf, X_train_feature_creation2, target, cv=10 ).mean()
print(current_score, params)
 return current_score
space_gb = {
'n_estimators': hp.choice('n_estimators', range(100, 1000)) ,
'max_depth': hp.choice('max_depth', np.arange(2, 10, dtype=int))
}
best = fmin(fn=hyperopt_gb_score, space=space_gb, algo=tpe.suggest, max_evals=10)
 print('best:')
 print(best )
res.to_csv('blend_ez.csv',index=False )<set_options>
params2 = space_eval(space_gb, best)
 params2
plt.style.use('seaborn-white')
 sns.set_style("white")
<train_model>
gradient_boosting2 = GradientBoostingClassifier(**params2)
 gradient_boosting2.fit(X_train_feature_creation2, target)
 Y_pred2 = gradient_boosting2.predict(X_test_feature_creation2 ).astype(int)
 gradient_boosting2.score(X_train_feature_creation2, target)
 acc_gradient_boosting2 = round(gradient_boosting2.score(X_train_feature_creation2, target)* 100, 2)
 acc_gradient_boosting2
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
<load_from_csv>
pd.DataFrame({'Survived': test2['Survived_LR'].astype(int)}, \
index=testdf.index ).reset_index().to_csv('survived_LR3.csv', index=False)
 pd.DataFrame({'Survived': test2['Survived_AF'].astype(int)}, \
index=testdf.index ).reset_index().to_csv('survived_Autofeat3.csv', index=False)
 pd.DataFrame({'Survived': Y_pred2}, \
index=testdf.index ).reset_index().to_csv('survived_GBC3.csv', index=False )
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]<feature_engineering>
LB_LR3 = 0.67942
LB_Autofeat3 = 0.69377
LB_GBC3 = 0.83253
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
models = pd.DataFrame({
'Model': ['Simple rule','Linear Regression without Autofeat', 'Linear Regression with Autofeat',
'GradientBoostingClassifier with Autofeat'],
'LB_for_16_features': [LB_simple_rule, LB_LR16, LB_Autofeat16, LB_GBC16],
'LB_for_3opt_features': [LB_simple_rule, LB_LR3, LB_Autofeat3, LB_GBC3]} )
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
models.sort_values(by=['LB_for_3opt_features', 'LB_for_16_features'], ascending=False )
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2 )<categorify>
models.sort_values(by=['LB_for_16_features', 'LB_for_3opt_features'], ascending=False )