Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_search_model>
%matplotlib inline
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation == True:
x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate = False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate:
x = BatchActivate(x)
 return x 
<choose_model_class>
train_df = pd.read_csv('.. /input/train.csv', index_col='PassengerId')
 test_df = pd.read_csv('.. /input/test.csv', index_col='PassengerId')
 df = pd.concat([train_df, test_df])
 df.sample(10 )
def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
conv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1, True)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2, True)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4, True)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8, True)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16, True)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8, True)
 deconv3 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4, True)
 deconv2 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2, True)
 deconv1 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding="same", activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer<compute_test_metric>
df.isnull().sum()
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred >0], tf.float64 )<compute_train_metric>
is_estimated_or_null = lambda x: pd.isnull(x)or(x>1 and divmod(x, 1)[1] == 0.5)
 df['estimated_age'] = df.Age.apply(lambda age: 1 if is_estimated_or_null(age)else 0 )
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss<concatenate>
age_grouped = df[['Pclass','Sex','Embarked','Age']].groupby(['Pclass','Sex','Embarked'] ).median()
age_grouped
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train.shape)
 print(y_valid.shape )<choose_model_class>
real_age = lambda row: row.Age if not pd.isnull(row.Age)else age_grouped.loc[row.Pclass].loc[row.Sex].loc[row.Embarked].Age
df['Age'] = df[['Pclass','Sex','Embarked','Age']].apply(real_age, axis=1 )
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16,0.5)
 model1 = Model(input_layer, output_layer)
 c = optimizers.adam(lr = 0.01)
 model1.compile(loss="binary_crossentropy", optimizer=c, metrics=[my_iou_metric])
<train_model>
df['cabin_letter'] = df.Cabin.apply(lambda c: c[0] if not pd.isnull(c)else 'N')
 df.sample(5 )
model_checkpoint = ModelCheckpoint(save_model_name,monitor='my_iou_metric',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='my_iou_metric', mode = 'max',factor=0.5, patience=6, min_lr=0.0001, verbose=1)
 epochs = 55
batch_size = 32
history = model1.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr],
verbose=2 )<choose_model_class>
survival_ratio = df[['cabin_letter','Pclass','Survived']].groupby(['cabin_letter'] ).mean()
people_count = df[['cabin_letter','Name']].groupby(['cabin_letter'] ).count().rename(columns={'Name': 'passenger_count'})
 pd.concat([survival_ratio,people_count], axis=1 )
model1 = load_model(save_model_name,custom_objects={'my_iou_metric': my_iou_metric})
 input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
 c = optimizers.adam(lr = 0.01)
 model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
<train_model>
df['surname'] = df.Name.apply(lambda n: n.split(',')[0])
 df.sample(10 )
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 epochs = 50
batch_size = 32
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr,early_stopping],
verbose=2 )<load_pretrained>
surnames = df[['surname','Cabin','Pclass','Name']].groupby(['surname','Pclass'] ).count()
surnames.head()
model = load_model(save_model_name,custom_objects={'my_iou_metric_2': my_iou_metric_2,
'lovasz_loss': lovasz_loss})
<predict_on_test>
missing = surnames[(surnames.Cabin>0)&(surnames.Cabin<surnames.Name)]
missing.rename(columns={'Name': 'passenger_count'} )
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2<predict_on_test>
df[df.surname=='Brown']
preds_valid = predict_result(model,x_valid,img_size_target )<compute_train_metric>
df[df.surname=='Hoyt']
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )<categorify>
tickets_grouped = df[['Ticket','Cabin','Name']].groupby('Ticket' ).count()
candidate_tickets = tickets_grouped[(tickets_grouped['Cabin']>=1)&(tickets_grouped['Name']>=2)&(tickets_grouped['Cabin']<tickets_grouped['Name'])]
candidate_tickets
def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<predict_on_test>
df[df.Ticket=='113781']
preds_test = predict_result(model,x_test,img_size_target )<save_to_csv>
shared_tickets = candidate_tickets.index.tolist()
find_cabin_given_ticket = lambda ticket: df[(df.Ticket==ticket)&(pd.notnull(df.Cabin)) ].Cabin.values[0]
def assign_cabin(row):
if pd.isnull(row.Cabin)and row.Ticket in shared_tickets:
return find_cabin_given_ticket(row.Ticket)
 return row.Cabin
df['Cabin'] = df[['Cabin', 'Ticket']].apply(assign_cabin, axis=1)
 df['cabin_letter'] = df['Cabin'].apply(lambda c: c[0] if not pd.isnull(c)else 'N')
 df[df.Ticket=='113781']
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )<train_model>
df.Cabin.isnull().sum()
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )<set_options>
df[['Embarked', 'Survived', 'Name', 'Pclass']].groupby('Embarked' ).agg(
 {'Name': ['count'], 'Pclass': ['mean'], 'Survived': ['mean']} )
plt.style.use('seaborn-white')
 sns.set_style("white")
 %matplotlib inline
t_start = time.time()<define_variables>
df['Embarked'].fillna('S', inplace=True )
version = 5
basic_name = f'Unet_resnet_v{version}'
save_model_name = basic_name + '.model'
submission_file = basic_name + '.csv'
print(save_model_name)
 print(submission_file )<train_model>
df[pd.isnull(df.Fare)]
img_size_ori = 101
img_size_target = 101
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True )<load_from_csv>
estimated_fare = df[(df.Embarked=='S')&(df.Pclass==3)&(df.Sex=='male')].Fare.mean()
df['Fare'].fillna(estimated_fare, inplace=True )
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
len(train_df )<feature_engineering>
df[df['Age']>=80]
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
df = df[df['Age']<80]
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<categorify>
df['decade'] = df['Age'].apply(lambda age: int(age/10))
useful = ['Age', 'decade']
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
<split>
useful.append('Fare' )
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state= 1234 )<define_search_model>
df['small_family'] = df['family_size'].apply(lambda size: 1 if size<=4 else 0)
 df['big_family'] = df['family_size'].apply(lambda size: 1 if size>=7 else 0)
 df['no_family'] = df['family_size'].apply(lambda s: 1 if s==1 else 0)
 useful.extend(['SibSp', 'Parch', 'family_size', 'small_family', 'big_family', 'no_family'] )
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation == True:
x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate = False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate:
x = BatchActivate(x)
 return x 
<choose_model_class>
df['male'] = df['Sex'].map({'male': 1, 'female': 0})
 useful.append('male' )
def build_model(input_layer, start_neurons, DropoutRatio = 0.4):
conv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1, True)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2, True)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4, True)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8, True)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16, True)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8, True)
 deconv3 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4, True)
 deconv2 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2, True)
 deconv1 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding="same", activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer<compute_test_metric>
df = pd.concat([df, pd.get_dummies(df['Embarked'], prefix='embarked')], axis=1)
 useful.extend(['embarked_{}'.format(x)for x in ['C', 'S', 'Q']])
 df.sample(5 )
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred >0], tf.float64 )<compute_train_metric>
useful.append('Pclass' )
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss<concatenate>
df = pd.concat([df, pd.get_dummies(df['cabin_letter'], prefix='deck')], axis=1)
 letters = df['cabin_letter'].unique()
useful.extend(['deck_{}'.format(x)for x in letters] )
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train.shape)
 print(y_valid.shape )<choose_model_class>
ticket_count = df[['Ticket', 'Name']].groupby('Ticket' ).count().rename(columns={'Name':'count'} ).sort_values(by='count', ascending=False)
 ticket_count.head()
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16,0.5)
 model1 = Model(input_layer, output_layer)
 c = optimizers.adam(lr = 0.01)
 model1.compile(loss="binary_crossentropy", optimizer=c, metrics=[my_iou_metric])
<train_model>
df[df['Ticket']=='CA.2343']
model_checkpoint = ModelCheckpoint(save_model_name,monitor='my_iou_metric',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='my_iou_metric', mode = 'max',factor=0.6, patience=5, min_lr=0.0001, verbose=1)
 epochs = 50
batch_size = 32
history = model1.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr],
verbose=2 )<choose_model_class>
df[df['Ticket']=='1601']
model1 = load_model(save_model_name,custom_objects={'my_iou_metric': my_iou_metric})
 input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
 c = optimizers.adam(lr = 0.01)
 model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
<train_model>
df['ticket_owners'] = df['Ticket'].apply(lambda x: ticket_count.loc[x])
 df['shared_fare'] = df['Fare'] / df['ticket_owners']
df['alone'] = df[['ticket_owners','no_family']].apply(lambda row: 1 if row.ticket_owners==1 and row.no_family==1 else 0 , axis=1)
 useful.extend(['ticket_owners', 'shared_fare', 'alone'] )
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.6, patience=5, min_lr=0.0001, verbose=1)
 epochs = 50
batch_size = 32
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr,early_stopping],
verbose=2 )<load_pretrained>
older_age = df[['Ticket', 'Age']].groupby('Ticket' ).max()
df['older_relative_age'] = df['Ticket'].apply(lambda ticket: older_age.loc[ticket])
 useful.extend(['older_relative_age'] )
model = load_model(save_model_name,custom_objects={'my_iou_metric_2': my_iou_metric_2,
'lovasz_loss': lovasz_loss})
<predict_on_test>
def ticket_type(t):
if re.match('^\d+$', t):
return 'len' + str(len(t))
else:
return re.sub('[^A-Z]', '', t)
 df['ticket_type'] = df['Ticket'].apply(ticket_type)
 df[['ticket_type', 'Survived']].groupby(
 'ticket_type' ).agg({'Survived': ['mean', 'std','count']} ).sort_values(( 'Survived','count'), ascending=False )
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2<predict_on_test>
def useful_ticket_type(ticket_type):
useful_types = ['A', 'SOTONOQ', 'WC']
if ticket_type in useful_types:
return ticket_type
else:
return 'other'
df['useful_ticket_type'] = df['ticket_type'].apply(useful_ticket_type)
 df = pd.concat(
 [df, pd.get_dummies(df['useful_ticket_type'], prefix='ticket_type')], axis=1)
 letters = df['useful_ticket_type'].unique()
useful.extend(['ticket_type_{}'.format(x)for x in letters])
 df.sample(10 )
preds_valid = predict_result(model,x_valid,img_size_target )<compute_train_metric>
df['name_length_short'] = df['name_length'].apply(lambda s: 1 if s <= 35 else 0)
 df['name_length_mid'] = df['name_length'].apply(lambda s: 1 if 35 < s <=58 else 0)
 df['name_length_long'] = df['name_length'].apply(lambda s: 1 if s > 58 else 0)
 useful.extend(['name_length', 'name_length_short', 'name_length_mid', 'name_length_long'] )
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )<categorify>
df['lang'] = df['Name'].apply(lambda n: langid.classify(n)[0])
 df[['Name','lang']].sample(10 )
def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<predict_on_test>
lang_count = df[['lang','Name']].groupby('lang' ).count().rename(columns={'Name':'count'})
 lang_class = df[['lang','Pclass']].groupby('lang' ).mean()
lang_survived = df[['lang','Survived']].groupby('lang' ).mean()
pd.concat([lang_count, lang_class, lang_survived], axis=1 ).sort_values(by='count', ascending=False ).head(15 )
preds_test = predict_result(model,x_test,img_size_target )<save_to_csv>
language_groups = {
'uk':('cy', 'en'),
'germanic':('da', 'de', 'nl'),
'latin':('es', 'fr', 'it', 'la', 'pt', 'br', 'ro'),
'african':('af', 'rw', 'xh'),
'asian':('id', 'tl', 'tr')
 }
language_map = { y:x for x in language_groups for y in language_groups[x]}
df['lang_group'] = df['lang'].apply(lambda l: language_map[l] if l in language_map else 'other')
 survived_avg_per_group = df[['lang_group','Survived']].groupby('lang_group' ).mean()
survived_std_per_group = df[['lang_group','Survived']].groupby('lang_group' ).std().rename(columns={'Survived':'std'})
 pd.concat([survived_avg_per_group, survived_std_per_group], axis=1 )
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )<train_model>
df = pd.concat([df, pd.get_dummies(df['lang_group'], prefix='lang_group')], axis=1)
 langs = df['lang_group'].unique()
useful.extend(['lang_group_{}'.format(x)for x in langs] )
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )<categorify>
surnames = df[['surname', 'Name']].groupby('surname' ).count().rename(columns={'Name':'count'})
 df['surname_count'] = df['surname'].apply(lambda x: surnames.loc[x])
 useful.append('surname_count' )
def rle_decode(mask_rle):

 s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(101*101, dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(101,101)
 def rle_encode(im):

 pixels = im.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def crf(original_image, mask_img):
if(len(mask_img.shape)<3):
mask_img = gray2rgb(mask_img)
 annotated_label = mask_img[:,:,0] +(mask_img[:,:,1]<<8)+(mask_img[:,:,2]<<16)
 colors, labels = np.unique(annotated_label, return_inverse=True)
 n_labels = 2
d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0], n_labels)
 U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
 d.setUnaryEnergy(U)
 d.addPairwiseGaussian(sxy=(3, 3), compat=3, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
 Q = d.inference(10)
 MAP = np.argmax(Q, axis=0)
 return MAP.reshape(( original_image.shape[0],original_image.shape[1]))
df = pd.read_csv('.. /input/candsub/blend_ez.csv')
 i = 0
j = 0
plt.figure(figsize=(30,15))
plt.subplots_adjust(bottom=0.2, top=0.8, hspace=0.2)
 while True:
if str(df.loc[i,'rle_mask'])!=str(np.nan):
decoded_mask = rle_decode(df.loc[i,'rle_mask'])
 plt.subplot(1,6,j+1)
 plt.imshow(decoded_mask)
 plt.title('ID: '+df.loc[i,'id'])
 j = j + 1
if j>5:
break
i = i + 1
for i in tqdm(range(df.shape[0])) :
if str(df.loc[i,'rle_mask'])!=str(np.nan):
decoded_mask = rle_decode(df.loc[i,'rle_mask'])
 orig_img = imread('.. /input/tgs-salt-identification-challenge/test/images/'+df.loc[i,'id']+'.png')
 crf_output = crf(orig_img,decoded_mask)
 df.loc[i,'rle_mask'] = rle_encode(crf_output)
 df.to_csv('crf_correction_unet_bn_diceloss.csv',index=False )<set_options>
df['title'] = df['Name'].apply(lambda n: n.split(',')[1].split('.')[0].strip())
 df[['title', 'Survived']].groupby('title' ).agg({'Survived': [
'mean', 'std', 'count']} ).sort_values(( 'Survived', 'count'), ascending=False )
plt.style.use('seaborn-white')
 sns.set_style("white")
 %matplotlib inline 
<define_variables>
title_groups = {
"Capt": "sacrifies",
"Col": "army",
"Rev": "sacrifies",
"Major": "army",
"Mr" : "Mr",
"Master": "Master",
"Miss": "Miss",
"Mrs": "Mrs",
"Mme": "Mrs",
"Ms": "Mrs",
"Mlle": "Miss"
}
df['title_group'] = df['title'].apply(lambda t: title_groups[t] if t in title_groups else 'other')
 df = pd.concat([df, pd.get_dummies(df['title_group'], prefix='title_group')], axis=1)
 t_g = df['title_group'].unique()
useful.extend(['title_group_{}'.format(x)for x in t_g] )
img_size_ori = 101
img_size_target = 101
im_width = 101
im_height = 101
im_chan = 1
basicpath = '.. /input/'
path_train = basicpath + 'train/'
path_test = basicpath + 'test/'
path_train_images = path_train + 'images/'
path_train_masks = path_train + 'masks/'
path_test_images = path_test + 'images/' 
<load_from_csv>
df.corr() ['Survived'].sort_values()
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
len(train_df )<feature_engineering>
train = df[df['Survived'].notnull() ]
train_X = train[useful]
train_y = train['Survived']
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
clf = RandomForestClassifier(n_estimators=100, random_state=1)
 clf = clf.fit(train_X, train_y )
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<categorify>
useful = importances.tail(33 ).index.tolist()
useful.remove('title_group_Mr')
 useful.remove('Fare')
 useful.remove('older_relative_age')
 train_X=train[useful]
train_X.shape
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
<split>
search_best_hyperparameters = False
if search_best_hyperparameters:
parameter_grid = {
'n_estimators': [10, 20, 50, 100, 200, 500],
'learning_rate': [0.1, 0.2, 0.5, 1, 1.2],
'random_state': [1]
}
model = AdaBoostClassifier()
gs = GridSearchCV(
 model,
scoring='accuracy',
param_grid=parameter_grid,
cv=4,
n_jobs=-1)
 gs.fit(train_X, train_y)
 params = gs.best_params_
print(params)
 else:
params = {
'learning_rate': 0.1,
'n_estimators': 500,
'random_state': 1
}
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state= 1234 )<define_search_model>
clf = AdaBoostClassifier(**params)
 clf = clf.fit(train_X, train_y)
 clf.score(train_X, train_y )
ACTIVATION = "relu"
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 x = BatchNormalization()(x)
 if activation == True:
x = Activation(ACTIVATION )(x)
 return x
def residual_block(blockInput, num_filters=16):
x = Activation(ACTIVATION )(blockInput)
 x = BatchNormalization()(x)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 return x 
<choose_model_class>
test = df[df['Survived'].isnull() ]
test_X = test[useful]
test_y = clf.predict(test_X )
def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
conv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = Activation(ACTIVATION )(conv1)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = Activation(ACTIVATION )(conv2)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = Activation(ACTIVATION )(conv3)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = Activation(ACTIVATION )(conv4)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16)
 convm = Activation(ACTIVATION )(convm)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = Activation(ACTIVATION )(uconv4)
 deconv3 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = Activation(ACTIVATION )(uconv3)
 deconv2 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = Activation(ACTIVATION )(uconv2)
 deconv1 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = Activation(ACTIVATION )(uconv1)
 uconv1 = Dropout(DropoutRatio/2 )(uconv1)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv1)
 return output_layer<compute_train_metric>
submit = pd.DataFrame(test_y.astype(int), index=test_X.index, columns=['Survived'])
 submit.head()
<concatenate><EOS> 
submit.to_csv('submission.csv' )