Kaggle competition: Google Cloud & NCAA® ML Competition 2018-Men's
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation==True: x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate=False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate: x = BatchActivate(x)
 return x<choose_model_class>
df_seeds = pd.read_csv('.. /input/NCAATourneySeeds.csv')
 df_tour = pd.read_csv('.. /input/NCAATourneyCompactResults.csv' )
def build_model(input_layer, start_neurons, DropoutRatio=0.5):
conv1 = Conv2D(start_neurons*1,(3,3), activation=None, padding='same' )(input_layer)
 conv1 = residual_block(conv1, start_neurons*1)
 conv1 = residual_block(conv1, start_neurons*1, True)
 pool1 = MaxPooling2D(( 2,2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons*2,(3,3), activation=None, padding='same' )(pool1)
 conv2 = residual_block(conv2, start_neurons*2)
 conv2 = residual_block(conv2, start_neurons*2, True)
 pool2 = MaxPooling2D(( 2,2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons*4,(3,3), activation=None, padding='same' )(pool2)
 conv3 = residual_block(conv3, start_neurons*4)
 conv3 = residual_block(conv3, start_neurons*4, True)
 pool3 = MaxPooling2D(( 2,2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons*8,(3,3), activation=None, padding='same' )(pool3)
 conv4 = residual_block(conv4, start_neurons*8)
 conv4 = residual_block(conv4, start_neurons*8, True)
 pool4 = MaxPooling2D(( 2,2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons*16,(3,3), activation=None, padding='same' )(pool4)
 convm = residual_block(convm, start_neurons*16)
 convm = residual_block(convm, start_neurons*16, True)
 deconv4 = Conv2DTranspose(start_neurons*8,(3,3), strides=(2,2), padding='same' )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons*8,(3,3), activation=None, padding='same' )(uconv4)
 uconv4 = residual_block(uconv4, start_neurons*8)
 uconv4 = residual_block(uconv4, start_neurons*8, True)
 deconv3 = Conv2DTranspose(start_neurons*4,(3,3), strides=(2,2), padding='valid' )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons*4,(3,3), activation=None, padding='same' )(uconv3)
 uconv3 = residual_block(uconv3, start_neurons*4)
 uconv3 = residual_block(uconv3, start_neurons*4, True)
 deconv2 = Conv2DTranspose(start_neurons*2,(3,3), strides=(2,2), padding='same' )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons*2,(3,3), activation=None, padding='same' )(uconv2)
 uconv2 = residual_block(uconv2, start_neurons*2)
 uconv2 = residual_block(uconv2, start_neurons*2, True)
 deconv1 = Conv2DTranspose(start_neurons*1,(3,3), strides=(2,2), padding='valid' )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons*1,(3,3), activation=None, padding='same' )(uconv1)
 uconv1 = residual_block(uconv1, start_neurons*1)
 uconv1 = residual_block(uconv1, start_neurons*1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding='same', activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer<compute_train_metric>
df_seeds['seed_int'] = df_seeds['Seed'].apply(lambda x : int(x[1:3]))
df_winseeds = df_seeds.loc[:, ['TeamID', 'Season', 'seed_int']].rename(columns={'TeamID':'WTeamID', 'seed_int':'WSeed'})
 df_lossseeds = df_seeds.loc[:, ['TeamID', 'Season', 'seed_int']].rename(columns={'TeamID':'LTeamID', 'seed_int':'LSeed'})
 df_dummy = pd.merge(left=df_tour, right=df_winseeds, how='left', on=['Season', 'WTeamID'])
 df_concat = pd.merge(left=df_dummy, right=df_lossseeds, on=['Season', 'LTeamID'] )
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred >0], tf.float64 )<compute_train_metric>
df_concat['DiffSeed'] = df_concat[['LSeed', 'WSeed']].apply(lambda x : 0 if x[0] == x[1] else 1, axis = 1 )
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.elu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss<concatenate>
df_sample_sub = pd.read_csv('.. /input/SampleSubmissionStage1.csv' )
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train.shape)
 print(y_valid.shape )<choose_model_class>
df_sample_sub['Season'] = df_sample_sub['ID'].apply(lambda x : int(x.split('_')[0]))
df_sample_sub['TeamID1'] = df_sample_sub['ID'].apply(lambda x : int(x.split('_')[1]))
df_sample_sub['TeamID2'] = df_sample_sub['ID'].apply(lambda x : int(x.split('_')[2]))
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16,0.5)
 model1 = Model(input_layer, output_layer)
 c = optimizers.adam(lr = 0.005)
 model1.compile(loss="binary_crossentropy", optimizer=c, metrics=[my_iou_metric])
 model1.summary()<train_model>
winners = df_concat.rename(columns = { 'WTeamID' : 'TeamID1',
'LTeamID' : 'TeamID2',
'WScore' : 'Team1_Score',
'LScore' : 'Team2_Score'} ).drop(['WSeed', 'LSeed', 'WLoc'], axis = 1)
 winners['Result'] = 1.0
losers = df_concat.rename(columns = { 'WTeamID' : 'TeamID2',
'LTeamID' : 'TeamID1',
'WScore' : 'Team2_Score',
'LScore' : 'Team1_Score'} ).drop(['WSeed', 'LSeed', 'WLoc'], axis = 1)
 losers['Result'] = 0.0
train = pd.concat([winners, losers], axis = 0 ).reset_index(drop = True)
 train['Score_Ratio'] = train['Team1_Score'] / train['Team2_Score']
train['Score_Total'] = train['Team1_Score'] + train['Team2_Score']
train['Score_Pct'] = train['Team1_Score'] / train['Score_Total']
early_stopping = EarlyStopping(monitor='val_my_iou_metric', mode = 'max',patience=15, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name, monitor='my_iou_metric', mode='max',
save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric', mode='max', factor=0.5, patience=5,
min_lr=0.0001, verbose=1)
 epochs = 100
batch_size = 128
t_model1_start = time.time()
history = model1.fit(x_train, y_train,
validation_data = [x_valid, y_valid],
epochs = epochs,
batch_size = batch_size,
callbacks = [early_stopping, model_checkpoint, reduce_lr],
verbose = 2)
 t_model1_end = time.time()
print(f"Run time = {(t_model1_end-t_model1_start)/3600} hours" )<choose_model_class>
years = [2014, 2015, 2016, 2017]
model1 = load_model(save_model_name, custom_objects={'my_iou_metric':my_iou_metric})
 input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
 c = optimizers.adam(lr=0.01)
 model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
 model.summary()<train_model>
train_test_inner = pd.merge(train.loc[ train['Season'].isin(years), : ].reset_index(drop = True),
df_sample_sub.drop(['ID', 'Pred'], axis = 1),
on = ['Season', 'TeamID1', 'TeamID2'], how = 'inner' )
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=30, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.5, patience=5,
min_lr=0.00005, verbose=1)
 epochs = 120
batch_size = 64
t_model2_start = time.time()
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr,early_stopping],
verbose=2)
 t_model2_end = time.time()
print(f"Run time = {(t_model2_end-t_model2_start)/3600} hours" )<load_pretrained>
def geo_mean(x):
return np.exp(np.mean(np.log(x)))
 def harm_mean(x):
return np.mean(x ** -1.0)** -1.0
team1d_score_spread = train_test_inner.groupby(['Season', 'TeamID1'])[['Score_Ratio', 'Score_Pct']]\ 
.agg({ 'Score_Ratio': geo_mean, 'Score_Pct' : harm_mean} ).reset_index() \ 
.set_index('Season' ).rename(columns = {'Score_Ratio' : 'Score_Ratio1', 'Score_Pct' : 'Score_Pct1'})
 team2d_score_spread = train_test_inner.groupby(['Season', 'TeamID2'])[['Score_Ratio', 'Score_Pct']]\ 
.agg({ 'Score_Ratio': geo_mean, 'Score_Pct' : harm_mean} ).reset_index() \ 
.set_index('Season' ).rename(columns = {'Score_Ratio' : 'Score_Ratio2', 'Score_Pct' : 'Score_Pct2'})
 score_spread = team1d_score_spread.join(team2d_score_spread ).reset_index()
score_spread['Score_Ratio'] = score_spread[['Score_Ratio1', 'Score_Ratio2']].apply(lambda x :(x[0] *(x[1] ** -1.0)) , axis = 1)** 0.5
score_spread['Score_Pct'] = score_spread[['Score_Pct1', 'Score_Pct2']].apply(lambda x : 0.5*(x[0] ** -1.0)+ 0.5*(1.0 - x[1])** -1.0, axis = 1)** -1.0
score_spread.head()
model = load_model(save_model_name,custom_objects={'my_iou_metric_2': my_iou_metric_2,
'lovasz_loss': lovasz_loss} )<predict_on_test>
X_train = train_test_inner.loc[:, ['Season', 'NumOT', 'Score_Ratio', 'Score_Pct']]
train_labels = train_test_inner['Result']
train_test_outer = pd.merge(train.loc[ train['Season'].isin(years), : ].reset_index(drop = True),
df_sample_sub.drop(['ID', 'Pred'], axis = 1),
on = ['Season', 'TeamID1', 'TeamID2'], how = 'outer')
 train_test_outer = train_test_outer.loc[ train_test_outer['Result'].isnull() ,
['TeamID1', 'TeamID2', 'Season']]
train_test_missing = pd.merge(pd.merge(score_spread.loc[:, ['TeamID1', 'TeamID2', 'Season', 'Score_Ratio', 'Score_Pct']],
train_test_outer, on = ['TeamID1', 'TeamID2', 'Season']),
num_ot.loc[:, ['TeamID1', 'TeamID2', 'Season', 'NumOT']],
on = ['TeamID1', 'TeamID2', 'Season'] )
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2<predict_on_test>
X_test = train_test_missing.loc[:, ['Season', 'NumOT', 'Score_Ratio', 'Score_Pct']]
n = X_train.shape[0]
train_test_merge = pd.concat([X_train, X_test], axis = 0 ).reset_index(drop = True)
 train_test_merge = pd.concat([pd.get_dummies(train_test_merge['Season'].astype(object)) ,
train_test_merge.drop('Season', axis = 1)], axis = 1)
 train_test_merge = pd.concat([pd.get_dummies(train_test_merge['NumOT'].astype(object)) ,
train_test_merge.drop('NumOT', axis = 1)], axis = 1)
 X_train = train_test_merge.loc[:(n - 1), :].reset_index(drop = True)
 X_test = train_test_merge.loc[n:, :].reset_index(drop = True )
preds_valid = predict_result(model,x_valid,img_size_target )<compute_train_metric>
x_max = X_train.max()
x_min = X_train.min()
X_train =(X_train - x_min)/(x_max - x_min + 1e-14)
 X_test =(X_test - x_min)/(x_max - x_min + 1e-14 )
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )<categorify>
log_clf = LogisticRegressionCV(cv = 5)
 log_clf.fit(X_train, train_labels )
def rle_encode(im):

 pixels = im.flatten(order='F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<predict_on_test>
train_test_inner['Pred1'] = log_clf.predict_proba(X_train)[:,1]
train_test_missing['Pred1'] = log_clf.predict_proba(X_test)[:,1]
preds_test = predict_result(model,x_test,img_size_target )<compute_test_metric>
sub = pd.merge(df_sample_sub,
pd.concat([train_test_missing.loc[:, ['Season', 'TeamID1', 'TeamID2', 'Pred1']],
train_test_inner.loc[:, ['Season', 'TeamID1', 'TeamID2', 'Pred1']] ],
axis = 0 ).reset_index(drop = True),
on = ['Season', 'TeamID1', 'TeamID2'], how = 'outer' )
pred_dict = {idx: rle_encode(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }<save_to_csv>
team1_probs = sub.groupby('TeamID1')['Pred1'].apply(lambda x :(x ** -1.0 ).mean() ** -1.0 ).fillna(0.5 ).to_dict()
team2_probs = sub.groupby('TeamID2')['Pred1'].apply(lambda x :(x ** -1.0 ).mean() ** -1.0 ).fillna(0.5 ).to_dict()
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )<train_model>
sub['Pred'] = sub[['TeamID1', 'TeamID2','Pred1']]\ 
.apply(lambda x : team1_probs.get(x[0])*(1 - team2_probs.get(x[1])) if np.isnan(x[2])else x[2],
axis = 1 )
<set_options><EOS> 
sub[['ID', 'Pred']].to_csv('sub.csv', index = False )