Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<save_to_csv>
plt.style.use('seaborn-white')
 sns.set_style("white")
 %matplotlib inline
t_start = time.time()
submission = pd.DataFrame({'id': test.index, 'trip_duration': y_pred})
 submission.to_csv('submission.csv', index=False)
<load_from_csv>
version = 5
basic_name = f'Unet_resnet_v{version}'
save_model_name = basic_name + '.model'
submission_file = basic_name + '.csv'
print(save_model_name)
 print(submission_file )
train = pd.read_csv(".. /input/train.csv" )<count_values>
img_size_ori = 101
img_size_target = 101
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True )
print(train.store_and_fwd_flag.value_counts() )<feature_engineering>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
len(train_df )
def drop_duration(df):
df.loc[df.trip_duration < trip_duration_min,'trip_duration'] = np.nan
value=df.trip_duration.min()
df.trip_duration.fillna(value=value, inplace=True)
 df.loc[df.trip_duration > trip_duration_max,'trip_duration'] = np.nan
value=df.trip_duration.max()
df.trip_duration.fillna(value=value, inplace=True)
 return<drop_column>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
drop_duration(train )<feature_engineering>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
def conf_int_duration(df):

 conf_int_duration = np.percentile(df.trip_duration, [2.5,97.5])
 print('
Confidental interval trip_duration: {}'.format(conf_int_duration))
df.loc[df.trip_duration < conf_int_duration[0],'trip_duration'] = np.nan
value=df.trip_duration.min()
df.trip_duration.fillna(value=value, inplace=True)
 df.loc[df.trip_duration > conf_int_duration[1],'trip_duration'] = np.nan
value=df.trip_duration.max()
df.trip_duration.fillna(value=value, inplace=True)
 print("Trip_duration describe past drop:
",df.trip_duration.describe())
 return<load_from_csv>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
test = pd.read_csv(".. /input/test.csv")
 test.info()<feature_engineering>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state= 1234 )
result['pickup_longitude'] = result.pickup_longitude.round(5)
 result['pickup_latitude'] = result.pickup_latitude.round(5)
 result.loc[result.pickup_latitude < pickup_lat_min, 'pickup_latitude'] = np.nan
value=result.pickup_latitude.min()
result.pickup_latitude.fillna(value=value, inplace=True)
 result.loc[result.pickup_latitude > pickup_lat_max, 'pickup_latitude'] = np.nan
value=result.pickup_latitude.max()
result.pickup_latitude.fillna(value=value, inplace=True)
 result.loc[result.pickup_longitude < pickup_long_min, 'pickup_longitude'] = np.nan
value=result.pickup_longitude.min()
result.pickup_longitude.fillna(value=value, inplace=True)
 result.loc[result.pickup_longitude > pickup_long_max, 'pickup_longitude'] = np.nan
value=result.pickup_longitude.max()
result.pickup_longitude.fillna(value=value, inplace=True)
 result['dropoff_longitude'] = result.dropoff_longitude.round(5)
 result['dropoff_latitude'] = result.dropoff_latitude.round(5)
 result.loc[result.dropoff_latitude < dropoff_lat_min, 'dropoff_latitude'] = np.nan
value=result.dropoff_latitude.min()
result.dropoff_latitude.fillna(value=value, inplace=True)
 result.loc[result.dropoff_latitude > dropoff_lat_max, 'dropoff_latitude'] = np.nan
value=result.dropoff_latitude.max()
result.dropoff_latitude.fillna(value=value, inplace=True)
 result.loc[result.dropoff_longitude < dropoff_long_min, 'dropoff_longitude'] = np.nan
value=result.dropoff_longitude.min()
result.dropoff_longitude.fillna(value=value, inplace=True)
 result.loc[result.dropoff_longitude > dropoff_long_max, 'dropoff_longitude'] = np.nan
value=result.dropoff_longitude.max()
result.dropoff_longitude.fillna(value=value, inplace=True)
 print(result.describe() )<categorify>
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation == True:
x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate = False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate:
x = BatchActivate(x)
 return x
AVG_EARTH_RADIUS = 6371
def haversine(df, miles=True):

 lat1, lng1, lat2, lng2 =(df.pickup_latitude[:],
df.pickup_longitude[:],
df.dropoff_latitude[:],
df.dropoff_longitude[:])
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat*0.5)**2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng*0.5)**2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
if miles:
df['trip_distance'] = h * 0.621371
df['trip_distance'] = df.trip_distance.round(2)
 print(df.trip_distance.describe())
 return
else:
df['trip_distance'] = h
df['trip_distance'] = df.trip_distance.round(2)
 print(df.trip_distance.describe())
 return<compute_test_metric>
def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
conv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1, True)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2, True)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4, True)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8, True)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16, True)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8, True)
 deconv3 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4, True)
 deconv2 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2, True)
 deconv1 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding="same", activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer
haversine(result, miles=True )<prepare_x_and_y>
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred >0], tf.float64 )
def arrays_bearing(df):

 lats1, lngs1, lats2, lngs2 =(df['pickup_latitude'][:],
df['pickup_longitude'][:],
df['dropoff_latitude'][:],
df['dropoff_longitude'][:])
 lats1_rads = np.radians(lats1)
 lats2_rads = np.radians(lats2)
 lngs_delta_rads = np.radians(lngs2 - lngs1)
 y = np.sin(lngs_delta_rads)* np.cos(lats2_rads)
 x = np.cos(lats1_rads)* np.sin(lats2_rads)- \
np.sin(lats1_rads)* np.cos(lats2_rads)* np.cos(lngs_delta_rads)
 df['bearing'] = np.degrees(np.arctan2(y, x))
df['bearing'] = df.bearing.round(0)
 print(df.bearing.describe())
 return<drop_column>
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss
result.drop('dropoff_datetime', axis=1, inplace=True )<data_type_conversions>
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16,0.5)
 model1 = Model(input_layer, output_layer)
 c = optimizers.adam(lr = 0.01)
 model1.compile(loss="binary_crossentropy", optimizer=c, metrics=[my_iou_metric])
result['pickup_datetime'] = pd.to_datetime(result.pickup_datetime )<feature_engineering>
model_checkpoint = ModelCheckpoint(save_model_name,monitor='my_iou_metric',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='my_iou_metric', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 epochs = 40
batch_size = 32
history = model1.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr],
verbose=2 )
result['days_in_month'] = result['pickup_datetime'][:].dt.days_in_month<feature_engineering>
model1 = load_model(save_model_name,custom_objects={'my_iou_metric': my_iou_metric})
 input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
 c = optimizers.adam(lr = 0.01)
 model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
result['weekday'] = result['pickup_datetime'].dt.weekday<feature_engineering>
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 epochs = 40
batch_size = 32
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr,early_stopping],
verbose=2 )
result['hour'] = result['pickup_datetime'][:].dt.hour<feature_engineering>
model = load_model(save_model_name,custom_objects={'my_iou_metric_2': my_iou_metric_2,
'lovasz_loss': lovasz_loss})
result['minute'] = result['pickup_datetime'][:].dt.minute<feature_engineering>
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2
result['month'] = result['pickup_datetime'][:].dt.month<drop_column>
preds_valid = predict_result(model,x_valid,img_size_target )
result.drop(['pickup_datetime', 'store_and_fwd_flag', 'id'], axis=1, inplace=True )<drop_column>
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )
test.drop('trip_duration', axis=1, inplace=True )<feature_engineering>

 def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
train.trip_duration =(train['trip_duration']+1 ).apply(np.log)
 trip_duration = train.trip_duration.values<drop_column>
preds_test = predict_result(model,x_test,img_size_target )
train.drop('trip_duration', axis=1, inplace=True )<prepare_x_and_y>
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )
<import_modules><EOS> 
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )