Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<prepare_x_and_y>
plt.style.use('seaborn-white')
 sns.set_style('white')
 %matplotlib inline
t_start = time.time()
y_train = train["trip_duration"]
X_train = train[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","dist"]]
X_datatest = test[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","dist"]]<train_on_grid>
version = 5
basic_name = f'Unet_resnet_v{version}'
save_model_name = basic_name + '.model'
submission_file = basic_name + '.csv'
print(save_model_name)
 print(submission_file )
<split>
img_size_ori = 101
img_size_target = 101
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True )
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size = 0.1, random_state=42 )<train_model>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
len(train_df )
rfr = RandomForestRegressor(n_estimators=100,min_samples_leaf=10, min_samples_split=15, max_depth=80,verbose=0,max_features="auto",bootstrap=True,n_jobs=-1)
 rfr.fit(X_train, y_train )<compute_train_metric>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
cv_scores = cross_val_score(rfr, X_train, y_train, cv=5, scoring= 'neg_mean_squared_log_error' )<feature_engineering>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
for i in range(len(cv_scores)) :
cv_scores[i] = np.sqrt(abs(cv_scores[i]))
cv_scores 
<predict_on_test>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )
train_pred = rfr.predict(X_datatest)
 train_pred[:5]<prepare_output>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = \
train_test_split(train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state=555 )
my_submission = pd.DataFrame({"id": test.id, "trip_duration": np.exp(train_pred)})
 print(my_submission )<save_to_csv>
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation==True: x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate=False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate: x = BatchActivate(x)
 return x
my_submission.to_csv('submission.csv', index=False)
<set_options>
def build_model(input_layer, start_neurons, DropoutRatio=0.5):
conv1 = Conv2D(start_neurons*1,(3,3), activation=None, padding='same' )(input_layer)
 conv1 = residual_block(conv1, start_neurons*1)
 conv1 = residual_block(conv1, start_neurons*1, True)
 pool1 = MaxPooling2D(( 2,2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons*2,(3,3), activation=None, padding='same' )(pool1)
 conv2 = residual_block(conv2, start_neurons*2)
 conv2 = residual_block(conv2, start_neurons*2, True)
 pool2 = MaxPooling2D(( 2,2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons*4,(3,3), activation=None, padding='same' )(pool2)
 conv3 = residual_block(conv3, start_neurons*4)
 conv3 = residual_block(conv3, start_neurons*4, True)
 pool3 = MaxPooling2D(( 2,2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons*8,(3,3), activation=None, padding='same' )(pool3)
 conv4 = residual_block(conv4, start_neurons*8)
 conv4 = residual_block(conv4, start_neurons*8, True)
 pool4 = MaxPooling2D(( 2,2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons*16,(3,3), activation=None, padding='same' )(pool4)
 convm = residual_block(convm, start_neurons*16)
 convm = residual_block(convm, start_neurons*16, True)
 deconv4 = Conv2DTranspose(start_neurons*8,(3,3), strides=(2,2), padding='same' )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons*8,(3,3), activation=None, padding='same' )(uconv4)
 uconv4 = residual_block(uconv4, start_neurons*8)
 uconv4 = residual_block(uconv4, start_neurons*8, True)
 deconv3 = Conv2DTranspose(start_neurons*4,(3,3), strides=(2,2), padding='valid' )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons*4,(3,3), activation=None, padding='same' )(uconv3)
 uconv3 = residual_block(uconv3, start_neurons*4)
 uconv3 = residual_block(uconv3, start_neurons*4, True)
 deconv2 = Conv2DTranspose(start_neurons*2,(3,3), strides=(2,2), padding='same' )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons*2,(3,3), activation=None, padding='same' )(uconv2)
 uconv2 = residual_block(uconv2, start_neurons*2)
 uconv2 = residual_block(uconv2, start_neurons*2, True)
 deconv1 = Conv2DTranspose(start_neurons*1,(3,3), strides=(2,2), padding='valid' )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons*1,(3,3), activation=None, padding='same' )(uconv1)
 uconv1 = residual_block(uconv1, start_neurons*1)
 uconv1 = residual_block(uconv1, start_neurons*1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding='same', activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer
%matplotlib inline<load_from_csv>
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred >0], tf.float64 )
TRAIN_PATH = os.path.join('.. ', 'input', 'train.csv')
 df = pd.read_csv(TRAIN_PATH, index_col=0)
 df.head()<count_duplicates>
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.elu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss
df.duplicated().sum()<remove_duplicates>
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train.shape)
 print(y_valid.shape )
df_no_duplicates = df.drop_duplicates()
df_no_duplicates.shape[0]<count_missing_values>
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16,0.5)
 model1 = Model(input_layer, output_layer)
 c = optimizers.adam(lr = 0.005)
 model1.compile(loss="binary_crossentropy", optimizer=c, metrics=[my_iou_metric])
 model1.summary()
df_no_duplicates.isna().sum()<filter>
early_stopping = EarlyStopping(monitor='val_my_iou_metric', mode = 'max',patience=15, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name, monitor='my_iou_metric', mode='max',
save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric', mode='max', factor=0.5, patience=5,
min_lr=0.0001, verbose=1)
 epochs = 100
batch_size = 128
t_model1_start = time.time()
history = model1.fit(x_train, y_train,
validation_data = [x_valid, y_valid],
epochs = epochs,
batch_size = batch_size,
callbacks = [early_stopping, model_checkpoint, reduce_lr],
verbose = 2)
 t_model1_end = time.time()
print(f"Run time = {(t_model1_end-t_model1_start)/3600} hours" )
df_no_duplicates.loc[df_no_duplicates['trip_duration'] > 100000]<filter>
model1 = load_model(save_model_name, custom_objects={'my_iou_metric':my_iou_metric})
 input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
 c = optimizers.adam(lr=0.01)
 model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
 model.summary()
df_clean = df_no_duplicates.loc[df_no_duplicates['trip_duration'] < 100000]
df_clean.shape[0]<feature_engineering>
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=30, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.5, patience=5,
min_lr=0.00005, verbose=1)
 epochs = 120
batch_size = 64
t_model2_start = time.time()
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr,early_stopping],
verbose=2)
 t_model2_end = time.time()
print(f"Run time = {(t_model2_end-t_model2_start)/3600} hours" )
df_enhanced = df_clean.copy()
df_enhanced["delta_longitude"] = df_clean["pickup_longitude"] - df_clean["dropoff_longitude"]
df_enhanced["delta_latitude"] = df_clean["pickup_latitude"] - df_clean["dropoff_latitude"]
df_enhanced["delta_total"] = np.sqrt(np.square(df_enhanced["delta_longitude"])+ np.square(df_enhanced["delta_latitude"]))<data_type_conversions>
model = load_model(save_model_name,custom_objects={'my_iou_metric_2': my_iou_metric_2,
'lovasz_loss': lovasz_loss} )
df_enhanced["pickup_Timestamp"] = pd.to_datetime(df_clean["pickup_datetime"], format='%Y/%m/%d')
 df_enhanced["pickup_hour"] = df_enhanced["pickup_Timestamp"].dt.strftime('%-H' ).astype(int)
 df_enhanced["pickup_minute"] = df_enhanced["pickup_Timestamp"].dt.strftime('%-M' ).astype(float)
 df_enhanced["pickup_daynumber"] = df_enhanced["pickup_Timestamp"].dt.strftime('%w' ).astype(int)
 df_enhanced["pickup_month"] = df_enhanced["pickup_Timestamp"].dt.strftime('%m' ).astype(int)
 df_enhanced["pickup_weeknumber"] = df_enhanced["pickup_Timestamp"].dt.strftime('%U' ).astype(int )<prepare_x_and_y>
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2
df_features = df_enhanced[['vendor_id', 'passenger_count', 'pickup_hour', 'pickup_minute', 'pickup_daynumber', 'pickup_weeknumber', 'pickup_month', 'delta_longitude', 'delta_latitude', 'delta_total']]
df_target = np.log(df_enhanced['trip_duration'].values)
 df_features.head()<split>
preds_valid = predict_result(model,x_valid,img_size_target )
X_train, X_test, y_train, y_test = train_test_split(df_features, df_target, test_size=0.1, random_state=42)
 X_train.shape, X_test.shape, y_train.shape, y_test.shape<choose_model_class>
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )
rf = RandomForestRegressor(n_estimators=500, min_samples_leaf=10, min_samples_split=30, max_depth=20, random_state=42, n_jobs=-1 )<train_model>
def rle_encode(im):

 pixels = im.flatten(order='F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
rf.fit(X_train, y_train )<compute_test_metric>
preds_test = predict_result(model,x_test,img_size_target )
rms = sqrt(mean_squared_error(y_test, rf.predict(X_test)) )<load_from_csv>
pred_dict = {idx: rle_encode(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
TEST_PATH = os.path.join('.. ', 'input', 'test.csv')
 test = pd.read_csv(TEST_PATH, index_col=0 )<feature_engineering>
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )
<predict_on_test><EOS> 
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )