Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<load_from_csv>
plt.style.use('seaborn-white')
 sns.set_style("white")
 %matplotlib inline
t_start = time.time()
df_test = pd.read_csv(TESTPATH)
 df_test.head()<count_duplicates>
version = 5
basic_name = f'Unet_resnet_v{version}'
save_model_name = basic_name + '.model'
submission_file = basic_name + '.csv'
print(save_model_name)
 print(submission_file )
df.duplicated().sum()<count_duplicates>
img_size_ori = 101
img_size_target = 101
df = df.drop_duplicates()
df.duplicated().sum() 
<count_missing_values>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
len(train_df )
df.isna().sum()<define_variables>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) for idx in tqdm_notebook(train_df.index)]
CAT_VARS = ['store_and_fwd_flag']<data_type_conversions>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
for col in CAT_VARS:
df[col] = df[col].astype('category' ).cat.codes
df.head()<data_type_conversions>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
for col in CAT_VARS:
df_test[col] = df_test[col].astype('category' ).cat.codes
df_test.head()<import_modules>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state= 1618 )
from sklearn.decomposition import PCA<categorify>
def build_model() :
def conv(f, k=3, act='relu'):
return Conv2D(f,(k, k), activation=act, kernel_initializer='he_normal', padding='same')
 def _res_conv(inputs, f, k=3):
channels = int(inputs.shape[-1])
 cs = inputs
cs = BatchNormalization()(cs)
 cs = Activation('relu' )(cs)
 cs = conv(f, 3, act=None )(cs)
 cs = BatchNormalization()(cs)
 cs = Activation('relu' )(cs)
 cs = conv(f, 3, act=None )(cs)
 if f!=channels:
t1 = conv(f, 1, None )(inputs)
 else:
t1 = inputs
out = Add()([t1, cs])
 return out
def pool() :
return MaxPooling2D(( 2, 2))
def up(x, shape):
x = Lambda(lambda x: tf.image.resize_bilinear(x, shape, align_corners=True))(x)
 x = Conv2D(K.int_shape(x)[-1]//2,(2, 2), activation=None, kernel_initializer='he_normal', padding='same' )(x)
 return x
inputs = Input(shape=(img_size_ori, img_size_ori, 1))
depths = Input(shape=(1,))
preprocess1 = Lambda(lambda x: x/255.0 )(inputs)
 r = 16
rep = 4
mid_rep = 3
x = preprocess1
skip_connections = []
shapes = []
pad_mode = ['same', 'valid', 'same', 'valid']
for t in range(rep):
x = conv(r*int(2**t), 3, None )(x)
 x = _res_conv(x, r*int(2**t), 3)
 x = _res_conv(x, r*int(2**t), 3)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 skip_connections.append(x)
 shapes.append(K.int_shape(x)[1:3])
 x = pool()(x)
 x = Dropout(0.3 )(x)
 shape = K.int_shape(x)[1:3]
aux = Lambda(lambda x: x/1000.0 )(depths)
 aux = RepeatVector(np.prod(np.asarray(shape)) )(aux)
 aux = Reshape(( *shape,1))(aux)
 x = concatenate([x, aux])
 x = conv(r*int(2**rep), 3, None )(x)
 for t in range(mid_rep):
x = _res_conv(x, r*int(2**rep))
for t, s, p in zip(reversed(range(rep)) , reversed(skip_connections), reversed(shapes)) :
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = up(x, p)
 x = concatenate([s, x])
 x = Dropout(0.3 )(x)
 x = conv(r*int(2**t), 3, None )(x)
 x = _res_conv(x, r*int(2**t), 3)
 x = _res_conv(x, r*int(2**t), 3)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 outputs = Conv2D(1,(1, 1), activation=None, kernel_initializer='he_normal', padding='valid' )(x)
 return Model([inputs, depths], [outputs] )
coords = np.vstack(( df[['pickup_latitude', 'pickup_longitude']].values,
df[['dropoff_latitude', 'dropoff_longitude']].values,
df_test[['pickup_latitude', 'pickup_longitude']].values,
df_test[['dropoff_latitude', 'dropoff_longitude']].values))
pca = PCA().fit(coords)
 df['pickup_pca0'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 0]
df['pickup_pca1'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 1]
df['dropoff_pca0'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df['dropoff_pca1'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
df_test['pickup_pca0'] = pca.transform(df_test[['pickup_latitude', 'pickup_longitude']])[:, 0]
df_test['pickup_pca1'] = pca.transform(df_test[['pickup_latitude', 'pickup_longitude']])[:, 1]
df_test['dropoff_pca0'] = pca.transform(df_test[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df_test['dropoff_pca1'] = pca.transform(df_test[['dropoff_latitude', 'dropoff_longitude']])[:, 1]<data_type_conversions>
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def lb(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0], tf.float64 )
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 df['dropoff_datetime'] = pd.to_datetime(df['dropoff_datetime'])
 df_test['pickup_datetime'] = pd.to_datetime(df_test['pickup_datetime'] )<feature_engineering>
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss
df['hour'] = df.pickup_datetime.dt.hour
df['day'] = df.pickup_datetime.dt.dayofweek
df['month'] = df.pickup_datetime.dt.month
df_test['hour'] = df_test.pickup_datetime.dt.hour
df_test['day'] = df_test.pickup_datetime.dt.dayofweek
df_test['month'] = df_test.pickup_datetime.dt.month<feature_engineering>
def bce(y_true, y_pred):
return K.mean(K.binary_crossentropy(y_true, K.sigmoid(y_pred)) )
df['distance2'] = np.sqrt(( df['pickup_pca0']-df['dropoff_pca0'])**2
+(df['pickup_pca1']-df['dropoff_pca1'])**2)
 df_test['distance2'] = np.sqrt(( df_test['pickup_pca0']-df_test['dropoff_pca0'])**2
+(df_test['pickup_pca1']-df_test['dropoff_pca1'])**2 )<feature_engineering>
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 depth_train = np.append(depth_train, depth_train, axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train.shape)
 print(depth_train.shape)
 print(y_valid.shape )
df['log_trip_duration'] = np.log(df['trip_duration'])
<define_variables>
model = build_model()
c = optimizers.adam(lr = 0.01)
 model.compile(loss=bce, optimizer=c, metrics=[lb])
 model.summary()
NUM_VARS = ['pickup_pca0', 'pickup_pca1', 'dropoff_pca0', 'dropoff_pca1', 'month', 'hour', 'day', 'distance2']
TARGET = 'log_trip_duration'<define_variables>
model_checkpoint = ModelCheckpoint(save_model_name,monitor='lb',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='lb', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 epochs = 50
batch_size = 32
history = model.fit([x_train, depth_train], y_train,
validation_data=[[x_valid, depth_test], y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr],
verbose=2, shuffle=True )
num_features = NUM_VARS 
<prepare_x_and_y>
model.load_weights(save_model_name)
 c = optimizers.adam(lr = 0.01)
 model.compile(loss=lovasz_loss, optimizer=c, metrics=[lb])
X_train = df.loc[:, num_features]
y_train = df[TARGET]
X_test = df_test.loc[:, num_features]
X_train.shape, y_train.shape, X_test.shape<import_modules>
early_stopping = EarlyStopping(monitor='val_lb', mode = 'max',patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_lb',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_lb', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 epochs = 60
batch_size = 32
history = model.fit([x_train, depth_train], y_train,
validation_data=[[x_valid, depth_test], y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr,early_stopping],
verbose=2, shuffle=True )
from sklearn.ensemble import RandomForestRegressor 
<train_model>
model.load_weights(save_model_name )
training = RandomForestRegressor(n_estimators=19, min_samples_leaf=10, min_samples_split=15, max_features='auto', max_depth=80, bootstrap=True)
 training.fit(X_train, y_train )<import_modules>
def predict_result(model,x_test,depth_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict([x_test,depth_test] ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict([x_test_reflect,depth_test] ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2
from sklearn.model_selection import cross_val_score 
<compute_train_metric>
preds_valid = predict_result(model,x_valid,depth_test,img_size_target )
cv_scores = cross_val_score(training, X_train, y_train, cv=5, scoring='neg_mean_squared_log_error' )<feature_engineering>
thresholds = np.linspace(-0.5, 0.5, 100)
 ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )
for i in range(len(cv_scores)) :
cv_scores[i] = np.sqrt(abs(cv_scores[i]))
cv_scores<predict_on_test>

 def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
y_test_pred = training.predict(X_test)
 y_test_pred[:5]<save_to_csv>
preds_test = predict_result(model,x_test,x_depth_test,img_size_target )
predection = pd.DataFrame({'id': df_test.id, 'trip_duration': np.exp(y_test_pred)})
 predection.to_csv('submission.csv', index=False )<import_modules>
t1 = time.time()
pred_dict = {idx: rle_encode(preds_test[i] > threshold_best)for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
t2 = time.time()
print(f"Usedtime = {t2-t1} s" )
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler<load_from_csv>
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )
<count_missing_values><EOS> 
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )