Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<feature_engineering>
plt.style.use('seaborn-white')
 sns.set_style("white")
 t_start = time.time()
train['dist'] = np.sqrt(np.square(train['pickup_longitude'] - train['dropoff_longitude'])+ np.square(train['pickup_latitude'] - train['dropoff_latitude']))
test['dist'] = np.sqrt(np.square(test['pickup_longitude'] - test['dropoff_longitude'])+ np.square(test['pickup_latitude'] - test['dropoff_latitude']))
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'])
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'])
 train['month'] = train['pickup_datetime'].dt.month
test['month'] = test['pickup_datetime'].dt.month
train['day'] = train['pickup_datetime'].dt.dayofweek
test['day'] = test['pickup_datetime'].dt.dayofweek
train['hour'] = train['pickup_datetime'].dt.hour
test['hour'] = test['pickup_datetime'].dt.hour<filter>
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
train = train.loc[train['hour']>=8]<prepare_x_and_y>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 print(train_df.shape)
 print(train_df.head)
 train_df = train_df.join(depths_df)
 print(train_df.shape)
 print(train_df.head)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
print("test_df")
 print(test_df.head )
y = train["log_trip_duration"]
X_train = train[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","dist"]]
X_test = test[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","dist"]]<split>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx),grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
xtrain, xvalid, ytrain, yvalid = train_test_split(X_train,y, test_size=0.2, random_state=42)
 xtrain.shape, xvalid.shape, xtrain.shape, yvalid.shape<create_dataframe>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
dtrain = lgb.Dataset(X_train, y)
 lgb_params = {
'learning_rate': 0.1,
'max_depth': 25,
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'num_leaves': 1000,
'objective': 'regression',
'max_bin': 1000}<compute_train_metric>
train_df["coverage"] = train_df.masks.map(np.sum)/ 3/pow(img_size_ori, 2 )
resultsCV = lgb.cv(lgb_params,dtrain,num_boost_round=100,nfold=3,metrics='mae',early_stopping_rounds=10,stratified=False)
 print('best score :', resultsCV['l1-mean'][-1] )<train_model>
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )
model_lgb = lgb.train(lgb_params,
dtrain,
num_boost_round=1200 )<predict_on_test>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class )
pred_test = np.exp(model_lgb.predict(X_test))
pred_test 
<load_from_csv>
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation == True:
x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate = False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate:
x = BatchActivate(x)
 return x
submit = pd.read_csv('.. /input/sample_submission.csv' )<feature_engineering>
def build_model(input_layer, start_neurons):
conv1 = Conv2D(start_neurons * 1 ,(3,3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1,True)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(0.25 )(pool1)
 conv2 = Conv2D(start_neurons * 2 ,(3,3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2,True)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(0.5 )(pool2)
 conv3 = Conv2D(start_neurons * 4 ,(3,3), activation="relu", padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4,True)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(0.5 )(pool3)
 conv4 = Conv2D(start_neurons * 8 ,(3,3), activation="relu", padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8,True)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(0.5 )(pool4)
 convm = Conv2D(start_neurons * 16 ,(3,3), activation="relu", padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16,True)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(0.5 )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3,3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8, True)
 deconv3 = Conv2DTranspose(start_neurons * 4 ,(3, 3), strides=(2, 2), padding="same" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(0.5 )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3,3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4, True)
 deconv2 = Conv2DTranspose(start_neurons * 2 ,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(0.5 )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3,3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2, True)
 deconv1 = Conv2DTranspose(start_neurons * 1 ,(3, 3), strides=(2, 2), padding="same" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(0.5 )(uconv1)
 uconv1 = Conv2D(start_neurons * 1 *4,(3, 3), activation="relu", padding="same" )(uconv1)
 uconv1 = Conv2D(start_neurons * 1 *4,(3, 3), activation="relu", padding="same" )(uconv1)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv1)
 return output_layer
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 32 )
submit['trip_duration'] = pred_test
submit.head(20 )<create_dataframe>
model = Model(input_layer, output_layer)
 model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
 model.summary()
submit_file = pd.DataFrame({"id": test.id, "trip_duration": pred_test} )<save_to_csv>
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 x_train = np.append(x_train, [np.flipud(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.flipud(x)for x in y_train], axis=0)
 x_valid = np.append(x_valid, [np.flipud(x)for x in x_valid], axis=0)
 y_valid = np.append(y_valid, [np.flipud(x)for x in y_valid], axis=0)
 x_valid = np.append(x_valid, [np.flipud(x)for x in x_valid], axis=0)
 y_valid = np.append(y_valid, [np.flipud(x)for x in y_valid], axis=0 )
submit_file.to_csv('submission.csv', index=False )<set_options>
early_stopping = EarlyStopping(patience=10, verbose=1)
 model_checkpoint = ModelCheckpoint("./keras.model", save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(factor=0.1, patience=5, min_lr=0.00001, verbose=1)
 epochs = 200
batch_size = 32
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[early_stopping, model_checkpoint, reduce_lr] )
%matplotlib inline<load_from_csv>
preds_valid = model.predict(x_valid ).reshape(-1, img_size_target, img_size_target)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 y_valid_ori = np.array([train_df.loc[idx].masks for idx in ids_valid] )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sample = pd.read_csv('.. /input/sample_submission.csv')
<count_missing_values>
thresholds = np.linspace(0, 1, 50)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )
train.isna().sum()<feature_engineering>
threshold_best_index = np.argmax(ious[9:-10])+ 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
train['trip_duration'] = np.log(train['trip_duration'].values )<count_values>
def RLenc(img, order='F', format=True):

 bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
 runs = []
r = 0
pos = 1
for c in bytes:
if(c == 0):
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
 return z[:-1]
else:
return runs
train['passenger_count'].value_counts()<compute_test_metric>
preds_test = model.predict(x_test )
def haversine(lat1, lon1, lat2, lon2):
R = 6372800
phi1, phi2 = math.radians(lat1), math.radians(lat2)
 dphi = math.radians(lat2 - lat1)
 dlambda = math.radians(lon2 - lon1)
 a = math.sin(dphi/2)**2 + \
math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2
return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a))
train['dist_long'] = train['pickup_longitude'] - train['dropoff_longitude']
test['dist_long'] = test['pickup_longitude'] - test['dropoff_longitude']
train['dist_lat'] = train['pickup_latitude'] - train['dropoff_latitude']
test['dist_lat'] = test['pickup_latitude'] - test['dropoff_latitude']
train['dist'] = np.sqrt(np.square(train['dist_long'])+ np.square(train['dist_lat']))
test['dist'] = np.sqrt(np.square(test['dist_long'])+ np.square(test['dist_lat']))<feature_engineering>
pred_dict = {idx: RLenc(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'])
 train['dropoff_datetime'] = pd.to_datetime(train['dropoff_datetime'])
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'])
 train['hour'] = train.pickup_datetime.dt.hour
train['day'] = train.pickup_datetime.dt.dayofweek
train['month'] = train.pickup_datetime.dt.month
test['hour'] = test.pickup_datetime.dt.hour
test['day'] = test.pickup_datetime.dt.dayofweek
test['month'] = test.pickup_datetime.dt.month<count_missing_values>
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv' )
<count_missing_values><EOS> 
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )