Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<save_to_csv>
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import seaborn as sns
import xgboost as xgb
from catboost import CatBoostRegressor, Pool
from catboost import cv as catboost_cv
from IPython.display import display
from sklearn.cluster import MiniBatchKMeans
from sklearn.model_selection import train_test_split, RandomizedSearchCV
from sklearn.metrics import mean_squared_error
df_sub = pd.DataFrame(torch.softmax(preds, -1 ).cpu().numpy().clip(1e-3, 1-1e-3), columns=["A", "B", "NEITHER"])
 df_sub["ID"] = df_test.ID
df_sub.to_csv("submission.csv", index=False)
 df_sub.head()<load_pretrained>
%matplotlib inline
!wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip
with zipfile.ZipFile("uncased_L-12_H-768_A-12.zip","r")as zip_ref:
zip_ref.extractall()
!ls 'uncased_L-12_H-768_A-12'<load_from_url>
DO_XGBOOST = True
DO_CATBOOST = not DO_XGBOOST
!wget https://raw.githubusercontent.com/google-research/bert/master/modeling.py
!wget https://raw.githubusercontent.com/google-research/bert/master/extract_features.py
!wget https://raw.githubusercontent.com/google-research/bert/master/tokenization.py<load_pretrained>
train_raw = pd.read_csv('.. /input/train/train.csv', index_col='id',
parse_dates=['pickup_datetime', 'dropoff_datetime'])
 test_raw = pd.read_csv('.. /input/test/test.csv', index_col='id',
parse_dates=['pickup_datetime'] )
nlp = spacy.load('en_core_web_lg' )<load_from_csv>
print('NaN or empty data(train): {}'.format(train_raw.isnull().values.any()))
print('NaN or empty data(test): {}'.format(test_raw.isnull().values.any()))
test_df = pd.read_table('.. /input/gap-coreference/gap-development.tsv')
 train_df = pd.read_table('.. /input/gap-coreference/gap-test.tsv')
 val_df = pd.read_table('.. /input/gap-coreference/gap-validation.tsv')
 test_df.head()<save_to_csv>
train_set = train_raw.copy(deep=True ).drop('dropoff_datetime', axis=1)
 test_set = test_raw.copy(deep=True )
test_dist_df = extract_dist_features(test_df)
 test_dist_df.to_csv('test_dist_df.csv', index=False)
 val_dist_df = extract_dist_features(val_df)
 val_dist_df.to_csv('val_dist_df.csv', index=False)
 train_dist_df = extract_dist_features(train_df)
 train_dist_df.to_csv('train_dist_df.csv', index=False )<save_to_csv>
checkpoints = {
'pre_engineering': {
'train': train_set.copy(deep=True),
'test': test_set.copy(deep=True)
 }
}
def count_char(text, offset):
count = 0
for pos in range(offset):
if text[pos] != " ": count +=1
return count
def candidate_length(candidate):
count = 0
for i in range(len(candidate)) :
if candidate[i] != " ": count += 1
return count
def count_token_length_special(token):
count = 0
special_token = ["
for i in range(len(token)) :
if token[i] not in special_token: count+=1
return count
def embed_by_bert(df):
text = df['Text']
text.to_csv('input.txt', index=False, header=False)
 os.system("python3 extract_features.py \
--input_file=input.txt \
--output_file=output.jsonl \
--vocab_file=uncased_L-12_H-768_A-12/vocab.txt \
--bert_config_file=uncased_L-12_H-768_A-12/bert_config.json \
--init_checkpoint=uncased_L-12_H-768_A-12/bert_model.ckpt \
--layers=-1 \
--max_seq_length=256 \
--batch_size=8")
 bert_output = pd.read_json("output.jsonl", lines = True)
 bert_output.head()
os.system("rm input.txt")
 os.system("rm output.jsonl")
 index = df.index
columns = ["emb_A", "emb_B", "emb_P", "label"]
emb = pd.DataFrame(index = index, columns = columns)
 emb.index.name = "ID"
for i in tqdm(range(len(text))):
features = bert_output.loc[i, "features"]
P_char_start = count_char(df.loc[i, 'Text'], df.loc[i, 'Pronoun-offset'])
 A_char_start = count_char(df.loc[i, 'Text'], df.loc[i, 'A-offset'])
 B_char_start = count_char(df.loc[i, 'Text'], df.loc[i, 'B-offset'])
 A_length = candidate_length(df.loc[i, 'A'])
 B_length = candidate_length(df.loc[i, 'B'])
 emb_A = np.zeros(768)
 emb_B = np.zeros(768)
 emb_P = np.zeros(768)
 char_count = 0
cnt_A, cnt_B = 0, 0
for j in range(2, len(features)) :
token = features[j]["token"]
token_length = count_token_length_special(token)
 if char_count == P_char_start:
emb_P += np.asarray(features[j]["layers"][0]['values'])
 if char_count in range(A_char_start, A_char_start+A_length):
emb_A += np.asarray(features[j]["layers"][0]['values'])
 cnt_A += 1
if char_count in range(B_char_start, B_char_start+B_length):
emb_B += np.asarray(features[j]["layers"][0]['values'])
 cnt_B += 1
char_count += token_length
emb_A /= cnt_A
emb_B /= cnt_B
label = "Neither"
if(df.loc[i,"A-coref"] == True):
label = "A"
if(df.loc[i,"B-coref"] == True):
label = "B"
emb.iloc[i] = [emb_A, emb_B, emb_P, label]
return emb<categorify>
def add_checkpoint(key):
d = { 'train': train_set.copy(deep=True), 'test': test_set.copy(deep=True)}
checkpoints.update({
key: d
} )
test_emb = embed_by_bert(test_df)
 test_emb.to_json("contextual_embeddings_gap_test.json", orient = 'columns')
 validation_emb = embed_by_bert(val_df)
 validation_emb.to_json("contextual_embeddings_gap_validation.json", orient = 'columns')
 train_emb = embed_by_bert(train_df)
 train_emb.to_json("contextual_embeddings_gap_train.json", orient = 'columns' )<set_options>
def restore_checkpoint(key):
return checkpoints[key]['train'].copy(deep=True), checkpoints[key]['test'].copy(deep=True )
class End2End_NCR() :
def __init__(self, word_input_shape, dist_shape, embed_dim=20):
self.word_input_shape = word_input_shape
self.dist_shape = dist_shape
self.embed_dim = embed_dim
self.buckets = [1, 2, 3, 4, 5, 8, 16, 32, 64]
self.hidden_dim = 150
def build(self):
A, B, P = Input(( self.word_input_shape,)) , Input(( self.word_input_shape,)) , Input(( self.word_input_shape,))
dist1, dist2 = Input(( self.dist_shape,)) , Input(( self.dist_shape,))
inputs = [A, B, P]
dist_inputs = [dist1, dist2]
self.dist_embed = Embedding(len(self.buckets)+1, self.embed_dim)
 self.ffnn = Sequential([Dense(self.hidden_dim, use_bias=True),
Activation('relu'),
Dropout(rate=0.2, seed = 7),
Dense(1, activation='linear')])
 dist_embeds = [self.dist_embed(dist)for dist in dist_inputs]
dist_embeds = [Flatten()(dist_embed)for dist_embed in dist_embeds]
PA = Multiply()([inputs[0], inputs[2]])
 PB = Multiply()([inputs[1], inputs[2]])
 PA = Concatenate(axis=-1 )([P, A, PA, dist_embeds[0]])
 PB = Concatenate(axis=-1 )([P, B, PB, dist_embeds[1]])
 PA_score = self.ffnn(PA)
 PB_score = self.ffnn(PB)
 score_e = Lambda(lambda x: K.zeros_like(x))(PB_score)
 output = Concatenate(axis=-1 )([PA_score, PB_score, score_e])
 output = Activation('softmax' )(output)
 model = Model(inputs+dist_inputs, output)
 return model 
<categorify>
def cat_encode(*dfs):
for df in dfs:
for col in df.select_dtypes('object' ).columns:
df[col] = df[col].astype('category' ).cat.codes
def create_input(embed_df, dist_df):
assert len(embed_df)== len(dist_df)
 all_P, all_A, all_B = [] ,[] ,[]
all_label = []
all_dist_PA, all_dist_PB = [], []
url_A, url_B = [], []
for i in tqdm(range(len(embed_df))):
all_P.append(embed_df.loc[i, "emb_P"])
 all_A.append(embed_df.loc[i, "emb_A"])
 all_B.append(embed_df.loc[i, "emb_B"])
 all_dist_PA.append(dist_df.loc[i, "D_PA"])
 all_dist_PB.append(dist_df.loc[i, "D_PB"])
 if dist_df.loc[i, "IN_URL"] == 0:
url_A.append(1)
 url_B.append(0)
 elif dist_df.loc[i, "IN_URL"] == 1:
url_A.append(0)
 url_B.append(1)
 else:
url_A.append(0)
 url_B.append(0)
 label = embed_df.loc[i, "label"]
if label == "A":
all_label.append(0)
 elif label == "B":
all_label.append(1)
 else:
all_label.append(2)
 return [np.asarray(all_A), np.asarray(all_B), np.asarray(all_P),
np.expand_dims(np.asarray(all_dist_PA),axis=1),
np.expand_dims(np.asarray(all_dist_PB),axis=1)],all_label<concatenate>
cat_encode(train_set, test_set )
new_emb_df = pd.concat([train_emb, validation_emb])
 new_emb_df = new_emb_df.reset_index(drop=True)
 new_dist_df = pd.concat([train_dist_df, val_dist_df])
 new_dist_df = new_dist_df.reset_index(drop=True)
 new_emb_df.head()<prepare_x_and_y>
def extract_date_columns(*dfs):
for df in dfs:
df['pickup_datetime_month'] = df.pickup_datetime.dt.month
df['pickup_datetime_hour'] = df.pickup_datetime.dt.hour
df['pickup_datetime_minute'] = df.pickup_datetime.dt.minute
df['pickup_datetime_dow'] = df.pickup_datetime.dt.dayofweek
X_train, y_train = create_input(new_emb_df, new_dist_df)
 X_test, y_test = create_input(test_emb, test_dist_df )<choose_model_class>
extract_date_columns(train_set, test_set )
model = End2End_NCR(word_input_shape=X_train[0].shape[1], dist_shape=X_train[3].shape[1] ).build()
model.summary()<train_on_grid>
def _haversine_distance(lat_a, long_a, lat_b, long_b):

 del_lat_rad = lat_b - lat_a
del_long_rad = long_b - long_a
lat_a, long_a, lat_b, long_b, del_lat_rad, del_long_rad = \
map(np.radians, [lat_a, long_a, lat_b, long_b, del_lat_rad, del_long_rad])
 def _a(del_lat, del_long, lat_a, lat_b):
return(
(np.sin(del_lat / 2)** 2)+ 
(np.cos(lat_a)* np.cos(lat_b)*
np.sin(del_long / 2)** 2)
)
 def _c(a):
return 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a)) ;
R = 6371e3
a = _a(del_lat_rad, del_long_rad, lat_a, lat_b)
 c = _c(a)
 return R * c
def add_manhattan_haversine_distance(*dfs):

 for df in dfs:
inbounds = df.loc[(df['pickup_longitude'] <= -73.94)&
df['dropoff_longitude'] <= -73.94]
inbounds_idx = inbounds.index
oob = df.loc[~df.index.isin(inbounds_idx), :]
oob_idx = oob.index
ib_lat_a = inbounds['pickup_latitude']
ib_lat_b = inbounds['dropoff_latitude']
ib_long_a = inbounds['pickup_longitude']
ib_long_b = inbounds['dropoff_longitude']
oob_lat_a = oob['pickup_latitude']
oob_lat_b = oob['dropoff_latitude']
oob_long_a = oob['pickup_longitude']
oob_long_b = oob['dropoff_longitude']
df['city_distance'] = pd.Series(np.zeros(shape=(len(df.index))), index=df.index)
 df.loc[inbounds_idx, 'city_distance'] = _haversine_distance(ib_lat_a, ib_long_a,
ib_lat_a, ib_long_b)+ \
_haversine_distance(ib_lat_a, ib_long_a,
ib_lat_b, ib_long_a)
 df.loc[oob_idx, 'city_distance'] = _haversine_distance(oob_lat_a, oob_long_a,
oob_lat_b, oob_long_b)
min_loss = 1.0
best_model = 0
n_fold = 5
kfold = KFold(n_splits=n_fold, shuffle=True, random_state=3)
 for fold_n,(train_index, valid_index)in enumerate(kfold.split(X_train[0])) :
X_tr = [inputs[train_index] for inputs in X_train]
X_val = [inputs[valid_index] for inputs in X_train]
y_tr = np.asarray(y_train)[train_index]
y_val = np.asarray(y_train)[valid_index]
model = End2End_NCR(word_input_shape=X_train[0].shape[1], dist_shape=X_train[3].shape[1] ).build()
model.compile(optimizer=optimizers.Adam(lr=0.001), loss="sparse_categorical_crossentropy")
 file_path = "best_model_{}.hdf5".format(fold_n+1)
 check_point = callbacks.ModelCheckpoint(file_path, monitor = "val_loss", verbose = 0, save_best_only = True, mode = "min")
 early_stop = callbacks.EarlyStopping(monitor = "val_loss", mode = "min", patience=100)
 hist = model.fit(X_tr, y_tr, batch_size=128, epochs=1000, validation_data=(X_val, y_val), verbose=0,
shuffle=True, callbacks = [check_point, early_stop])
 if min(hist.history['val_loss'])< min_loss:
min_loss = min(hist.history['val_loss'])
 best_model = fold_n + 1<drop_column>
add_manhattan_haversine_distance(train_set, test_set )
del model<feature_engineering>
train_set.drop('pickup_datetime', axis=1, inplace=True)
 test_set.drop('pickup_datetime', axis=1, inplace=True)
 add_checkpoint('pre_modeling' )
model = End2End_NCR(word_input_shape=X_train[0].shape[1], dist_shape=X_train[3].shape[1] ).build()
model.load_weights("./best_model_{}.hdf5".format(best_model))
pred = model.predict(x = X_test, verbose = 0)
 sub_df_path = os.path.join('.. /input/gendered-pronoun-resolution/', 'sample_submission_stage_1.csv')
 sub_df = pd.read_csv(sub_df_path)
 sub_df.loc[:, 'A'] = pd.Series(pred[:, 0])
 sub_df.loc[:, 'B'] = pd.Series(pred[:, 1])
 sub_df.loc[:, 'NEITHER'] = pd.Series(pred[:, 2])
 sub_df.head(20 )<compute_test_metric>
X = train_set.drop('trip_duration', axis=1)
 y = np.log1p(train_set['trip_duration'] )
y_one_hot = np.zeros(( 2000, 3))
for i in range(len(y_test)) :
y_one_hot[i, y_test[i]] = 1
log_loss(y_one_hot, pred )<save_to_csv>
X_train, X_validate, y_train, y_validate = train_test_split(X, y )
sub_df.to_csv("submission.csv", index=False )<load_from_csv>
xgb_train = xgb.DMatrix(X_train, label=y_train)
 xgb_validate = xgb.DMatrix(X_validate, label=y_validate)
 evallist = [(xgb_train, 'training-set'),(xgb_validate, 'validation-set')]
xgb_params = {
'n_estimators': 250,
'eta': 0.2,
'subsample': 0.6,
'max_depth': 10,
'colsample_bytree': 0.75,
'min_child_weight': 25,
'objective': 'reg:linear',
'booster': 'gbtree',
'nthread': -1,
'silent': 1,
'eval_metric': 'rmse'
}
nlp = spacy.load('en_core_web_sm')
 test = pd.read_csv('.. /input/test_stage_2.tsv', delimiter='\t' ).rename(columns={'A': 'A_Noun', 'B': 'B_Noun'})
 sub = pd.read_csv('.. /input/sample_submission_stage_2.csv')
 test.shape, sub.shape<load_from_csv>
if DO_XGBOOST:
xgb_regressor = xgb.train(xgb_params, xgb_train, 100, evallist,
early_stopping_rounds=100, maximize=False,
verbose_eval=10 )
gh_test = pd.read_csv("https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-test.tsv", delimiter='\t')
 gh_train = pd.read_csv("https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-development.tsv", delimiter='\t')
 gh_valid = pd.read_csv("https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-validation.tsv", delimiter='\t')
 train = pd.concat(( gh_test, gh_train, gh_valid)).rename(columns={'A': 'A_Noun', 'B': 'B_Noun'} ).reset_index(drop=True)
 train.shape<feature_engineering>
cat_features = [0, 6, 12, 13]
train_pool = Pool(X_train, label=y_train, cat_features=cat_features)
 validation_pool = Pool(X_validate, label=y_validate, cat_features=cat_features)
 cat_regressor = CatBoostRegressor(
 loss_function='RMSE',
eval_metric='RMSE',
iterations=250,
learning_rate=0.2,
depth=10
)
def name_replace(s, r1, r2):
s = str(s ).replace(r1,r2)
 for r3 in r1.split(' '):
s = str(s ).replace(r3,r2)
 return s
def get_features(df):
df['section_min'] = df[['Pronoun-offset', 'A-offset', 'B-offset']].min(axis=1)
 df['Pronoun-offset2'] = df['Pronoun-offset'] + df['Pronoun'].map(len)
 df['A-offset2'] = df['A-offset'] + df['A_Noun'].map(len)
 df['B-offset2'] = df['B-offset'] + df['B_Noun'].map(len)
 df['A-dist_abs'] =(df['Pronoun-offset'] - df['A-offset'] ).abs()
df['B-dist_abs'] =(df['Pronoun-offset'] - df['B-offset'] ).abs()
df['A-dist'] =(df['Pronoun-offset'] - df['A-offset'])
 df['B-dist'] =(df['Pronoun-offset'] - df['B-offset'])
 df['section_max'] = df[['Pronoun-offset2', 'A-offset2', 'B-offset2']].max(axis=1)
 df['A_max'] =(df['A-offset2'] == df['section_max'] ).astype(int)
 df['A_min'] =(df['A-offset2'] == df['section_min'] ).astype(int)
 df['B_max'] =(df['B-offset2'] == df['section_max'] ).astype(int)
 df['B_min'] =(df['B-offset2'] == df['section_min'] ).astype(int)
 df['wc'] = df.apply(lambda r: len(str(r['Text'][r['section_min']: r['section_max']] ).split(' ')) , axis=1)
 df['Text'] = df.apply(lambda r: name_replace(r['Text'], r['A_Noun'], 'subjectone'), axis=1)
 df['Text'] = df.apply(lambda r: name_replace(r['Text'], r['B_Noun'], 'subjecttwo'), axis=1)
 return(df)
 train = get_features(train)
 test = get_features(test )<feature_engineering>
if DO_CATBOOST:
cat_regressor.fit(train_pool, eval_set=validation_pool )
<data_type_conversions><EOS> 
def create_xgb_submission() :
preds = xgb_regressor.predict(xgb.DMatrix(test_set))
preds_df = pd.DataFrame(np.expm1(preds), index=test_set.index)
 preds_df.columns = ['trip_duration']
preds_df.to_csv('submission.csv')
 display(pd.read_csv('submission.csv' ).head())
 def create_catboost_submission() :
pool = Pool(test_set, cat_features=cat_features)
 preds = cat_regressor.predict(pool)
 preds_df = pd.DataFrame(np.expm1(preds), index=test_set.index)
 preds_df.columns = ['trip_duration']
preds_df.to_csv('submission.csv')
 display(pd.read_csv('submission.csv' ).head())
 if DO_XGBOOST:
create_xgb_submission()
if DO_CATBOOST:
create_catboost_submission()