Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<train_on_grid>
import numpy as np
import pandas as pd
from math import sqrt
import os
CV_clf = GridSearchCV(estimator=clf, param_grid=param_grid, cv=5)
 CV_clf.fit(train3, pred)
 CV_clf.best_params_<train_model>
%matplotlib inline
plt.style.use('fivethirtyeight')
 palette = sns.color_palette('Paired', 10)
 seed = 39
clf1 = RandomForestClassifier(random_state=0, n_estimators=200, criterion='gini', max_features='auto', max_depth=8)
 clf1.fit(train3, pred )<predict_on_test>
TRAIN_PATH = '.. /input/train.csv'
TEST_PATH = '.. /input/test.csv'
SAMPLE_SUBMISSION_PATH = '.. /input/sample_submission.csv'
pred3 = clf1.predict(test3 )<save_to_csv>
TRAIN_DATASIZE = 6_000_000
CHUNK_SIZE = 1_000_000
train_dtype = {
'key' : 'object',
'fare_amount' : 'float32',
'pickup_datetime' : 'str',
'pickup_longitude': 'float32',
'pickup_latitude' : 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude' : 'float32',
'passenger_count' : 'int16'
}
test_dtype = {
'key' : 'object',
'pickup_datetime' : 'str',
'pickup_longitude': 'float32',
'pickup_latitude' : 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude' : 'float32',
'passenger_count' : 'int16'
}
submission_dype = {
'key' : 'object',
'fare_amount' : 'float32'
}
pred_test = pred3
output = pd.DataFrame({
'PassengerId': test_data.PassengerId,
'Survived': pred_test
})
 output.to_csv('./submission.csv', index=False )<set_options>
def read_csv(path, dtype, chunksize = None, totalsize = None):

 if chunksize != None:
data_lst = []
reader = pd.read_csv(TRAIN_PATH, dtype = dtype, chunksize = chunksize)
 index = 1
for df_data in reader:
data_lst.append(df_data)
 del df_data
print('Rows read so far {0} M'.format(index * chunksize /(1e6)))
 if index * chunksize >= totalsize:
break
index = index + 1
df_data = pd.concat(data_lst)
 del data_lst
elif totalsize == None:
df_data = pd.read_csv(path, dtype = dtype)
 else:
df_data = pd.read_csv(path, dtype = dtype, nrows= totalsize)
 print('data ' + path + ' is read')
 return df_data
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore')
 SEED = 42<load_from_csv>
%%time
train = read_csv(path = TRAIN_PATH, dtype = train_dtype, chunksize = CHUNK_SIZE, totalsize = TRAIN_DATASIZE)
 test = read_csv(path = TEST_PATH, dtype = test_dtype)
 submission = read_csv(path = SAMPLE_SUBMISSION_PATH, dtype = submission_dype )
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
 train.shape, test.shape<concatenate>
for df in [train, test]:
df.drop('key', axis = 1, inplace = True)
 train.shape, test.shape,
def concat(train, test):
return pd.concat([train, test] ).reset_index(drop = True)
 def df_divide(df):
return df[:890], df[891:].drop('Survived', axis =1 )<concatenate>
str(100 * len(train[train.passenger_count.between(1, 6)])/ len(train)) + ' %'
df_all = concat(train, test)
 a, b = df_divide(df_all )<define_variables>
train = train[train.passenger_count.between(1,6)]
dfs = [train, test]<count_missing_values>
percentile_99_73 = train.fare_amount.quantile(0.9973)
 percentile_99_73
df_all.isna().sum()<feature_engineering>
train = train[train.fare_amount.between(2.5, 60)]
df_all['Age'] = df_all.groupby(['Sex', 'Pclass'])['Age'].apply(lambda x : x.fillna(x.median()))<data_type_conversions>
str(100 * len(train[train.pickup_latitude.between(40.5, 40.9)])/ len(train)) + ' %'
df_all['Embarked'].fillna('S', inplace = True )<filter>
str(100 * len(train[train.dropoff_latitude.between(40.5, 40.9)])/ len(train)) + ' %'
df_all[df_all['Embarked'].isna() ]<filter>
str(100 * len(train[train.pickup_longitude.between(-74.3, -73)])/ len(train)) + ' %'
df_all[df_all['Fare'].isna() ]<groupby>
str(100 * len(train[train.dropoff_longitude.between(-74.3, -73)])/ len(train)) + ' %'
g = df_all.groupby(['Pclass','Parch', 'SibSp'])['Fare'].median()
g<count_values>
train = train[train.pickup_latitude.between(40.5, 40.9)]
train = train[train.dropoff_latitude.between(40.5, 40.9)]
train = train[train.pickup_longitude.between(-74.3, -73)]
train = train[train.dropoff_longitude.between(-74.3, -73)]
df_all['Cabin'].value_counts()<data_type_conversions>
train.isnull().mean().sum() , test.isnull().mean().sum()
df_all['Deck'] = df_all['Cabin'].astype(str ).apply(lambda x : x[0] if x != 'nan' else 'M' )<feature_engineering>
corr = train.corr() ['fare_amount']
corr.sort_values(ascending = False )
i = df_all[df_all['Deck'] == 'T'].index
df_all['Deck'].iloc[i] = 'A'<categorify>
train['latitude_abs'] =(train.pickup_latitude - train.dropoff_latitude ).abs()
train['longitude_abs'] =(train.pickup_longitude - train.dropoff_longitude ).abs()
test['latitude_abs'] =(test.pickup_latitude - test.dropoff_latitude ).abs()
test['longitude_abs'] =(test.pickup_longitude - test.dropoff_longitude ).abs()
df_all['Deck'].replace(['A','B','C'], 'ABC', inplace = True)
 df_all['Deck'].replace(['D', 'E'], 'DE', inplace = True)
 df_all['Deck'].replace(['F', 'G'], 'FG', inplace = True )<count_missing_values>
PI = 3.14159265359
deg_to_rad = PI/180.0
earth_rad = 6371.009
def distance_bn_coordinates(lat1, log1, lat2, log2):
delta_lat =(lat2 - lat1)*deg_to_rad
delta_log =(log2 - log1)*deg_to_rad
mean_lat = deg_to_rad *(lat2 + lat1)/2.0
dist = earth_rad * np.sqrt(
 np.power(delta_lat,2.0)+
np.power(np.cos(mean_lat)*delta_log,2.0)
)
 return dist
df_all.isna().sum()<drop_column>
def calculate_euclidean_distance(lat1, log1, lat2, log2):
lat_sq = np.power(( lat2-lat1), 2.0)
 log_sq = np.power(( log2-log1), 2.0)
 return np.sqrt(lat_sq+log_sq )
df_all.drop('Cabin', axis = 1, inplace = True )<count_duplicates>
def haversine_distance(lat1, lon1, lat2, lon2):
R = 3959.87433
dLat = radians(lat2 - lat1)
 dLon = radians(lon2 - lon1)
 lat1 = radians(lat1)
 lat2 = radians(lat2)
 a = sin(dLat/2)**2 + cos(lat1)*cos(lat2)*sin(dLon/2)**2
c = 2*asin(sqrt(a))
return R * c
a[(a>0.1)&(a<1)].dropna().drop_duplicates().rename(columns = {0:"Correlations"} ).style.background_gradient()<count_missing_values>
use_feature = True
if use_feature == True:
train['distance'] = distance_bn_coordinates(train.pickup_latitude ,train.pickup_longitude,
train.dropoff_latitude ,train.dropoff_longitude)
 test['distance'] = distance_bn_coordinates(test.pickup_latitude ,test.pickup_longitude,
test.dropoff_latitude ,test.dropoff_longitude )
df_all.isnull().sum()<feature_engineering>
use_feature = False
if use_feature == True:
train['euclidean_distance'] = calculate_euclidean_distance(train.pickup_latitude ,train.pickup_longitude,
train.dropoff_latitude ,train.dropoff_longitude)
 test['euclidean_distance'] = calculate_euclidean_distance(test.pickup_latitude ,test.pickup_longitude,
test.dropoff_latitude ,test.dropoff_longitude )
df_all['Fare'] = pd.qcut(df_all['Fare'], 13 )<feature_engineering>
def get_timefeatures(df):
pickup_datetime = df.pickup_datetime.copy()
pickup_datetime = pickup_datetime.str.slice(0, 16)
 pickup_datetime = pd.to_datetime(pickup_datetime, utc=True, format='%Y-%m-%d %H:%M')
 df['year'] = pickup_datetime.dt.year.astype('uint16')
 df['month'] = pickup_datetime.dt.month.astype('uint8')
 df['day'] = pickup_datetime.dt.day.astype('uint8')
 df['dayofweek'] = pickup_datetime.dt.dayofweek.astype('uint8')
 df['hour'] = pickup_datetime.dt.hour.astype('uint8')
 df['minute'] = pickup_datetime.dt.minute.astype('uint8')
 del pickup_datetime
return df
df_all['Age'] = pd.qcut(df_all['Age'], 10 )<feature_engineering>
train = get_timefeatures(train)
 test = get_timefeatures(test )
df_all['Family Size'] = df_all['Parch'] + df_all['SibSp'] + 1<count_values>
basic_features = ['distance', 'longitude_abs', 'latitude_abs','passenger_count']
original_features = ['pickup_longitude', 'pickup_latitude',
'dropoff_longitude', 'dropoff_latitude']
time_features = ['year', 'month', 'day', 'dayofweek', 'hour', 'minute']
a = df_all['Ticket'].value_counts()
df_all['TicketC'] = df_all['Ticket'].map(a )<feature_engineering>
train.drop('pickup_datetime', axis= 1, inplace = True)
 test.drop('pickup_datetime', axis=1, inplace = True )
df_all['Title'] = df_all['Name'].apply(lambda x : x.split(', ')[1].split('.')[0] )<feature_engineering>
test_size = 100_000 / len(train)
 X_train, X_val, y_train, y_val = train_test_split(train, train.fare_amount, \
test_size = test_size, random_state = seed, stratify = train.fare_amount_bin)
 X_train.shape, X_val.shape, y_train.shape, y_val.shape, test.shape
df_all['Married'] = 0<feature_engineering>
del train
df_all['Married'].loc[(df_all['Title'] == 'Mrs')] = 1<categorify>
def calculate_rmse(y_true, y_pred):
root_mean_squared_error = sqrt(mean_squared_error(y_true, y_pred))
return root_mean_squared_error
def submit_fare_amount(y_pred, filename):

 if isinstance(y_pred, pd.Series):
submission.update(y_pred)
 elif isinstance(y_pred, float):
submission.fare_amount = y_pred
else:
raise ValueError('y_pred is neither Pandas Series nor float')
 submission.to_csv(filename, index=False)
 print('Successfully written in ' + filename)
 display(submission.fare_amount.head(5 ).ravel())
 tuples = [('RMSE', 'Train'),('RMSE', 'CV'),('RMSE', 'Val'),('RMSE', 'Submission')]
columns = pd.MultiIndex.from_tuples(tuples)
 Metrics = pd.DataFrame(columns = columns)
 Metrics.index.name = 'Model'
Metrics
df_all['Title'] = df_all['Title'].replace(['Miss', 'Mrs','Ms', 'Mlle', 'Lady', 'Mme', 'the Countess', 'Dona'], 'Miss/Mrs/Ms')
 df_all['Title'] = df_all['Title'].replace(['Dr', 'Col', 'Major', 'Jonkheer', 'Capt', 'Sir', 'Don', 'Rev'], 'Dr/Military/Noble/Clergy' )<feature_engineering>
kfold = KFold(n_splits = 5, shuffle = True, random_state = seed )
df_all['Family'] = df_all['Name'].apply(lambda x : x.split(', ')[0] )<split>
X_train = X_train.drop(['fare_amount','fare_amount_bin'], axis=1)
 X_val = X_val.drop(['fare_amount','fare_amount_bin'], axis=1)
 X_train.shape, X_val.shape, test.shape
train = df_all[:891]
test = df_all[891:]
dfs = [train,test]<groupby>
def run_model(model, model_name, features, target, usecv = False):
print('fitting the training data...')
 model.fit(features[0], target[0])
 print('model is trained')
 print('predicting train and test data')
 pred =[]
for f in features:
pred.append(model.predict(f))
if usecv == True:
print('using cross validation')
 cvres = cross_val_score(model, features[0],target[0], \
scoring = 'neg_mean_squared_error', cv = kfold, n_jobs = -1)
 cv_mean = np.mean(np.sqrt(np.abs(cvres)))
 cv_std = np.std(np.sqrt(np.abs(cvres)))
 else:
print('No cross validation')
 cv_mean =0
cv_std =0
rmse =[]
for t, p in zip(target, pred):
rmse.append(calculate_rmse(t, p))
print('submitting.. ')
 submission_pred = pd.Series(pred[2], index = submission.index)
 submission_pred.name = submission.fare_amount.name
print(submission_pred.head())
 submit_fare_amount(submission_pred, model_name + '.csv')
 print('updating metrics.. ')
 Metrics.loc[model_name] = [rmse[0], str(np.round(cv_mean,2)) +' +/- '+ str(np.round(cv_std,2)) ,rmse[1], '']
display(Metrics)
 return pred
fam_survival_rate = train.groupby('Family')['Survived', 'Family', 'Family Size'].median()
ticket_survival_rate = train.groupby('Ticket')['Survived', 'Ticket','TicketC'].median()<feature_engineering>
X = pd.concat([X_train, y_train], axis = 1)
 X_sample = X.sample(frac = 0.1 , random_state = seed)
 y_train_sample = X_sample.fare_amount
X_train_sample = X_sample.drop('fare_amount', axis=1)
 del X
X_train_sample.shape, y_train_sample.shape
family_rates = {}
ticket_rates = {}
for i in range(len(fam_survival_rate)) :
if fam_survival_rate.index[i] in non_unique_fams and fam_survival_rate.iloc[i, 1] > 1:
family_rates[fam_survival_rate.index[i]] = fam_survival_rate.iloc[i,0]
for i in range(len(ticket_survival_rate)) :
if ticket_survival_rate.index[i] in non_unique_tickets and ticket_survival_rate.iloc[i, 1]>1:
ticket_rates[ticket_survival_rate.index[i]] = ticket_survival_rate.iloc[i,0]<groupby>
%%time
execute_model = False
if execute_model:
linReg = LinearRegression(n_jobs = -1)
 model_name = 'Linear_Regression'
features = [X_train_sample[basic_features],X_val[basic_features],test[basic_features]]
target = [y_train_sample, y_val]
pred = run_model(linReg, model_name, features, target )
mean_survival_rate = train['Survived'].mean()
train_fam_survival = []
train_fam_survival_NA = []
test_fam_survival = []
test_fam_survival_NA = []
for i in range(len(train)) :
if(train['Family'].iloc[i] in family_rates):
train_fam_survival.append(family_rates[train['Family'].iloc[i]])
 train_fam_survival_NA.append(1)
 else:
train_fam_survival.append(mean_survival_rate)
 train_fam_survival_NA.append(0)
 for i in range(len(test)) :
if(test['Family'].iloc[i] in family_rates):
test_fam_survival.append(family_rates[test['Family'].iloc[i]])
 test_fam_survival_NA.append(1)
 else:
test_fam_survival.append(mean_survival_rate)
 test_fam_survival_NA.append(0)
<define_variables>
%%time
execute_model = False
if execute_model:
linReg = LinearRegression(n_jobs = -1)
 model_name = 'Linear_Regression_with_time_features'
features = [X_train_sample[basic_features+time_features],X_val[basic_features+time_features],test[basic_features+time_features]]
target = [y_train_sample, y_val]
pred = run_model(linReg, model_name, features, target )
train_ticket_survival = []
train_ticket_survival_NA = []
test_ticket_survival = []
test_ticket_survival_NA = []
for i in range(len(train)) :
if(train['Ticket'].iloc[i] in ticket_rates):
train_ticket_survival.append(ticket_rates[train['Ticket'].iloc[i]])
 train_ticket_survival_NA.append(1)
 else:
train_ticket_survival.append(mean_survival_rate)
 train_ticket_survival_NA.append(0)
 for i in range(len(test)) :
if(test['Ticket'].iloc[i] in ticket_rates):
test_ticket_survival.append(ticket_rates[test['Ticket'].iloc[i]])
 test_ticket_survival_NA.append(1)
 else:
test_ticket_survival.append(mean_survival_rate)
 test_ticket_survival_NA.append(0 )<feature_engineering>
%%time
execute_model = False
if execute_model:
linReg = LinearRegression(n_jobs = -1)
 model_name = 'Linear_Regression_with_time_and_original_features'
features = [X_train_sample[basic_features+time_features+original_features],\
X_val[basic_features+time_features+original_features],test[basic_features+time_features+original_features]]
target = [y_train_sample, y_val]
pred = run_model(linReg, model_name, features, target )
train['Family_survival_rate'] = train_fam_survival
test['Family_survival_rate'] = test_fam_survival
train['Family_survival_rate_NA'] = train_fam_survival_NA
test['Family_survival_rate_NA'] = test_fam_survival_NA
train['Ticket_survival_rate'] = train_ticket_survival
test['Ticket_survival_rate'] = test_ticket_survival
train['Ticket_survival_rate_NA'] = train_ticket_survival_NA
test['Ticket_survival_rate_NA'] = test_ticket_survival_NA<feature_engineering>
%%time
execute_model = False
if execute_model:
randReg = RandomForestRegressor(n_jobs=-1, random_state=seed, max_depth = 10, n_estimators = 20, verbose = 0)
 model_name = 'Simple_RandomForest_Regressor_all_features'
features = [X_train_sample,X_val,test]
target = [y_train_sample, y_val]
pred = run_model(randReg, model_name, features, target )
for df in [train, test]:
df['Survival_Rate'] =(df['Family_survival_rate'] + df['Ticket_survival_rate'])/2
df['Survival_Rate_NA'] =(df['Family_survival_rate_NA'] + df['Ticket_survival_rate_NA'])/2<categorify>
%%time
execute_model = False
if execute_model:
xgbReg = XGBRegressor(n_jobs=-1, random_state=seed, max_depth = 20, \
n_estimators = 50, silent = True)
 model_name = 'XGBoost_Regressor'
features = [X_train_sample,X_val,test]
target = [y_train_sample, y_val]
pred = run_model(xgbReg, model_name, features, target )
non_numerica_features = ['Embarked', 'Sex', 'Deck', 'Title', 'Family Size Grouped', 'Age', 'Fare']
for df in dfs:
for feature in non_numerica_features:
df[feature] = LabelEncoder().fit_transform(df[feature])
<categorify>
def xbg_GridSearchCV(model, params, param_grid, dtrain, num_boost_round = 200, \
nfold = 3, metrics = 'rmse', early_stopping_rounds = 1000, seed = seed):
cvres = {}
for param_name, grid in param_grid.items() :
for value in grid:
params[param_name] = value
res = model.cv(params, dtrain, num_boost_round = num_boost_round, \
nfold = nfold, metrics = metrics, early_stopping_rounds = early_stopping_rounds, seed = seed)
 cvres[param_name + '_' + str(value)] = res.iloc[-1,:]
display(pd.DataFrame(cvres ).T)
 print('
')
 cvres = pd.DataFrame(cvres ).T
idx = cvres['test-rmse-mean'].idxmin()
cvres.loc[idx, 'best'] = 'yes'
cvres.fillna('', inplace=True)
 return cvres
cat_features = ['Sex', 'Pclass', 'Embarked', 'Title', 'Family Size Grouped', 'Deck']
encoded_features =[]
for df in dfs:
for feature in cat_features:
encoded_feature = OneHotEncoder().fit_transform(df[feature].values.reshape(-1,1)).toarray()
n = df[feature].nunique()
cols = ['{}_{}'.format(feature,n)for n in range(1,n+1)]
encoded_df = pd.DataFrame(encoded_feature, columns = cols)
 encoded_df.index = df.index
encoded_features.append(encoded_df )<concatenate>
data = X_train_sample
label = y_train_sample
dtrain = xgb.DMatrix(data = data, label = label, feature_names = data.columns)
 dtest = xgb.DMatrix(data = X_val, label = y_val, feature_names = X_val.columns )
train = pd.concat([train, *encoded_features[:6]], axis = 1)
 test = pd.concat([test, *encoded_features[6:]], axis = 1)
 train.head()<concatenate>
params = {}
params['objective'] = 'reg:linear'
params['eval_metric'] = 'rmse'
params['seed'] = seed
params['silent'] = 1
params['nthread'] = -1
params['eta'] = 0.2
params['gamma'] = 0
params['max_depth'] = 5
params['min_child_weight'] = 1
params['max_delta_step'] = 0
params['subsample'] = 1
params['colsample_bytree'] = 1
params['colsample_bylevel'] = 1
params['lambda'] = 1
params['alpha'] = 0
watchlist = [(dtrain, 'train'),(dtest, 'eval')]
df_all = concat(train, test )<drop_column>
num_boost_round = 2
bst = xgb.train(params, dtrain, num_boost_round, watchlist, early_stopping_rounds = 50 )
drop_cols = ['Deck', 'Embarked', 'Family', 'Family Size', 'Family Size Grouped', 'Survived',
'Name', 'Parch', 'PassengerId', 'Pclass', 'Sex', 'SibSp', 'Ticket', 'Title',
'Ticket_survival_rate', 'Family_survival_rate', 'Ticket_survival_rate_NA', 'Family_survival_rate_NA']
df_all.drop(columns = drop_cols, inplace = True)
 df_all.head()<prepare_x_and_y>
%%time
tuned_model = True
if tuned_model == False:
num_boost_round = 115
bst = xgb.train(params, dtrain, num_boost_round,watchlist, early_stopping_rounds = 50)
 print(bst.eval_set(watchlist))
X_train = train.drop(columns= drop_cols)
 Y_train = train['Survived'].values
X_test = test.drop(columns = drop_cols )<normalization>
ptrain = bst.predict(dtrain, output_margin = True)
 ptest = bst.predict(dtest, output_margin = True)
 dtrain.set_base_margin(ptrain)
 dtest.set_base_margin(ptest )
X_train = StandardScaler().fit_transform(X_train )<normalization>
%%time
tuned_model = True
if tuned_model == False:
param_grid = {'eta': [0.05, 0.1, 0.2, 0.3]}
num_boost_round = 115
cvres = xbg_GridSearchCV(xgb, params, param_grid, dtrain, num_boost_round )
X_test = StandardScaler().fit_transform(X_test )<choose_model_class>
%%time
tuned_model = False
if tuned_model == False:
xgbReg = XGBRegressor(**params, n_jobs=-1, n_estimators = 50)
 model_name = 'XGBoost_Regressor_with_100K'
features = [X_train_sample,X_val,test]
target = [y_train_sample, y_val]
pred = run_model(xgbReg, model_name, features, target )
<choose_model_class><EOS> 
%%time
tuned_model = False
if tuned_model == False:
xgbReg = XGBRegressor(**params, n_jobs=-1, n_estimators =50)
 model_name = 'XGBoost_Regressor_with_6M'
features = [X_train,X_val,test]
target = [y_train, y_val]
pred = run_model(xgbReg, model_name, features, target, usecv = False )