Kaggle competition: Titanic - Machine Learning from Disaster
no_diff = data[(data['abs_lat_diff'] == 0)&(data['abs_lon_diff'] == 0)]
no_diff.shape<compute_test_metric>
%matplotlib inline
sns.set_style('whitegrid' )
def minkowski_distance(x1, x2, y1, y2, p):
return(( abs(x2 - x1)** p)+(abs(y2 - y1)) ** p)**(1 / p )<compute_test_metric>
train = pd.read_csv('/kaggle/input/titanic/train.csv' )
minkowski_distance(0, 3, 0, 4, 2 )<groupby>
train['Ticket_First'] = train['Ticket'].apply(lambda x:x.replace('.','' ).replace('/','' ).split() [0][:2] )
data.groupby('fare-bin')['euclidean'].agg(['mean', 'count'] )<groupby>
ticket_freq = train[['Ticket_First','Survived']].groupby(['Ticket_First'] ).agg([('Nos people', 'count'),('Nos survived', 'sum')])
 ticket_freq.columns = ticket_freq.columns.get_level_values(1)
 ticket_freq = ticket_freq.reset_index(level = [0])
 ticket_freq['Survival %'] = round(ticket_freq['Nos survived']*100/ticket_freq['Nos people'])
 ticket_freq.sort_values(by = ['Nos people'], ascending = False)
data.groupby('passenger_count')['fare_amount'].agg(['mean', 'count'] )<feature_engineering>
def Ticket_Grp(col):
if col[0] in ticket_freq[ticket_freq['Nos people'] > 10]['Ticket_First'].to_list() :
return col[0]
else:
return 'Others'
test = pd.read_csv('.. /input/test.csv', parse_dates = ['pickup_datetime'])
 test['abs_lat_diff'] =(test['dropoff_latitude'] - test['pickup_latitude'] ).abs()
test['abs_lon_diff'] =(test['dropoff_longitude'] - test['pickup_longitude'] ).abs()
test_id = list(test.pop('key'))
test.describe()<feature_engineering>
train['Ticket_Grp'] = train[['Ticket_First']].apply(Ticket_Grp, axis =1)
 train['Ticket_Grp'].value_counts()
test['manhattan'] = minkowski_distance(test['pickup_longitude'], test['dropoff_longitude'],
test['pickup_latitude'], test['dropoff_latitude'], 1)
 test['euclidean'] = minkowski_distance(test['pickup_longitude'], test['dropoff_longitude'],
test['pickup_latitude'], test['dropoff_latitude'], 2 )<compute_test_metric>
train['Salute'] = train['Name'].apply(lambda x:x.split() [1] )
R = 6378
def haversine_np(lon1, lat1, lon2, lat2):

 lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])
 dlon = lon2 - lon1
dlat = lat2 - lat1
a = np.sin(dlat/2.0)**2 + np.cos(lat1)* np.cos(lat2)* np.sin(dlon/2.0)**2
c = 2 * np.arcsin(np.sqrt(a))
km = R * c
return km<compute_test_metric>
def Salute_group(col):
if col[0] in ['Mr.', 'Miss.', 'Mrs.', 'Master.']:
return col[0]
else:
return 'Others'
data['haversine'] = haversine_np(data['pickup_longitude'], data['pickup_latitude'],
data['dropoff_longitude'], data['dropoff_latitude'])
 test['haversine'] = haversine_np(test['pickup_longitude'], test['pickup_latitude'],
test['dropoff_longitude'], test['dropoff_latitude'] )<groupby>
train['Salute_Grp'] = train[['Salute']].apply(Salute_group, axis =1 )
data.groupby('fare-bin')['haversine'].agg(['mean', 'count'] )<choose_model_class>
PclassXSex_med = train[['Sex','Age','Pclass']].groupby(['Sex','Pclass'] ).median()
lr = LinearRegression()<split>
train['Age_PclXSex'] = train[['Age', 'Pclass', 'Sex']].apply(age_PclassSex, axis = 1 )
X_train, X_valid, y_train, y_valid = train_test_split(data, np.array(data['fare_amount']),
stratify = data['fare-bin'],
random_state = RSEED, test_size = 1_000_000 )<train_model>
train.drop(['Age', 'Cabin'], axis =1 , inplace = True )
lr.fit(X_train[['abs_lat_diff', 'abs_lon_diff', 'passenger_count']], y_train)
 print('Intercept', round(lr.intercept_, 4))
print('abs_lat_diff coef: ', round(lr.coef_[0], 4),
'\tabs_lon_diff coef:', round(lr.coef_[1], 4),
'\tpassenger_count coef:', round(lr.coef_[2], 4))<compute_test_metric>
train.dropna(inplace = True )
warnings.filterwarnings('ignore', category = RuntimeWarning)
 def metrics(train_pred, valid_pred, y_train, y_valid):

 train_rmse = np.sqrt(mean_squared_error(y_train, train_pred))
valid_rmse = np.sqrt(mean_squared_error(y_valid, valid_pred))
train_ape = abs(( y_train - train_pred)/ y_train)
 valid_ape = abs(( y_valid - valid_pred)/ y_valid)
 train_ape[train_ape == np.inf] = 0
train_ape[train_ape == -np.inf] = 0
valid_ape[valid_ape == np.inf] = 0
valid_ape[valid_ape == -np.inf] = 0
train_mape = 100 * np.mean(train_ape)
 valid_mape = 100 * np.mean(valid_ape)
 return train_rmse, valid_rmse, train_mape, valid_mape
def evaluate(model, features, X_train, X_valid, y_train, y_valid):

 train_pred = model.predict(X_train[features])
 valid_pred = model.predict(X_valid[features])
 train_rmse, valid_rmse, train_mape, valid_mape = metrics(train_pred, valid_pred,
y_train, y_valid)
 print(f'Training: rmse = {round(train_rmse, 2)} \t mape = {round(train_mape, 2)}')
 print(f'Validation: rmse = {round(valid_rmse, 2)} \t mape = {round(valid_mape, 2)}' )<compute_test_metric>
Sex_Dumm = pd.get_dummies(train['Sex'], drop_first = True)
 Embarked_Dumm = pd.get_dummies(train['Embarked'], drop_first = True)
 Ticket_Grp = pd.get_dummies(train['Ticket_Grp'], drop_first = True, prefix = 'Ticket')
 Salute_Group = pd.get_dummies(train['Salute_Grp'], drop_first = True )
evaluate(lr, ['abs_lat_diff', 'abs_lon_diff', 'passenger_count'],
X_train, X_valid, y_train, y_valid )<compute_train_metric>
train = pd.concat([train, Sex_Dumm, Embarked_Dumm, Ticket_Grp, Salute_Group], axis = 1)
 train.head()
train_mean = y_train.mean()
train_preds = [train_mean for _ in range(len(y_train)) ]
valid_preds = [train_mean for _ in range(len(y_valid)) ]
tr, vr, tm, vm = metrics(train_preds, valid_preds, y_train, y_valid)
 print(f'Baseline Training: rmse = {round(tr, 2)} \t mape = {round(tm, 2)}')
 print(f'Baseline Validation: rmse = {round(vr, 2)} \t mape = {round(vm, 2)}' )<save_to_csv>
y = train['Survived']
X_train, X_test, y_train, y_test = train_test_split(train[['Pclass', 'SibSp', 'Parch', 'Fare',
'Age_PclXSex', 'male', 'Q', 'S', 'Ticket_13', 'Ticket_17',
'Ticket_19', 'Ticket_23', 'Ticket_24', 'Ticket_25', 'Ticket_26',
'Ticket_28', 'Ticket_29', 'Ticket_31', 'Ticket_33', 'Ticket_34',
'Ticket_35', 'Ticket_36', 'Ticket_37', 'Ticket_A5', 'Ticket_CA',
'Ticket_Others', 'Ticket_PC', 'Ticket_SC', 'Ticket_SO', 'Ticket_ST',
'Miss.', 'Mr.', 'Mrs.', 'Others']], y, test_size = 0.3, random_state = 143)
preds = lr.predict(test[['abs_lat_diff', 'abs_lon_diff', 'passenger_count']])
 sub = pd.DataFrame({'key': test_id, 'fare_amount': preds})
 sub.to_csv('sub_lr_simple.csv', index = False )<create_dataframe>
rf = RandomForestClassifier()
rf.fit(X_train,y_train )
sns.distplot(sub['fare_amount'])
 plt.title('Distribution of Linear Regression Predictions');<filter>
pred = rf.predict(X_test )
test.loc[sub[sub['fare_amount'] > 100].index]<filter>
print(accuracy_score(y_test, pred))
sub[sub['fare_amount'] > 100]<train_model>
n_estimators = [int(x)for x in np.arange(200, 2200, 200)]
max_features = ['auto','sqrt']
max_depth = [int(x)for x in np.arange(10,110,10)]
max_depth.append(None)
 min_samples_leaf = [1,2,3,4,5]
min_samples_split = [2,4,6,8,10]
bootstrap = [True,False]
param_grid = {'n_estimators' : n_estimators,
'max_features' : max_features,
'max_depth' : max_depth,
'min_samples_leaf' : min_samples_leaf,
'min_samples_split' : min_samples_split,
'bootstrap' : bootstrap}
rf1 = RandomForestClassifier()
rf1_random = RandomizedSearchCV(rf1, param_grid, n_iter = 100, cv = 3, verbose =2 , random_state = 143, n_jobs = -1)
lr.fit(X_train[['haversine', 'abs_lat_diff', 'abs_lon_diff', 'passenger_count']], y_train)
 evaluate(lr, ['haversine', 'abs_lat_diff', 'abs_lon_diff', 'passenger_count'],
X_train, X_valid, y_train, y_valid )<train_model>
rf1_random.fit(X_train, y_train )
random_forest = RandomForestRegressor(n_estimators = 20, max_depth = 20,
max_features = None, oob_score = True,
bootstrap = True, verbose = 1, n_jobs = -1)
 random_forest.fit(X_train[['haversine', 'abs_lat_diff', 'abs_lon_diff', 'passenger_count']], y_train )<compute_test_metric>
rf1_random.best_params_
evaluate(random_forest, ['haversine', 'abs_lat_diff', 'abs_lon_diff', 'passenger_count'],
X_train, X_valid, y_train, y_valid )<save_to_csv>
rf1_random.best_estimator_
preds = random_forest.predict(test[['haversine', 'abs_lat_diff', 'abs_lon_diff', 'passenger_count']])
 sub = pd.DataFrame({'key': test_id, 'fare_amount': preds})
 sub.to_csv('sub_rf_simple.csv', index = False)
 sns.distplot(sub['fare_amount'])
 plt.title('Distribution of Random Forest Predicted Fare Amount');<filter>
pred2 = rf1_random.best_estimator_.predict(X_test )
sub.loc[simple_over_100]<predict_on_test>
print(accuracy_score(y_test, pred2))
lr_tpred = lr.predict(X_train[['haversine', 'abs_lat_diff', 'abs_lon_diff', 'passenger_count']])
 rf_tpred = random_forest.predict(X_train[['haversine', 'abs_lat_diff', 'abs_lon_diff', 'passenger_count']])
 lr_pred = lr.predict(X_valid[['haversine', 'abs_lat_diff', 'abs_lon_diff', 'passenger_count']])
 rf_pred = random_forest.predict(X_valid[['haversine', 'abs_lat_diff', 'abs_lon_diff', 'passenger_count']])
 train_pred =(lr_tpred + rf_tpred)/ 2
valid_pred =(lr_pred + rf_pred)/ 2
tr, vr, tm, vm = metrics(train_pred, valid_pred, y_train, y_valid)
 print(f'Combined Training: rmse = {round(tr, 2)} \t mape = {round(tm, 2)}')
 print(f'Combined Validation: rmse = {round(vr, 2)} \t mape = {round(vm, 2)}' )<predict_on_test>
def model_rf(X_train, X_valid, y_train, y_valid, test, features,
model = RandomForestRegressor(n_estimators = 20, max_depth = 20,
n_jobs = -1),
return_model = False):

 model.fit(X_train[features], y_train)
 evaluate(model, features, X_train, X_valid, y_train, y_valid)
 preds = model.predict(test[features])
 sub = pd.DataFrame({'key': test_id, 'fare_amount': preds})
 feature_importances = pd.DataFrame({'feature': features,
'importance': model.feature_importances_} ).\
sort_values('importance', ascending = False ).set_index('feature')
 if return_model:
return sub, feature_importances, model
return sub, feature_importances<compute_test_metric>
test = pd.read_csv('/kaggle/input/titanic/test.csv' )
sub, fi = model_rf(X_train, X_valid, y_train, y_valid, test,
features = ['abs_lat_diff', 'abs_lon_diff', 'haversine', 'passenger_count',
'pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude'] )<save_to_csv>
test['Ticket_First'] = test['Ticket'].apply(lambda x:x.replace('.','' ).replace('/','' ).split() [0][:2] )
sub.to_csv('sub_rf_8_features.csv', index = False)
 sub['fare_amount'].plot.hist() ;<create_dataframe>
test['Ticket_Grp'] = test[['Ticket_First']].apply(Ticket_Grp2, axis =1 )
def extract_dateinfo(df, date_col, drop=True, time=False,
start_ref = pd.datetime(1900, 1, 1),
extra_attr = False):

 df = df.copy()
fld = df[date_col]
fld_dtype = fld.dtype
if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):
fld_dtype = np.datetime64
if not np.issubdtype(fld_dtype, np.datetime64):
df[date_col] = fld = pd.to_datetime(fld, infer_datetime_format=True)
 pre = re.sub('[Dd]ate', '', date_col)
 pre = re.sub('[Tt]ime', '', pre)
 attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear', 'Days_in_month', 'is_leap_year']
if extra_attr:
attr = attr + ['Is_month_end', 'Is_month_start', 'Is_quarter_end',
'Is_quarter_start', 'Is_year_end', 'Is_year_start']
if time:
attr = attr + ['Hour', 'Minute', 'Second']
for n in attr:
df[pre + n] = getattr(fld.dt, n.lower())
 df[pre + 'Days_in_year'] = df[pre + 'is_leap_year'] + 365
if time:
df[pre + 'frac_day'] =(( df[pre + 'Hour'])+(df[pre + 'Minute'] / 60)+(df[pre + 'Second'] / 60 / 60)) / 24
df[pre + 'frac_week'] =(df[pre + 'Dayofweek'] + df[pre + 'frac_day'])/ 7
df[pre + 'frac_month'] =(df[pre + 'Day'] +(df[pre + 'frac_day'])) /(df[pre + 'Days_in_month'] + 1)
 df[pre + 'frac_year'] =(df[pre + 'Dayofyear'] + df[pre + 'frac_day'])/(df[pre + 'Days_in_year'] + 1)
 df[pre + 'Elapsed'] =(fld - start_ref ).dt.total_seconds()
if drop:
df = df.drop(date_col, axis=1)
 return df<drop_column>
test['Salute'] = test['Name'].apply(lambda x:x.split() [1] )
test = extract_dateinfo(test, 'pickup_datetime', drop = False,
time = True, start_ref = data['pickup_datetime'].min())
 test.head()<groupby>
test['Salute'] = test['Name'].apply(lambda x:x.split() [1])
 def Salute_group(col):
if col[0] in ['Mr.', 'Miss.', 'Mrs.', 'Master.']:
return col[0]
else:
return 'Others'
fare_counts = data.groupby('fare_amount')['haversine'].agg(['count', pd.Series.nunique] ).sort_values('count', ascending = False)
 fare_counts.head()<split>
test['Salute_Grp'] = test[['Salute']].apply(Salute_group, axis =1 )
X_train, X_valid, y_train, y_valid = train_test_split(data, np.array(data['fare_amount']),
stratify = data['fare-bin'],
random_state = RSEED, test_size = 1_000_000 )<define_variables>
PclassXSex_med = test[['Sex','Age','Pclass']].groupby(['Sex','Pclass'] ).median()
PclassXSex_med
time_features = ['pickup_frac_day', 'pickup_frac_week', 'pickup_frac_year', 'pickup_Elapsed']
features = ['abs_lat_diff', 'abs_lon_diff', 'haversine', 'passenger_count',
'pickup_latitude', 'pickup_longitude',
'dropoff_latitude', 'dropoff_longitude'] + time_features<compute_test_metric>
test['Age_PclXSex'] = test[['Age', 'Pclass', 'Sex']].apply(age_PclassSex, axis = 1 )
sub, fi = model_rf(X_train, X_valid, y_train, y_valid, test,
features = features )<train_model>
test.drop(['Cabin', 'Age'], axis =1 , inplace = True )
lr = LinearRegression()
lr.fit(X_train[features], y_train)
 evaluate(lr, features, X_train, X_valid, y_train, y_valid )<save_to_csv>
test[pd.isnull(test['Fare'])]
sub.to_csv('sub_rf_frac_time.csv', index = False )<drop_column>
Fare_med = test[['Pclass','Fare','Sex', 'Embarked']].groupby(['Pclass','Sex', 'Embarked'] ).agg(['count', 'mean'])
 Fare_med
features = list(data.columns)
 for f in ['pickup_datetime', 'fare_amount', 'fare-bin', 'color']:
features.remove(f)
 len(features )<compute_train_metric>
test['Fare'].fillna(12.718, inplace = True )
sub, fi, random_forest = model_rf(X_train, X_valid, y_train, y_valid, test,
features = features, return_model = True )<save_to_csv>
Sex_Dumm = pd.get_dummies(test['Sex'], drop_first = True)
 Embarked_Dumm = pd.get_dummies(test['Embarked'], drop_first = True)
 Ticket_Grp = pd.get_dummies(test['Ticket_Grp'], drop_first = True, prefix = 'Ticket')
 Salute_Group = pd.get_dummies(test['Salute_Grp'], drop_first = True )
sub.to_csv('sub_rf_all_features.csv', index = False )<train_on_grid>
test = pd.concat([test, Sex_Dumm, Embarked_Dumm, Ticket_Grp, Salute_Group], axis = 1)
 test.head()
param_grid = {
'n_estimators': np.linspace(10, 100 ).astype(int),
'max_depth': [None] + list(np.linspace(5, 30 ).astype(int)) ,
'max_features': ['auto', 'sqrt', None] + list(np.arange(0.5, 1, 0.1)) ,
'max_leaf_nodes': [None] + list(np.linspace(10, 50, 500 ).astype(int)) ,
'min_samples_split': [2, 5, 10],
'bootstrap': [True, False]
}
estimator = RandomForestRegressor(random_state = RSEED)
 rs = RandomizedSearchCV(estimator, param_grid, n_jobs = -1,
scoring = 'neg_mean_absolute_error', cv = 3,
n_iter = 100, verbose = 1, random_state=RSEED )<train_model>
X = train[['Pclass', 'SibSp', 'Parch', 'Fare',
'Age_PclXSex', 'male', 'Q', 'S', 'Ticket_13', 'Ticket_17',
'Ticket_19', 'Ticket_23', 'Ticket_24', 'Ticket_25', 'Ticket_26',
'Ticket_28', 'Ticket_29', 'Ticket_31', 'Ticket_33', 'Ticket_34',
'Ticket_35', 'Ticket_36', 'Ticket_37', 'Ticket_A5', 'Ticket_CA',
'Ticket_Others', 'Ticket_PC', 'Ticket_SC', 'Ticket_SO', 'Ticket_ST',
'Miss.', 'Mr.', 'Mrs.', 'Others']]
y = train['Survived']
tune_data = data.sample(100_000, random_state = RSEED)
 time_features = ['pickup_frac_day', 'pickup_frac_week', 'pickup_frac_year', 'pickup_Elapsed']
features = ['abs_lat_diff', 'abs_lon_diff', 'haversine', 'passenger_count',
'pickup_latitude', 'pickup_longitude',
'dropoff_latitude', 'dropoff_longitude'] + time_features
rs.fit(tune_data[features], np.array(tune_data['fare_amount']))<find_best_params>
rf_fin = rf1_random.best_estimator_
rf_fin.fit(X,y )
model = rs.best_estimator_
print(f'The best parameters were {rs.best_params_} with a negative mae of {rs.best_score_}' )<train_model>
test.set_index('PassengerId', inplace = True )
model.n_jobs = -1
model.fit(X_train[features], y_train)
 evaluate(model, features, X_train, X_valid, y_train, y_valid )<save_to_csv>
test_fin =test[['Pclass', 'SibSp', 'Parch', 'Fare',
'Age_PclXSex', 'male', 'Q', 'S', 'Ticket_13', 'Ticket_17',
'Ticket_19', 'Ticket_23', 'Ticket_24', 'Ticket_25', 'Ticket_26',
'Ticket_28', 'Ticket_29', 'Ticket_31', 'Ticket_33', 'Ticket_34',
'Ticket_35', 'Ticket_36', 'Ticket_37', 'Ticket_A5', 'Ticket_CA',
'Ticket_Others', 'Ticket_PC', 'Ticket_SC', 'Ticket_SO', 'Ticket_ST',
'Miss.', 'Mr.', 'Mrs.', 'Others']]
test_fin
pred = np.array(model.predict(test[features])).reshape(( -1))
sub = pd.DataFrame({'key': test_id, 'fare_amount': pred})
 sub.to_csv('sub_rf_tuned.csv', index = False)
 sub['fare_amount'].plot.hist() ;
plt.title('Predicted Test Fare Distribution');<set_options>
pred_fin = rf_fin.predict(test_fin)
 pred_df = pd.DataFrame(pred_fin, columns = ['Survived'],index = test_fin.index)
 pred_df
plt.figure(figsize=(8, 5), dpi=80)
 sns.set_style("darkgrid" )<load_from_csv>
pred_df['Survived'].to_csv('My_Titanic_Predictions.csv', index = True, header = True )
test_dataset = pd.read_csv('.. /input/test.csv')
 train_dataset = pd.read_csv('.. /input/train.csv', nrows=2_000_000 )<drop_column>
feat_imp = pd.DataFrame(importance, index = X.columns, columns = ['Importance'])
 feat_imp.sort_values(['Importance'], inplace = True)