Kaggle competition: Titanic - Machine Learning from Disaster
def sphere_dist(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):

 R_earth = 6371
pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,
[pickup_lat, pickup_lon,
dropoff_lat, dropoff_lon])
 dlat = dropoff_lat - pickup_lat
dlon = dropoff_lon - pickup_lon
a = np.sin(dlat/2.0)**2 + np.cos(pickup_lat)* np.cos(dropoff_lat)* np.sin(dlon/2.0)**2
return 2 * R_earth * np.arcsin(np.sqrt(a))
def add_airport_dist(dataset):

 jfk_coord =(40.639722, -73.778889)
 ewr_coord =(40.6925, -74.168611)
 lga_coord =(40.77725, -73.872611)
 pickup_lat = dataset['pickup_latitude']
dropoff_lat = dataset['dropoff_latitude']
pickup_lon = dataset['pickup_longitude']
dropoff_lon = dataset['dropoff_longitude']
pickup_jfk = sphere_dist(pickup_lat, pickup_lon, jfk_coord[0], jfk_coord[1])
 dropoff_jfk = sphere_dist(jfk_coord[0], jfk_coord[1], dropoff_lat, dropoff_lon)
 pickup_ewr = sphere_dist(pickup_lat, pickup_lon, ewr_coord[0], ewr_coord[1])
 dropoff_ewr = sphere_dist(ewr_coord[0], ewr_coord[1], dropoff_lat, dropoff_lon)
 pickup_lga = sphere_dist(pickup_lat, pickup_lon, lga_coord[0], lga_coord[1])
 dropoff_lga = sphere_dist(lga_coord[0], lga_coord[1], dropoff_lat, dropoff_lon)
 dataset['jfk_dist'] = pd.concat([pickup_jfk, dropoff_jfk], axis=1 ).min(axis=1)
 dataset['ewr_dist'] = pd.concat([pickup_ewr, dropoff_ewr], axis=1 ).min(axis=1)
 dataset['lga_dist'] = pd.concat([pickup_lga, dropoff_lga], axis=1 ).min(axis=1)
 return dataset
def add_datetime_info(dataset):
dataset['pickup_datetime'] = pd.to_datetime(dataset['pickup_datetime'],format="%Y-%m-%d %H:%M:%S UTC")
 dataset['hour'] = dataset.pickup_datetime.dt.hour
dataset['day'] = dataset.pickup_datetime.dt.day
dataset['month'] = dataset.pickup_datetime.dt.month
dataset['weekday'] = dataset.pickup_datetime.dt.weekday
dataset['year'] = dataset.pickup_datetime.dt.year
return dataset 
<feature_engineering>
%matplotlib inline
sns.set_style('whitegrid' )
def transform_features(df):
df = add_datetime_info(df)
 df = add_airport_dist(df)
 df['distance'] = sphere_dist(df['pickup_latitude'], df['pickup_longitude'],
df['dropoff_latitude'] , df['dropoff_longitude'])
 df.drop(columns=['pickup_datetime'], inplace=True)
 return df<categorify>
train = pd.read_csv('/kaggle/input/titanic/train.csv' )
train_df = transform_features(train_df)
 test_df = transform_features(test_df)
 test_df.drop(columns=['key'], inplace=True )<prepare_x_and_y>
train['Salute'] = train['Name'].apply(lambda x:x.split() [1] )
train_df_y = train_df.fare_amount.copy()
train_df_X = train_df[test_df.columns]<create_dataframe>
def Salute_group(col):
if col[0] in ['Mr.', 'Miss.', 'Mrs.', 'Master.']:
return col[0]
else:
return 'Others'
dtrain = lgb.Dataset(train_df_X, label = train_df_y, free_raw_data = False )<init_hyperparams>
train['Salute_Grp'] = train[['Salute']].apply(Salute_group, axis =1 )
print("Light Gradient Boosting Regressor: ")
 lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'learning_rate' :'0.03',
'num_leaves':'31',
'max_depth' : '-1',
'subsample' :'.8',
'colsample_bytree' : '0.6',
'min_split_gain' : '0.5',
'min_child_weight' : '1',
'min_child_samples' :'10',
'scale_pos_weight' : '1',
'num_threads' : '4',
'seed' : '0',
'eval_freq' : '50'
}
folds = KFold(n_splits=5, shuffle=True, random_state=1)
 fold_preds = np.zeros(testshape[0])
 oof_preds = np.zeros(trainshape[0])
 dtrain.construct()
modelstart = time.time()
for trn_idx, val_idx in folds.split(train_df_X):
clf = lgb.train(
 params=lgbm_params,
train_set=dtrain.subset(trn_idx),
valid_sets=dtrain.subset(val_idx),
num_boost_round=10000,
early_stopping_rounds=125,
verbose_eval=500 
)
 oof_preds[val_idx] = clf.predict(dtrain.data.iloc[val_idx])
 fold_preds += clf.predict(test_df)/ folds.n_splits
print(mean_squared_error(train_df_y.iloc[val_idx], oof_preds[val_idx])**.5)
 print("Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))<load_from_csv>
train['Ticket_First'] = train['Ticket'].apply(lambda x:x.replace('.','' ).replace('/','' ).split() [0][:2])
test_df = pd.read_csv(TEST_PATH )<save_to_csv>
ticket_freq = train[['Ticket_First','Survived']].groupby(['Ticket_First'] ).agg([('Nos people', 'count'),('Nos survived', 'sum')])
 ticket_freq.columns = ticket_freq.columns.get_level_values(1)
 ticket_freq = ticket_freq.reset_index(level = [0])
 ticket_freq['Survival %'] = round(ticket_freq['Nos survived']*100/ticket_freq['Nos people'])
 ticket_freq.sort_values(by = ['Nos people'], ascending = False)
result = pd.DataFrame({'key':test_df['key'], 'fare_amount':fold_preds})
 result.head()
result.to_csv('taxi-fare-prediction.csv', index=False )<load_from_csv>
def Ticket_Grp(col):
if col[0] in ticket_freq[ticket_freq['Nos people'] > 10]['Ticket_First'].to_list() :
return col[0]
else:
return 'Others'
train_df = pd.read_csv('.. /input/train.csv', nrows = 6_000_000)
 train_df.dtypes<count_missing_values>
train['Ticket_Grp'] = train[['Ticket_First']].apply(Ticket_Grp, axis =1 )
print(train_df.isnull().sum() )<drop_column>
train['Ticket_Grp'].value_counts()
train_df = train_df.dropna(how = 'any', axis = 'rows' )<drop_column>
PclassXSex_med = train[['Sex','Age','Pclass']].groupby(['Sex','Pclass'] ).median()
def clean_df(df):
return df[(df.fare_amount > 0)& 
(df.pickup_longitude > -80)&(df.pickup_longitude < -70)& 
(df.pickup_latitude > 35)&(df.pickup_latitude < 45)& 
(df.dropoff_longitude > -80)&(df.dropoff_longitude < -70)& 
(df.dropoff_latitude > 35)&(df.dropoff_latitude < 45)& 
(df.passenger_count > 0)&(df.passenger_count < 10)]
train_df = clean_df(train_df)
 print(len(train_df))<concatenate>
train['Age_PclXSex'] = train[['Age', 'Pclass', 'Sex']].apply(age_PclassSex, axis = 1 )
def sphere_dist(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):

 R_earth = 6371
pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,
[pickup_lat, pickup_lon,
dropoff_lat, dropoff_lon])
 dlat = dropoff_lat - pickup_lat
dlon = dropoff_lon - pickup_lon
a = np.sin(dlat/2.0)**2 + np.cos(pickup_lat)* np.cos(dropoff_lat)* np.sin(dlon/2.0)**2
return 2 * R_earth * np.arcsin(np.sqrt(a))
def add_airport_dist(dataset):

 jfk_coord =(40.639722, -73.778889)
 ewr_coord =(40.6925, -74.168611)
 lga_coord =(40.77725, -73.872611)
 pickup_lat = dataset['pickup_latitude']
dropoff_lat = dataset['dropoff_latitude']
pickup_lon = dataset['pickup_longitude']
dropoff_lon = dataset['dropoff_longitude']
pickup_jfk = sphere_dist(pickup_lat, pickup_lon, jfk_coord[0], jfk_coord[1])
 dropoff_jfk = sphere_dist(jfk_coord[0], jfk_coord[1], dropoff_lat, dropoff_lon)
 pickup_ewr = sphere_dist(pickup_lat, pickup_lon, ewr_coord[0], ewr_coord[1])
 dropoff_ewr = sphere_dist(ewr_coord[0], ewr_coord[1], dropoff_lat, dropoff_lon)
 pickup_lga = sphere_dist(pickup_lat, pickup_lon, lga_coord[0], lga_coord[1])
 dropoff_lga = sphere_dist(lga_coord[0], lga_coord[1], dropoff_lat, dropoff_lon)
 dataset['jfk_dist'] = pd.concat([pickup_jfk, dropoff_jfk], axis=1 ).min(axis=1)
 dataset['ewr_dist'] = pd.concat([pickup_ewr, dropoff_ewr], axis=1 ).min(axis=1)
 dataset['lga_dist'] = pd.concat([pickup_lga, dropoff_lga], axis=1 ).min(axis=1)
 return dataset
def add_datetime_info(dataset):
dataset['pickup_datetime'] = pd.to_datetime(dataset['pickup_datetime'],format="%Y-%m-%d %H:%M:%S UTC")
 dataset['hour'] = dataset.pickup_datetime.dt.hour
dataset['day'] = dataset.pickup_datetime.dt.day
dataset['month'] = dataset.pickup_datetime.dt.month
dataset['weekday'] = dataset.pickup_datetime.dt.weekday
dataset['year'] = dataset.pickup_datetime.dt.year
return dataset
train_df = add_datetime_info(train_df)
 train_df = add_airport_dist(train_df)
 train_df['distance'] = sphere_dist(train_df['pickup_latitude'], train_df['pickup_longitude'],
train_df['dropoff_latitude'] , train_df['dropoff_longitude'])
 train_df.head()<drop_column>
train.drop(['Age', 'Cabin'], axis =1 , inplace = True )
train_df.drop(columns=['key', 'pickup_datetime'], inplace=True)
 train_df.head()<split>
train.dropna(inplace = True )
y = train_df['fare_amount']
train = train_df.drop(columns=['fare_amount'])
 x_train,x_test,y_train,y_test = train_test_split(train,y,random_state=0,test_size=0.01 )<train_on_grid>
Sex_Dumm = pd.get_dummies(train['Sex'], drop_first = True)
 Embarked_Dumm = pd.get_dummies(train['Embarked'], drop_first = True)
 Ticket_Grp = pd.get_dummies(train['Ticket_Grp'], drop_first = True, prefix = 'Ticket')
 Salute_Group = pd.get_dummies(train['Salute_Grp'], drop_first = True )
params = {
'max_depth': 8,
'eta':.03,
'subsample': 1,
'colsample_bytree': 0.8,
'objective':'reg:linear',
'eval_metric':'rmse',
'silent': 1
}
CV=False
if CV:
dtrain = xgb.DMatrix(train,label=y)
 gridsearch_params = [ 
(eta)
 for eta in np.arange (.04, 0.12,.02)
 ]
min_rmse = float("Inf")
 best_params = None
for(eta)in gridsearch_params:
print("CV with eta={} ".format(
 eta))
params['eta'] = eta
cv_results = xgb.cv(
 params,
dtrain,
num_boost_round=1000,
nfold=3,
metrics={'rmse'},
early_stopping_rounds=10 
)
 mean_rmse = cv_results['test-rmse-mean'].min()
boost_rounds = cv_results['test-rmse-mean'].argmin()
print("\tRMSE {} for {} rounds".format(mean_rmse, boost_rounds))
if mean_rmse < min_rmse:
min_rmse = mean_rmse
best_params =(eta)
 print("Best params: {}, RMSE: {}".format(best_params, min_rmse))
else:
params['silent'] = 0
print(params )<train_model>
train = pd.concat([train, Sex_Dumm, Embarked_Dumm, Ticket_Grp, Salute_Group], axis = 1)
 train.head()
def XGBmodel(x_train,x_test,y_train,y_test,params):
matrix_train = xgb.DMatrix(x_train,label=y_train)
 matrix_test = xgb.DMatrix(x_test,label=y_test)
 model=xgb.train(params=params,
dtrain=matrix_train,num_boost_round=5000,
early_stopping_rounds=10,evals=[(matrix_test,'test')])
 return model
model = XGBmodel(x_train,x_test,y_train,y_test,params )<load_from_csv>
y = train['Survived']
X_train, X_test, y_train, y_test = train_test_split(train[['Pclass', 'SibSp', 'Parch', 'Fare',
'Age_PclXSex', 'male', 'Q', 'S', 'Ticket_13', 'Ticket_17',
'Ticket_19', 'Ticket_23', 'Ticket_24', 'Ticket_25', 'Ticket_26',
'Ticket_28', 'Ticket_29', 'Ticket_31', 'Ticket_33', 'Ticket_34',
'Ticket_35', 'Ticket_36', 'Ticket_37', 'Ticket_A5', 'Ticket_CA',
'Ticket_Others', 'Ticket_PC', 'Ticket_SC', 'Ticket_SO', 'Ticket_ST',
'Miss.', 'Mr.', 'Mrs.', 'Others']], y, test_size = 0.3, random_state = 143)
test_df = pd.read_csv('.. /input/test.csv')
 test_df = add_datetime_info(test_df)
 test_df = add_airport_dist(test_df)
 test_df['distance'] = sphere_dist(test_df['pickup_latitude'], test_df['pickup_longitude'],
test_df['dropoff_latitude'] , test_df['dropoff_longitude'])
 test_key = test_df['key']
x_pred = test_df.drop(columns=['key', 'pickup_datetime'])
 prediction = model.predict(xgb.DMatrix(x_pred), ntree_limit = model.best_ntree_limit )<save_to_csv>
from xgboost import XGBClassifier
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction.round(2)
 })
 submission.to_csv('taxi_fare_submission.csv',index=False)
 submission.head()<import_modules>
xgb_base = XGBClassifier(random_state = 105)
 xgb_base.fit(X_train, y_train)
 xgb_base
from geopy.distance import great_circle<set_options>
pred = xgb_base.predict(X_test)
 print(accuracy_score(y_test, pred))
warnings.filterwarnings('ignore' )<load_from_csv>
param_grid = {"n_est" : [10, 20, 25, 30, 40, 50, 100,150, 200],
"learning_rate" : [0.01, 0.02, 0.03, 0.05, 0.10, 0.15, 0.20, 0.25, 0.30 ] ,
"max_depth" : [ 2, 3, 4, 5, 6, 8, 10, 12, 15],
"min_child_weight" : [ 1, 3, 5, 7 ],
"gamma" : [ 0.0, 0.1, 0.2 , 0.3, 0.4 ],
"colsample_bytree" : [ 0.1, 0.2, 0.3, 0.4, 0.5 , 0.7 ] }
xgb_cl = XGBClassifier(random_state = 105)
 xgb_random = RandomizedSearchCV(xgb_cl, param_grid, n_iter = 500, cv = 5, verbose =2 ,
scoring = 'accuracy', random_state = 143, n_jobs = -1)
 xgb_random.fit(X_train, y_train )
train = pd.read_csv('.. /input/train.csv',nrows=10_000_000 )<filter>
xgb_random.best_params_
train = train.loc[(train['fare_amount'] > 0)&(train['fare_amount'] < 200)]
train = train.loc[(train['pickup_longitude'] > -150)&(train['pickup_longitude'] < 0)]
train = train.loc[(train['pickup_latitude'] > 0)&(train['pickup_latitude'] < 80)]
train = train.loc[(train['dropoff_longitude'] > -150)&(train['dropoff_longitude'] < 0)]
train = train.loc[(train['dropoff_latitude'] > 0)&(train['dropoff_longitude'] < 80)]
train = train.loc[train['passenger_count'] <= 8]<load_from_csv>
pred2 = xgb_random.best_estimator_.predict(X_test )
test = pd.read_csv('.. /input/test.csv' )<compute_test_metric>
print(accuracy_score(y_test, pred2))
def haversine(lon1, lat1, lon2, lat2):

 lon1 = np.radians(lon1)
 lat1 = np.radians(lat1)
 lon2 = np.radians(lon2)
 lat2 = np.radians(lat2)
 dlon = lon2 - lon1
dlat = lat2 - lat1
a = np.sin(dlat/2)**2 + np.cos(lat1)* np.cos(lat2)* np.sin(dlon/2)**2
c = 2 * np.arcsin(np.sqrt(a))
r = 6371
return c * r<feature_engineering>
test = pd.read_csv('/kaggle/input/titanic/test.csv' )
def add_travel_distance_vector_features(df):
df['distance'] = haversine(df.dropoff_longitude,df.dropoff_latitude,df.pickup_longitude,df.pickup_latitude)
 add_travel_distance_vector_features(train)
 add_travel_distance_vector_features(test )<drop_column>
test['Salute'] = test['Name'].apply(lambda x:x.split() [1] )
<count_missing_values>
def Salute_group(col):
if col[0] in ['Mr.', 'Miss.', 'Mrs.', 'Master.']:
return col[0]
else:
return 'Others'
train.isnull().sum()<count_missing_values>
test['Salute_Grp'] = test[['Salute']].apply(Salute_group, axis =1)
 test['Salute_Grp'].value_counts()
train.isnull().sum()<drop_column>
test['Ticket_First'] = test['Ticket'].apply(lambda x:x.replace('.','' ).replace('/','' ).split() [0][:2])
train.dropna(how = 'any', axis = 'rows', inplace=True )<data_type_conversions>
test['Ticket_Grp'] = test[['Ticket_First']].apply(Ticket_Grp2, axis =1 )
train['key'] = pd.to_datetime(train.key)
 test['key'] = pd.to_datetime(test.key )<feature_engineering>
PclassXSex_med = test[['Sex','Age','Pclass']].groupby(['Sex','Pclass'] ).median()
PclassXSex_med
train['hour'] = train['key'].dt.hour
test['hour'] = test['key'].dt.hour
train['day'] = train['key'].dt.day
test['day'] = test['key'].dt.day
train['month'] = train['key'].dt.month
test['month'] = test['key'].dt.month
train['year'] = train['key'].dt.year
test['year'] = test['key'].dt.year<feature_engineering>
test['Age_PclXSex'] = test[['Age', 'Pclass', 'Sex']].apply(age_PclassSex, axis = 1 )
train['daysinmonth'] = train['key'].dt.daysinmonth
test['daysinmonth'] = test['key'].dt.daysinmonth
train['weekofyear'] = train['key'].dt.weekofyear
test['weekofyear'] = test['key'].dt.weekofyear
train['dayofweek'] = train['key'].dt.dayofweek
test['dayofweek'] = test['key'].dt.dayofweek
train['dayofyear'] = train['key'].dt.dayofyear
test['dayofyear'] = test['key'].dt.dayofyear
train['quarter'] = train['key'].dt.quarter
test['quarter'] = test['key'].dt.quarter<data_type_conversions>
test.drop(['Cabin', 'Age'], axis =1 , inplace = True )
train.key = train.key.values.astype(np.int64)
 test.key = test.key.values.astype(np.int64 )<drop_column>
Fare_med = test[['Pclass','Fare','Sex', 'Embarked']].groupby(['Pclass','Sex', 'Embarked'] ).agg(['count', 'mean'])
 Fare_med
train.pop('pickup_datetime')
 test.pop('pickup_datetime')
 train.pop('key')
 test.pop('key' )<feature_engineering>
test['Fare'].fillna(12.718, inplace = True )
train['longitude_distance'] = abs(train['pickup_longitude'] - train['dropoff_longitude'])
 train['latitude_distance'] = abs(train['pickup_latitude'] - train['dropoff_latitude'])
 test['longitude_distance'] = abs(test['pickup_longitude'] - test['dropoff_longitude'])
 test['latitude_distance'] = abs(test['pickup_latitude'] - test['dropoff_latitude'])
 train['distance_travelled_sin'] = np.sin(( train['longitude_distance'] ** 2 * train['latitude_distance'] ** 2)**.5)
 test['distance_travelled_sin'] = np.sin(( test['longitude_distance'] ** 2 * test['latitude_distance'] ** 2)**.5 )<prepare_x_and_y>
Sex_Dumm = pd.get_dummies(test['Sex'], drop_first = True)
 Embarked_Dumm = pd.get_dummies(test['Embarked'], drop_first = True)
 Ticket_Grp = pd.get_dummies(test['Ticket_Grp'], drop_first = True, prefix = 'Ticket')
 Salute_Group = pd.get_dummies(test['Salute_Grp'], drop_first = True )
y = train.pop('fare_amount')
 X = train<import_modules>
test = pd.concat([test, Sex_Dumm, Embarked_Dumm, Ticket_Grp, Salute_Group], axis = 1)
 test.columns
from sklearn.preprocessing import StandardScaler<normalization>
X = train[['Pclass', 'SibSp', 'Parch', 'Fare',
'Age_PclXSex', 'male', 'Q', 'S', 'Ticket_13', 'Ticket_17',
'Ticket_19', 'Ticket_23', 'Ticket_24', 'Ticket_25', 'Ticket_26',
'Ticket_28', 'Ticket_29', 'Ticket_31', 'Ticket_33', 'Ticket_34',
'Ticket_35', 'Ticket_36', 'Ticket_37', 'Ticket_A5', 'Ticket_CA',
'Ticket_Others', 'Ticket_PC', 'Ticket_SC', 'Ticket_SO', 'Ticket_ST',
'Miss.', 'Mr.', 'Mrs.', 'Others']]
y = train['Survived']
scaler = StandardScaler()<normalization>
xgb_fin_base = XGBClassifier()
xgb_fin_base.fit(X,y )
X = scaler.fit_transform(X)
 test = scaler.transform(test )<split>
test.set_index('PassengerId', inplace = True )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.001, random_state=124 )<import_modules>
test_fin =test[['Pclass', 'SibSp', 'Parch', 'Fare',
'Age_PclXSex', 'male', 'Q', 'S', 'Ticket_13', 'Ticket_17',
'Ticket_19', 'Ticket_23', 'Ticket_24', 'Ticket_25', 'Ticket_26',
'Ticket_28', 'Ticket_29', 'Ticket_31', 'Ticket_33', 'Ticket_34',
'Ticket_35', 'Ticket_36', 'Ticket_37', 'Ticket_A5', 'Ticket_CA',
'Ticket_Others', 'Ticket_PC', 'Ticket_SC', 'Ticket_SO', 'Ticket_ST',
'Miss.', 'Mr.', 'Mrs.', 'Others']]
test_fin
from keras.layers import Dense, Input,Dropout,LeakyReLU,LSTM,BatchNormalization
from keras.models import Model
from keras import backend as Backend
import keras.activations as act<compute_test_metric>
pred_fin_base = xgb_fin_base.predict(test_fin)
 pred_base_df = pd.DataFrame(pred_fin_base, columns = ['Survived'],index = test_fin.index)
 pred_base_df
def rmse(y_true, y_pred):
return Backend.sqrt(Backend.mean(Backend.square(y_pred - y_true), axis=-1))<choose_model_class>
pred_base_df['Survived'].to_csv('My_Titanic_Predictions.csv', index = True, header = True )
<choose_model_class>
xgb_tuned = xgb_random.best_estimator_
def lstm(n):
model_in = Input(shape=(1,n))
model = LSTM(100 )(model_in)
 model = Dropout(0.2 )(model)
 model = Dense(1,activation=act.selu )(model)
 model = Model(model_in,model)
 model.compile(loss='mse', optimizer='nadam',metrics=[rmse])
 return model<train_model>
xgb_tuned.fit(X,y )
history = model.fit(X_train,y_train,batch_size=10000,epochs=100,verbose=1,validation_data=(X_test,y_test))<compute_test_metric>
pred_fin_tuned = xgb_tuned.predict(test_fin)
 pred_tuned = pd.DataFrame(pred_fin_tuned, columns = ['Survived'],index = test_fin.index)
 pred_tuned
<predict_on_test><EOS> 
pred_tuned['Survived'].to_csv('My_Titanic_Predictions2.csv', index = True, header = True )