Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<filter>
import numpy as np
import pandas as pd
import os
import seaborn as sns
import matplotlib.pyplot as plt
train.loc[(train['H_Distance']!=0)&(train['fare_amount']==0)]<feature_engineering>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
scenario_3 = train.loc[(train['H_Distance']!=0)&(train['fare_amount']==0)]<sort_values>
def percent_null_value(df):
total = df.isnull().sum()
percent = df.isnull().sum() / len(df)
 df = pd.DataFrame([total, percent] ).T
df.columns = ['Total', 'Percent']
return df
scenario_3.sort_values('H_Distance', ascending=False )<feature_engineering>
def percent_unique_value(df, col):
total = df[col].value_counts()
percent = df[col].value_counts() / len(df)
 df = pd.DataFrame([total, percent] ).T
df.columns = ['Total', 'Percent']
return df
scenario_3['fare_amount'] = scenario_3.apply(
 lambda row:(( row['H_Distance'] * 1.56)+ 2.50), axis=1
)<filter>
df = percent_null_value(train)
 df.head(len(df))
train.loc[(train['H_Distance']==0)&(train['fare_amount']!=0)]<feature_engineering>
df = percent_null_value(test)
 df.head(len(df))
scenario_4 = train.loc[(train['H_Distance']==0)&(train['fare_amount']!=0)]<filter>
train[train.Embarked.isnull() ]
scenario_4.loc[(scenario_4['fare_amount']<=3.0)&(scenario_4['H_Distance']==0)]<filter>
train[(train.Pclass == 1)&(train.Sex == 'female')]['Embarked'].value_counts()
scenario_4.loc[(scenario_4['fare_amount']>3.0)&(scenario_4['H_Distance']==0)]<filter>
train.Embarked.fillna('S', inplace = True )
scenario_4_sub = scenario_4.loc[(scenario_4['fare_amount']>3.0)&(scenario_4['H_Distance']==0)]<feature_engineering>
test[test.Fare.isnull() ]
scenario_4_sub['H_Distance'] = scenario_4_sub.apply(
 lambda row:(( row['fare_amount']-2.50)/1.56), axis=1
)<drop_column>
missing_value = test[(test.Pclass == 3)&(test.Sex == 'male')]['Fare'].mean()
test.Fare.fillna(missing_value, inplace = True )
train = train.drop(['key','pickup_datetime'], axis = 1)
 test = test.drop(['key','pickup_datetime'], axis = 1 )<prepare_x_and_y>
data = [train, test]
for dataset in data:
dataset.Cabin.fillna('N', inplace = True)
 dataset['Cabin'] = dataset['Cabin'].apply(lambda x : x[0] )
x_train = train.iloc[:,train.columns!='fare_amount']
y_train = train['fare_amount'].values
x_test = test<choose_model_class>
survived = train.Survived
all_data = pd.concat([train, test] )
rf = RandomForestRegressor()
rf.fit(x_train, y_train)
 rf_predict = rf.predict(x_test)
<save_to_csv>
mapping = {'N': 0,
'C': 1,
'B': 2,
'D': 3,
'E': 4,
'A': 5,
'F': 6,
'G': 6,
'T': 6}
all_data['Cabin'] = all_data['Cabin'].map(mapping )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['fare_amount'] = rf_predict
submission.to_csv('submission_1.csv', index=False)
 submission.head(20 )<load_from_csv>
all_data['title'] = all_data['Name'].apply(lambda x : x.split('.')[0])
 all_data['title'] = all_data['title'].apply(lambda x : x.split(' ')[-1] )
train_df=pd.read_csv(".. /input/train.csv",nrows=1000000)
 test_df=pd.read_csv(".. /input/test.csv" )<count_missing_values>
mapping = {'Mr': 0,
'Miss': 1,
'Mrs': 2,
'Master': 3,
'Dr': 4,
'Rev': 4,
'Col': 4,
'Mlle': 4,
'Major': 4,
'Ms': 4,
'Mme': 4,
'Lady': 4,
'Countess':4,
'Don': 4,
'Sir': 4,
'Jonkheer': 4,
'Capt': 4,
'Dona': 4}
all_data['title'] = all_data['title'].map(mapping )
train_df.isnull().sum()<drop_column>
all_data['FamilySize'] = all_data['SibSp'] + all_data['Parch'] + 1
train_df= train_df.drop(train_df[train_df.isnull().any(1)].index, axis = 0 )<count_values>
mapping = {1: 1, 2: 2, 3: 3, 4: 0, 6: 0, 5: 0, 7: 0, 11: 0, 8: 0}
all_data['FamilySize'] = all_data['FamilySize'].map(mapping )
Counter(train_df['fare_amount']<0 )<drop_column>
all_data['is_alone'] = all_data['FamilySize'].apply(lambda x : 1 if x == 1 else 0 )
train_df= train_df.drop(train_df[train_df['fare_amount']<0].index, axis = 0)
 train_df.shape<count_values>
all_data['Age'].fillna(all_data.groupby(['title'])['Age'].transform('median'), inplace = True )
Counter(train_df['passenger_count']>6 )<drop_column>
def splitAgeColumns(x):
if x <= 16.0:
return 0
elif x > 16.0 and x <= 26.0:
return 1
elif x > 26.0 and x <= 36.0:
return 2
elif x > 36.0 and x <= 46.0:
return 3
else:
return 4
train_df= train_df.drop(train_df[train_df['passenger_count']>6].index, axis = 0)
 train_df.shape<count_values>
all_data['Age'] = all_data['Age'].apply(lambda x : splitAgeColumns(x))
Counter(train_df['pickup_latitude']<-90 )<count_values>
all_data['calculate_fare'] = all_data['Fare'] /(all_data['Parch'] + all_data['SibSp'] + 1 )
Counter(train_df['pickup_latitude']>90 )<drop_column>
def splitFareColumns(x):
if x <= 5.0:
return 0
elif x > 5.0 and x <= 10.0:
return 1
elif x > 10.0 and x <= 15.0:
return 2
elif x > 15.0 and x <= 20.0:
return 3
elif x > 20.0 and x <= 25.0:
return 4
else:
return 5
train_df = train_df.drop(((train_df[train_df['pickup_latitude']<-90])|(train_df[train_df['pickup_latitude']>90])).index, axis=0 )<count_values>
all_data['calculate_fare'] = all_data['calculate_fare'].apply(lambda x : splitFareColumns(x))
Counter(train_df['pickup_longitude']<-180 )<count_values>
def splitSibSpColumns(x):
if x <= 0.0:
return 0
elif x > 0.0 and x <= 1.0:
return 1
else:
return 2
def splitParchColumns(x):
if x <= 0.0:
return 0
elif x > 0.0 and x <= 2.0:
return 1
else:
return 2
Counter(train_df['pickup_longitude']>180 )<drop_column>
all_data['SibSp'] = all_data['SibSp'].apply(lambda x : splitSibSpColumns(x))
all_data['Parch'] = all_data['Parch'].apply(lambda x : splitParchColumns(x))
train_df = train_df.drop(( train_df[train_df['pickup_longitude']<-180] ).index, axis=0 )<data_type_conversions>
data = [train, test]
for dataset in data:
dataset['pclass_fare'] = dataset['Fare'] * dataset['Pclass']
train_df['key']=pd.to_datetime(train_df['key'])
 train_df['pickup_datetime']=pd.to_datetime(train_df['pickup_datetime'] )<data_type_conversions>
percent_null_value(train )
test_df['key']=pd.to_datetime(test_df['key'])
 test_df['pickup_datetime']=pd.to_datetime(test_df['pickup_datetime'] )<feature_engineering>
df = all_data.drop(['Name', 'Ticket', 'Survived', 'Fare'], axis = 1 )
data=[train_df,test_df]
for i in data:
i['date']=i['pickup_datetime'].dt.day
i['month']=i['pickup_datetime'].dt.month
i['day_of_week']=i['pickup_datetime'].dt.dayofweek
i['hour']=i['pickup_datetime'].dt.hour
i['year']=i['pickup_datetime'].dt.year 
<compute_test_metric>
mapping = {'male' : 0, 'female' : 1}
df['Sex'] = df['Sex'].map(mapping )
def sphere_distance(lat1,long1,lat2,long2):
data=[train_df,test_df]
for i in data:
R=6367
phi1 = np.radians(i[lat1])
 phi2 = np.radians(i[lat2])
 delta_phi = np.radians(i[lat2]-i[lat1])
 delta_lambda = np.radians(i[long2]-i[long1])
 a = np.sin(delta_phi / 2.0)** 2 + np.cos(phi1)* np.cos(phi2)* np.sin(delta_lambda / 2.0)** 2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))
d =(R * c)
 i['S_Distance'] = d
return d<sort_values>
def getOneHotEncode(df, col):
cat = pd.get_dummies(df[col], prefix = col)
 df = pd.concat([df, cat], axis = 1)
 df.drop([col], axis = 1, inplace = True)
 return df
train_df.sort_values(['S_Distance','fare_amount'], ascending=False )<filter>
one_hot_columns = ['Pclass', 'Cabin', 'Embarked', 'title']
dis_0 = train_df.loc[(train_df['S_Distance'] == 0), ['S_Distance']]
dis_1 = train_df.loc[(train_df['S_Distance'] > 0)&(train_df['S_Distance'] <= 10), ['S_Distance']]
dis_2 = train_df.loc[(train_df['S_Distance'] > 10)&(train_df['S_Distance'] <= 50), ['S_Distance']]
dis_3 = train_df.loc[(train_df['S_Distance'] > 50)&(train_df['S_Distance'] <= 100), ['S_Distance']]
dis_4 = train_df.loc[(train_df['S_Distance'] > 100)&(train_df['S_Distance'] <= 200), ['S_Distance']]
dis_5 = train_df.loc[(train_df['S_Distance'] > 200)&(train_df['S_Distance'] <= 300), ['S_Distance']]
dis_6 = train_df.loc[(train_df['S_Distance'] > 300)&(train_df['S_Distance'] <= 500), ['S_Distance']]
dis_7 = train_df.loc[(train_df['S_Distance'] > 500), ['S_Distance']]
dis_0['bins']='0'
dis_1['bins']='0-10'
dis_2['bins']='11-50'
dis_3['bins']='51-100'
dis_4['bins']='101-200'
dis_5['bins']='201-300'
dis_6['bins']='301-500'
dis_7['bins']='>500'
dis_bin=pd.concat([dis_0,dis_1,dis_2,dis_3,dis_4,dis_5,dis_6,dis_7])
 dis_bin<count_values>
df = df.sort_values('PassengerId' )
x=Counter(dis_bin['bins'])
 x<filter>
mapping = {'S' : 0, 'C' : 1, 'Q' : 2}
df['Embarked'] = df['Embarked'].map(mapping )
train_df.loc[(( train_df['pickup_latitude']==0)&(train_df['pickup_longitude']==0)) &(( train_df['dropoff_latitude']!=0)&(train_df['dropoff_longitude']!=0)) &(train_df['fare_amount']==0)]<filter>
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import accuracy_score
from sklearn.svm import SVC
from xgboost import XGBClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from catboost import CatBoostClassifier
train_df.loc[(( train_df['pickup_latitude']==0)&(train_df['pickup_longitude']==0)) &(( train_df['dropoff_latitude']!=0)&(train_df['dropoff_longitude']!=0)) &(train_df['fare_amount']==0)]<drop_column>
train, test = df[df.PassengerId <= 891], df[df.PassengerId > 891]
train, test = train.drop(['PassengerId'], axis = 1), test.drop(['PassengerId'], axis = 1)
 X_train, X_val, Y_train, Y_val = train_test_split(train, survived, test_size = 0.2, shuffle = True, random_state = 1 )
train_df = train_df.drop(train_df.loc[(( train_df['pickup_latitude']==0)&(train_df['pickup_longitude']==0)) &(( train_df['dropoff_latitude']!=0)&(train_df['dropoff_longitude']!=0)) &(train_df['fare_amount']==0)].index, axis=0 )<drop_column>
clf = SVC(kernel = 'linear')
 cross_val_score(clf, train, survived, cv = 5 )
train_df = train_df.drop(train_df.loc[(( train_df['pickup_latitude']==0)&(train_df['pickup_longitude']==0)) &(( train_df['dropoff_latitude']!=0)&(train_df['dropoff_longitude']!=0)) &(train_df['fare_amount']==0)].index, axis=0)
<filter>
clf = XGBClassifier()
cross_val_score(clf, train, survived, cv = 5 )
high_distance = train_df.loc[(train_df['S_Distance']>200)&(train_df['fare_amount']!=0)]<feature_engineering>
clf = DecisionTreeClassifier()
cross_val_score(clf, train, survived, cv = 5 )
high_distance['S_Distance'] = high_distance.apply(
 lambda row:(row['fare_amount'] - 2.50)/1.56,
axis=1
)<filter>
clf = RandomForestClassifier(n_estimators=400)
 cross_val_score(clf, train, survived, cv = 5 )
train_df[train_df['S_Distance']==0]<filter>
clf = CatBoostClassifier(verbose = False)
 cross_val_score(clf, train, survived, cv = 5 )
train_df[(train_df['S_Distance']==0)&(train_df['fare_amount']==0)]<drop_column>
clf = SVC()
clf.fit(X_train, Y_train )
train_df = train_df.drop(train_df[(train_df['S_Distance']==0)&(train_df['fare_amount']==0)].index, axis = 0 )<define_variables>
y_train_pred = clf.predict(X_train)
 y_val_pred = clf.predict(X_val)
 print('train accuracy: {}%'.format(accuracy_score(y_train_pred, Y_train)))
 print('validation accuracy: {}%'.format(accuracy_score(y_val_pred, Y_val)) )
rush_hour = train_df.loc[(((train_df['hour']>=6)&(train_df['hour']<=20)) &(( train_df['day_of_week']>=1)&(train_df['day_of_week']<=5)) &(train_df['S_Distance']==0)&(train_df['fare_amount'] < 2.5)) ]
rush_hour<drop_column>
y_test = clf.predict(test)
 result = pd.read_csv('.. /input/titanic/gender_submission.csv')
 result['Survived'] = y_test
result.to_csv('submission.csv', index = False )