Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<set_options>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
%env JOBLIB_TEMP_FOLDER=/tmp
%matplotlib inline 
<load_from_csv>
%matplotlib inline
train = pd.read_csv(".. /input/train.csv", parse_dates=['pickup_datetime'], usecols=range(1,8), nrows=700_000)
 test = pd.read_csv(".. /input/test.csv", parse_dates=['pickup_datetime'] )<drop_column>
train=pd.read_csv('.. /input/train.csv' )
train = train.dropna(how = 'any', axis = 'rows')
 train = train[train.eval('(fare_amount > 0)&(passenger_count <= 6)')]<filter>
family=[1 if i[6]>0 or i[7]>0 else 0 for i in train.values]
train['Family']=family
train = train[(train.pickup_longitude >= -77)& 
(train.pickup_longitude <= -70)& 
(train.dropoff_longitude >= -77)& 
(train.dropoff_longitude <= 70)& 
(train.pickup_latitude >= 35)& 
(train.pickup_latitude <= 45)& 
(train.dropoff_latitude >= 35)& 
(train.dropoff_latitude <= 45)
 ]<feature_engineering>
category=['child' if i[5]<=16 else i[4] for i in train.values]
train['Category']=category
train.pickup_datetime = train.pickup_datetime.dt.tz_localize('UTC')
 train.pickup_datetime = train.pickup_datetime.dt.tz_convert(tz.gettz('America/New_York'))
train['year'] = train.pickup_datetime.dt.year
train['dayofweek'] = train.pickup_datetime.dt.dayofweek
train['dayofyear'] = train.pickup_datetime.dt.dayofyear
train['hourofday'] = train.pickup_datetime.dt.hour
train = train.drop('pickup_datetime', axis=1 )<feature_engineering>
age_Pclass1=np.around(train[train['Pclass']==1]['Age'].mean())
 age_Pclass2=np.around(train[train['Pclass']==2]['Age'].mean())
 age_Pclass3=np.around(train[train['Pclass']==3]['Age'].mean())
 def impute_age(x):
if str(x[5] ).lower() =='nan':
if x[2]==1:
return age_Pclass1
elif x[2]==2:
return age_Pclass2
else:
return age_Pclass3
else:
return x[5]
train['Age']=train.apply(impute_age,axis=1 )
airports = {'jfk': [40.6441666, -73.7822222],
'laguardia': [40.7747222, -73.8719444],
'newark': [40.6897222, -74.175]}
pickup = train.apply(lambda x: distance.distance(( x.pickup_latitude, x.pickup_longitude),(airports.get('jfk')) ).miles, axis=1)
 dropoff = train.apply(lambda x: distance.distance(( x.dropoff_latitude, x.dropoff_longitude),(airports.get('jfk')) ).miles, axis=1)
 train['to_jfk'] = pd.concat(( pickup, dropoff), axis=1 ).min(axis=1)
 pickup = train.apply(lambda x: distance.distance(( x.pickup_latitude, x.pickup_longitude),(airports.get('laguardia')) ).miles, axis=1)
 dropoff = train.apply(lambda x: distance.distance(( x.dropoff_latitude, x.dropoff_longitude),(airports.get('laguardia')) ).miles, axis=1)
 train['to_laguardia'] = pd.concat(( pickup, dropoff), axis=1 ).min(axis=1)
 pickup = train.apply(lambda x: distance.distance(( x.pickup_latitude, x.pickup_longitude),(airports.get('newark')) ).miles, axis=1)
 dropoff = train.apply(lambda x: distance.distance(( x.dropoff_latitude, x.dropoff_longitude),(airports.get('newark')) ).miles, axis=1)
 train['to_newark'] = pd.concat(( pickup, dropoff), axis=1 ).min(axis=1)
 del pickup, dropoff<prepare_x_and_y>
embarked_mode=train['Embarked'].mode()
def impute_embarked(x):
if str(x ).lower() =='nan':
return embarked_mode[0]
else:
return x
train['Embarked']=train['Embarked'].apply(impute_embarked )
y = train.fare_amount
train = train.drop('fare_amount', axis=1)
<choose_model_class>
embarked_dummy=pd.get_dummies(train['Embarked'])
 embarked_dummy.head()
xgb_param = {'eval_metric': 'rmse',
'n_estimators': 3000,
'max_depth': 9,
'learning_rate': 0.1,
'subsample': 0.9,
'colsample_bytree': 0.8,
'gamma': 1e-4,
'reg_alpha': 1e-4,
'verbose': 0,
'n_jobs': -1
}
xgb_model = xgb.XGBRegressor(**xgb_param)
 xgb_model.fit(train, y)
 xgb.plot_importance(xgb_model )<drop_column>
category_dummy=pd.get_dummies(train['Category'])
 category_dummy.head()
test_key = test['key']
test = test.drop('key', axis=1)
<feature_engineering>
train=pd.concat([train,embarked_dummy],axis=1)
 train=pd.concat([train,category_dummy],axis=1)
 train.head()
test.pickup_datetime = test.pickup_datetime.dt.tz_localize('UTC')
 test.pickup_datetime = test.pickup_datetime.dt.tz_convert(tz.gettz('America/New_York'))
test['year'] = test.pickup_datetime.dt.year
test['dayofweek'] = test.pickup_datetime.dt.dayofweek
test['dayofyear'] = test.pickup_datetime.dt.dayofyear
test['hourofday'] = test.pickup_datetime.dt.hour
test = test.drop('pickup_datetime', axis=1)
 test['distance'] = test.apply(lambda x: distance.distance(( x.pickup_latitude, x.pickup_longitude),(x.dropoff_latitude, x.dropoff_longitude)).miles, axis = 1)
 pickup = test.apply(lambda x: distance.distance(( x.pickup_latitude, x.pickup_longitude),(airports.get('jfk')) ).miles, axis=1)
 dropoff = test.apply(lambda x: distance.distance(( x.dropoff_latitude, x.dropoff_longitude),(airports.get('jfk')) ).miles, axis=1)
 test['to_jfk'] = pd.concat(( pickup, dropoff), axis = 1 ).min(axis=1)
 pickup = test.apply(lambda x: distance.distance(( x.pickup_latitude, x.pickup_longitude),(airports.get('laguardia')) ).miles, axis=1)
 dropoff = test.apply(lambda x: distance.distance(( x.dropoff_latitude, x.dropoff_longitude),(airports.get('laguardia')) ).miles, axis=1)
 test['to_laguardia'] = pd.concat(( pickup, dropoff), axis = 1 ).min(axis=1)
 pickup = test.apply(lambda x: distance.distance(( x.pickup_latitude, x.pickup_longitude),(airports.get('newark')) ).miles, axis=1)
 dropoff = test.apply(lambda x: distance.distance(( x.dropoff_latitude, x.dropoff_longitude),(airports.get('newark')) ).miles, axis=1)
 test['to_newark'] = pd.concat(( pickup, dropoff), axis = 1 ).min(axis=1)
 del pickup, dropoff<predict_on_test>
X=train.iloc[:,[2,5,9,12,14,15,17,18]].values
y=train.iloc[:,1].values
xgb_predict = xgb_model.predict(test )<save_to_csv>
scale=StandardScaler()
xgb_submission = pd.DataFrame({ 'key': test_key,
'fare_amount': xgb_predict })
 xgb_submission.to_csv("xgb_submission.csv", index=False )<import_modules>
X=scale.fit_transform(X )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import geopy.distance as geo
import datetime
import time
import calendar
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report,confusion_matrix
from sklearn import preprocessing
from sklearn import metrics<set_options>
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=0 )
%matplotlib inline<load_from_csv>
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import AdaBoostClassifier
df = pd.read_csv('.. /input/train.csv', nrows = 2000000 )<drop_column>
dtree=DecisionTreeClassifier()
ml_algo=AdaBoostClassifier(base_estimator=dtree,n_estimators=2500,learning_rate=0.01 )
unwanted_indices = df[(abs(df['pickup_latitude'])> 90)| 
(abs(df['dropoff_latitude'])> 90)
 ].index
df.drop(list(unwanted_indices), inplace=True)
 del unwanted_indices
unwanted_indices = df[(abs(df['pickup_longitude'])> 180)| 
(abs(df['dropoff_longitude'])> 180)
 ].index
df.drop(list(unwanted_indices), inplace=True )<filter>
ml_algo.fit(X_train,y_train )
df[(abs(df['pickup_longitude'])> 180)| 
(abs(df['dropoff_longitude'])> 180)
 ].index<filter>
confusion_matrix(y_test,ml_algo.predict(X_test))
df[(abs(df['pickup_latitude'])> 90)| 
(abs(df['dropoff_latitude'])> 90)
 ].index<drop_column>
ml_algo.score(X_test,y_test )
unwanted_indices = df[df['dropoff_latitude'].isnull() ].index
df.drop(list(unwanted_indices), inplace=True)
 unwanted_indices = df[df['dropoff_longitude'].isnull() ].index
df.drop(list(unwanted_indices), inplace=True )<drop_column>
ml_algo.score(X_train,y_train )
if 'unwanted_indices' in globals() : del unwanted_indices
unwanted_indices = df[(df['passenger_count'] > 6)|(df['passenger_count'] == 0)].index
df.drop(list(unwanted_indices), inplace=True )<drop_column>
ml_algo.score(X,y )
unwanted_indices = df[df['fare_amount']<=0].index
df.drop(list(unwanted_indices), inplace=True )<drop_column>
ml_algo.fit(X,y )
<normalization>
test=pd.read_csv('.. /input/test.csv' )
df['Travel distance'] = list(map(lambda x1,x2,x3,x4:
geo.distance(( x3,x1),(x4,x2)).miles,
df['pickup_longitude'], df['dropoff_longitude'],
df['pickup_latitude'], df['dropoff_latitude']))<data_type_conversions>
test['Fare'].fillna(test['Fare'].mean() ,inplace=True )
newtimeStamp = pd.to_datetime(df['pickup_datetime'].apply(lambda x: x.split(' UTC')[0])) 
<feature_engineering>
age_Pclass1=np.around(test[test['Pclass']==1]['Age'].mean())
 age_Pclass2=np.around(test[test['Pclass']==2]['Age'].mean())
 age_Pclass3=np.around(test[test['Pclass']==3]['Age'].mean())
 def impute_age(x):
if str(x[4] ).lower() =='nan':
if x[1]==1:
return age_Pclass1
elif x[1]==2:
return age_Pclass2
else:
return age_Pclass3
else:
return x[4]
test['Age']=test.apply(impute_age,axis=1 )
df['Hour'] = newtimeStamp.apply(lambda x : x.hour)
 df['Month'] = newtimeStamp.apply(lambda x : x.month)
 df['Date'] = newtimeStamp.apply(lambda x : x.date() )<define_variables>
family=[1 if i[5]>0 or i[6]>0 else 0 for i in test.values]
test['Family']=family
for x in df['pickup_datetime'] :
y = x.split(' ')[2]
if(y!='UTC'):
print(y )<feature_engineering>
category=['child' if i[4]<=16 else i[3] for i in test.values]
test['Category']=category
Day_of_Week=newtimeStamp.apply(
 lambda x :
calendar.day_name[datetime.date(x.year,x.month,x.day)
.weekday() ] 
)
 dmap = {0:'Mon',1:'Tue',2:'Wed',3:'Thu',4:'Fri',5:'Sat',6:'Sun'}
Day_of_Week = newtimeStamp.apply(
 lambda x :
dmap[datetime.date(x.year,x.month,x.day)
.weekday() ] 
)
 df['Day of Week'] = Day_of_Week<feature_engineering>
embarked_dummy=pd.get_dummies(test['Embarked'])
 embarked_dummy.head()
df['qty'] = df['pickup_datetime'].apply(lambda x: 1 )<feature_engineering>
category_dummy=pd.get_dummies(test['Category'])
 category_dummy.head()
df['timestamp'] = newtimeStamp.apply(lambda x : time.mktime(
(x.year,x.month,x.day,
x.hour,x.minute,x.second,
1,x.day,-1))
)<groupby>
test=pd.concat([test,embarked_dummy],axis=1)
 test=pd.concat([test,category_dummy],axis=1)
 test.head()
dfm=df.groupby(['Day of Week','Hour'] ).count()
fp=dfm['qty'].unstack()<drop_column>
X_TEST=test.iloc[:,[1,4,8,11,13,14,16,17]].values
dftrain = df.drop([
'key',
'pickup_datetime',
'Hour',
'Month',
'Date',
'Day of Week',
'qty'
],axis=1 ).dropna()
dftrain.head(2 )<prepare_x_and_y>
scale=StandardScaler()
X_train = dftrain.drop('fare_amount',axis=1)
 y_train = dftrain['fare_amount']<split>
X_TEST=scale.fit_transform(X_TEST )
XX_train, XX_test, yy_train, yy_test = train_test_split(X_train, y_train, test_size=0.30 )<compute_test_metric>
pred=ml_algo.predict(X_TEST ).reshape(-1,1 )
<compute_test_metric>
id=test.iloc[:,0].values.reshape(-1,1)
 output=np.concatenate(( id,pred),axis=1 )
<compute_test_metric>
submission=pd.DataFrame(output,columns=['PassengerId','Survived'] )
<choose_model_class><EOS> 
submission.to_csv('Submission.csv',index=False )