Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<prepare_output>
%matplotlib inline
test_df=preparedataset(dataset_test)
 test_df.head(50 )<save_to_csv>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
test_df.to_csv('test_preprocessed.csv' )<split>
train['Survived'].value_counts(normalize=True )
y = df.fare_amount
X=df.drop('fare_amount',axis=1)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y )<find_best_params>
train['Survived'].groupby(train['Pclass'] ).mean()
warnings.filterwarnings("ignore")
 result={}
best_istemator=0
best_learing_rate=0
best_mae=100000000000
for lr in [X / 100 for X in range(10,50, 5)]:
for ns in range(200,701,50):
my_model = XGBRegressor(n_estimators=ns, learning_rate=lr,n_jobs=4)
 my_model.fit(X_train, y_train)
 predictions = my_model.predict(X_valid)
 mae=mean_absolute_error(predictions, y_valid)
 if(mae < best_mae):
best_mae=mae
best_istemator=ns
best_learing_rate=lr
print("better found")
 print(ns , lr, mae)
 result[(ns,lr)]=mae
my_model_2 = XGBRegressor(n_estimators=best_istemator, learning_rate=best_learing_rate, n_jobs=4)
 my_model_2.fit(X_train,y_train)
 predictions_2 = my_model_2.predict(X_valid)
 mae_2 = mean_absolute_error(y_valid, predictions_2)
 print("best_istemator:" , best_istemator)
 print("best_learing_rate:" , best_learing_rate)
 print("Mean Absolute Error:" , mae_2)
<save_to_csv>
train['Name_Title'] = train['Name'].apply(lambda x: x.split(',')[1] ).apply(lambda x: x.split() [0])
 train['Name_Title'].value_counts()
my_model_2 = XGBRegressor(n_estimators=700, learning_rate=0.2, n_jobs=4)
 my_model_2.fit(X_train,y_train)
 test_preds = my_model_2.predict(test_df)
 output = pd.DataFrame({'key': test_df.index,
'fare_amount': test_preds})
 output.to_csv('submission.csv', index=False )<load_from_csv>
train['Survived'].groupby(train['Name_Title'] ).mean()
train = pd.read_csv(".. /input/train.csv", nrows = 1000000)
 test = pd.read_csv(".. /input/test.csv" )<sort_values>
train['Name_Len'] = train['Name'].apply(lambda x: len(x))
train['Survived'].groupby(pd.qcut(train['Name_Len'],5)).mean()
train.isnull().sum().sort_values(ascending=False )<sort_values>
pd.qcut(train['Name_Len'],5 ).value_counts()
test.isnull().sum().sort_values(ascending=False )<correct_missing_values>
train['Sex'].value_counts(normalize=True )
train = train.dropna()<count_values>
train['Survived'].groupby(train['Sex'] ).mean()
Counter(train['fare_amount']<0 )<drop_column>
train['Survived'].groupby(train['Age'].isnull() ).mean()
train = train.drop(train[train['fare_amount']<0].index, axis=0)
 train.shape<sort_values>
train['Survived'].groupby(pd.qcut(train['Age'],5)).mean()
train['fare_amount'].sort_values(ascending=False )<filter>
pd.qcut(train['Age'],5 ).value_counts()
train[train['passenger_count']>6]<drop_column>
train['Survived'].groupby(train['SibSp'] ).mean()
train = train.drop(train[train['passenger_count']==208].index,axis=0 )<filter>
train['SibSp'].value_counts()
train[train['pickup_latitude']<-90]<filter>
train['Survived'].groupby(train['Parch'] ).mean()
train[train['pickup_latitude']>90]<drop_column>
train['Parch'].value_counts()
train = train.drop(((train[train['pickup_latitude']<-90])|(train[train['pickup_latitude']>90])).index, axis=0 )<filter>
train['Ticket_Len'] = train['Ticket'].apply(lambda x: len(x))
train[train['pickup_longitude']<-180]<filter>
train['Ticket_Len'].value_counts()
train[train['pickup_longitude']>180]<drop_column>
train['Ticket_Lett'] = train['Ticket'].apply(lambda x: str(x)[0] )
train = train.drop(((train[train['pickup_longitude']<-180])|(train[train['pickup_longitude']>180])).index, axis=0 )<filter>
train['Ticket_Lett'].value_counts()
train[train['dropoff_latitude']<-90]<filter>
train.groupby(['Ticket_Lett'])['Survived'].mean()
train[train['dropoff_latitude']>90]<drop_column>
pd.qcut(train['Fare'], 3 ).value_counts()
train = train.drop(((train[train['dropoff_latitude']<-90])|(train[train['dropoff_latitude']>90])).index, axis=0 )<filter>
train['Survived'].groupby(pd.qcut(train['Fare'], 3)).mean()
train[train['dropoff_latitude']<-180]|train[train['dropoff_latitude']>180]<data_type_conversions>
train['Cabin_Letter'] = train['Cabin'].apply(lambda x: str(x)[0] )
train['key'] = pd.to_datetime(train['key'])
 train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'] )<data_type_conversions>
train['Cabin_Letter'].value_counts()
test['key'] = pd.to_datetime(test['key'])
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'] )<compute_test_metric>
train['Survived'].groupby(train['Cabin_Letter'] ).mean()
def haversine_distance(lat1, long1, lat2, long2):
data = [train, test]
for i in data:
R = 6371
phi1 = np.radians(i[lat1])
 phi2 = np.radians(i[lat2])
 delta_phi = np.radians(i[lat2]-i[lat1])
 delta_lambda = np.radians(i[long2]-i[long1])
 a = np.sin(delta_phi / 2.0)** 2 + np.cos(phi1)* np.cos(phi2)* np.sin(delta_lambda / 2.0)** 2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))
d =(R * c)
 i['H_Distance'] = d
return d<concatenate>
train['Cabin_num'] = train['Cabin'].apply(lambda x: str(x ).split(' ')[-1][1:])
 train['Cabin_num'].replace('an', np.NaN, inplace = True)
 train['Cabin_num'] = train['Cabin_num'].apply(lambda x: int(x)if not pd.isnull(x)and x != '' else np.NaN )
haversine_distance('pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude' )<feature_engineering>
pd.qcut(train['Cabin_num'],3 ).value_counts()
data = [train,test]
for i in data:
i['Year'] = i['pickup_datetime'].dt.year
i['Month'] = i['pickup_datetime'].dt.month
i['Date'] = i['pickup_datetime'].dt.day
i['Day of Week'] = i['pickup_datetime'].dt.dayofweek
i['Hour'] = i['pickup_datetime'].dt.hour<sort_values>
train['Survived'].groupby(pd.qcut(train['Cabin_num'], 3)).mean()
train.sort_values(['H_Distance','fare_amount'], ascending=False )<filter>
train['Embarked'].value_counts()
train.loc[(( train['pickup_latitude']==0)&(train['pickup_longitude']==0)) &(( train['dropoff_latitude']!=0)&(train['dropoff_longitude']!=0)) &(train['fare_amount']==0)]<drop_column>
train['Embarked'].value_counts(normalize=True )
train = train.drop(train.loc[(( train['pickup_latitude']==0)&(train['pickup_longitude']==0)) &(( train['dropoff_latitude']!=0)&(train['dropoff_longitude']!=0)) &(train['fare_amount']==0)].index, axis=0 )<filter>
train['Survived'].groupby(train['Embarked'] ).mean()
test.loc[(( test['pickup_latitude']==0)&(test['pickup_longitude']==0)) &(( test['dropoff_latitude']!=0)&(test['dropoff_longitude']!=0)) ] 
<filter>
def names(train, test):
for i in [train, test]:
i['Name_Len'] = i['Name'].apply(lambda x: len(x))
i['Name_Title'] = i['Name'].apply(lambda x: x.split(',')[1] ).apply(lambda x: x.split() [0])
 del i['Name']
return train, test
train.loc[(( train['pickup_latitude']!=0)&(train['pickup_longitude']!=0)) &(( train['dropoff_latitude']==0)&(train['dropoff_longitude']==0)) &(train['fare_amount']==0)]<drop_column>
def age_impute(train, test):
for i in [train, test]:
i['Age_Null_Flag'] = i['Age'].apply(lambda x: 1 if pd.isnull(x)else 0)
 data = train.groupby(['Name_Title', 'Pclass'])['Age']
i['Age'] = data.transform(lambda x: x.fillna(x.mean()))
return train, test
train = train.drop(train.loc[(( train['pickup_latitude']!=0)&(train['pickup_longitude']!=0)) &(( train['dropoff_latitude']==0)&(train['dropoff_longitude']==0)) &(train['fare_amount']==0)].index, axis=0 )<filter>
def fam_size(train, test):
for i in [train, test]:
i['Fam_Size'] = np.where(( i['SibSp']+i['Parch'])== 0 , 'Solo',
np.where(( i['SibSp']+i['Parch'])<= 3,'Nuclear', 'Big'))
del i['SibSp']
del i['Parch']
return train, test
test.loc[(( test['pickup_latitude']!=0)&(test['pickup_longitude']!=0)) &(( test['dropoff_latitude']==0)&(test['dropoff_longitude']==0)) ]<filter>
def cabin(train, test):
for i in [train, test]:
i['Cabin_Letter'] = i['Cabin'].apply(lambda x: str(x)[0])
 del i['Cabin']
return train, test
high_distance = train.loc[(train['H_Distance']>200)&(train['fare_amount']!=0)]<feature_engineering>
def cabin_num(train, test):
for i in [train, test]:
i['Cabin_num1'] = i['Cabin'].apply(lambda x: str(x ).split(' ')[-1][1:])
 i['Cabin_num1'].replace('an', np.NaN, inplace = True)
 i['Cabin_num1'] = i['Cabin_num1'].apply(lambda x: int(x)if not pd.isnull(x)and x != '' else np.NaN)
 i['Cabin_num'] = pd.qcut(train['Cabin_num1'],3)
 train = pd.concat(( train, pd.get_dummies(train['Cabin_num'], prefix = 'Cabin_num')) , axis = 1)
 test = pd.concat(( test, pd.get_dummies(test['Cabin_num'], prefix = 'Cabin_num')) , axis = 1)
 del train['Cabin_num']
del test['Cabin_num']
del train['Cabin_num1']
del test['Cabin_num1']
return train, test
high_distance['H_Distance'] = high_distance.apply(
 lambda row:(row['fare_amount'] - 2.50)/1.56,
axis=1
)<train_model>
def embarked_impute(train, test):
for i in [train, test]:
i['Embarked'] = i['Embarked'].fillna('S')
 return train, test
train.update(high_distance )<filter>
test['Fare'].fillna(train['Fare'].mean() , inplace = True )
train[train['H_Distance']==0]<filter>
def dummies(train, test, columns = ['Pclass', 'Sex', 'Embarked', 'Ticket_Lett', 'Cabin_Letter', 'Name_Title', 'Fam_Size']):
for column in columns:
train[column] = train[column].apply(lambda x: str(x))
test[column] = test[column].apply(lambda x: str(x))
good_cols = [column+'_'+i for i in train[column].unique() if i in test[column].unique() ]
train = pd.concat(( train, pd.get_dummies(train[column], prefix = column)[good_cols]), axis = 1)
 test = pd.concat(( test, pd.get_dummies(test[column], prefix = column)[good_cols]), axis = 1)
 del train[column]
del test[column]
return train, test
train[(train['H_Distance']==0)&(train['fare_amount']==0)]<drop_column>
def drop(train, test, bye = ['PassengerId']):
for i in [train, test]:
for z in bye:
del i[z]
return train, test
train = train.drop(train[(train['H_Distance']==0)&(train['fare_amount']==0)].index, axis = 0 )<define_variables>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 train, test = names(train, test)
 train, test = age_impute(train, test)
 train, test = cabin_num(train, test)
 train, test = cabin(train, test)
 train, test = embarked_impute(train, test)
 train, test = fam_size(train, test)
 test['Fare'].fillna(train['Fare'].mean() , inplace = True)
 train, test = ticket_grouped(train, test)
 train, test = dummies(train, test, columns = ['Pclass', 'Sex', 'Embarked', 'Ticket_Lett',
'Cabin_Letter', 'Name_Title', 'Fam_Size'])
 train, test = drop(train, test )
rush_hour = train.loc[(((train['Hour']>=6)&(train['Hour']<=20)) &(( train['Day of Week']>=1)&(train['Day of Week']<=5)) &(train['H_Distance']==0)&(train['fare_amount'] < 2.5)) ]
rush_hour<drop_column>
rf = RandomForestClassifier(criterion='gini',
n_estimators=700,
min_samples_split=10,
min_samples_leaf=1,
max_features='auto',
oob_score=True,
random_state=1,
n_jobs=-1)
 rf.fit(train.iloc[:, 1:], train.iloc[:, 0])
 print("%.4f" % rf.oob_score_ )
train=train.drop(rush_hour.index, axis=0 )<feature_engineering>
pd.concat(( pd.DataFrame(train.iloc[:, 1:].columns, columns = ['variable']),
pd.DataFrame(rf.feature_importances_, columns = ['importance'])) ,
axis = 1 ).sort_values(by='importance', ascending = False)[:20]
<filter><EOS> 
predictions = rf.predict(test)
 predictions = pd.DataFrame(predictions, columns=['Survived'])
 test = pd.read_csv('.. /input/titanic/test.csv')
 predictions = pd.concat(( test.iloc[:, 0], predictions), axis = 1)
 predictions.to_csv('titanicUI1.csv', sep=",", index = False )