Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<count_missing_values>
%matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 GradientBoostingClassifier, ExtraTreesClassifier)
 print(os.listdir(".. /input"))
print("Check The NA value in test data")
 for i in list(test.columns[test.dtypes != "O"]):
print(i, ":", sum(test[i].isna()))<count_unique_values>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
len(pd.unique(test.matchId)) , sum(test.groupby("matchId" ).size() < 9 )<merge>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
temp = pd.DataFrame(test.groupby("matchId" ).size() , columns=["player"])
 temp.reset_index(level=0, inplace=True)
 test = test.merge(temp, left_on="matchId", right_on="matchId" )<feature_engineering>
train.columns[train.isnull().any() ].tolist()
test["matchType_1"] = "-"
test.loc[(test.matchType == "solo-fpp")| 
(test.matchType == "solo")| 
(test.matchType == "normal-solo-fpp")| 
(test.matchType == "normal-solo"), "matchType_1"] = "solo"
test.loc[(test.matchType == "duo-fpp")| 
(test.matchType == "duo")| 
(test.matchType == "normal-duo-fpp")| 
(test.matchType == "normal-duo"), "matchType_1"] = "duo"
test.loc[(test.matchType == "squad-fpp")| 
(test.matchType == "squad")| 
(test.matchType == "normal-squad-fpp")| 
(test.matchType == "normal-squad"), "matchType_1"] = "squad"
test.loc[(test.matchType == "flarefpp")| 
(test.matchType == "flaretpp")| 
(test.matchType == "crashfpp")| 
(test.matchType == "crashtpp"), "matchType_1"] = "etc"<feature_engineering>
print(train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean() )
test["matchType_2"] = "-"
test.loc[(test.matchType == "solo-fpp")| 
(test.matchType == "duo-fpp")| 
(test.matchType == "squad-fpp")| 
(test.matchType == "normal-solo-fpp")| 
(test.matchType == "normal-duo-fpp")| 
(test.matchType == "normal-squad-fpp")| 
(test.matchType == "crashfpp")| 
(test.matchType == "flarefpp"), "matchType_2"] = "fpp"
test.loc[(test.matchType == "solo")| 
(test.matchType == "duo")| 
(test.matchType == "squad")| 
(test.matchType == "normal-solo")| 
(test.matchType == "normal-duo")| 
(test.matchType == "normal-squad")| 
(test.matchType == "crashtpp")| 
(test.matchType == "flaretpp"), "matchType_2"] = "tpp"<feature_engineering>
print(train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean() )
test["solo"] = 0
test["duo"] = 0
test["squad"] = 0
test["etc"] = 0
test.loc[test.matchType_1 == "solo", "solo"] = 1
test.loc[test.matchType_1 == "duo", "duo"] = 1
test.loc[test.matchType_1 == "squad", "squad"] = 1
test.loc[test.matchType_1 == "etc", "etc"] = 1<feature_engineering>
print(train[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).count() )
test["fpp"] = 0
test["tpp"] = 0
test.loc[test.matchType_2 == "fpp", "fpp"] = 1
test.loc[test.matchType_2 == "tpp", "tpp"] = 1<feature_engineering>
full_data = [train, test]
<feature_engineering>
for dataset in full_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
print(train[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean() )
<categorify>
for dataset in full_data:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
<create_dataframe>
print(train[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean() )
<feature_engineering>
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 print(train[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean() )
<rename_columns>
for dataset in full_data:
avg_age = round(dataset['Age'].mean() ,2)
 dataset['Age'][np.isnan(dataset['Age'])] = avg_age
train['CategoricalAge'] = pd.cut(train['Age'], 5)
 print(train[['CategoricalAge', 'Survived']].groupby(['CategoricalAge'], as_index=False ).mean() )
test.set_index("Id", inplace=True)
 test.index.name = "Id"<prepare_x_and_y>
for dataset in full_data:
dataset['Fare'] = dataset['Fare'].fillna(train['Fare'].median())
 train['CategoricalFare'] = pd.qcut(train['Fare'], 4)
 print(train[['CategoricalFare', 'Survived']].groupby(['CategoricalFare'], as_index=False ).mean() )
temp_1 = test.loc[:, feature_1]
temp_2 = test.loc[:, feature_2]<feature_engineering>
def get_title(name):
title_search = name.split(",")[1].split(".")[0].strip()
if title_search:
return title_search
else:
return ""
for dataset in full_data:
dataset['Title'] = dataset['Name'].apply(get_title )
temp_1.groupby("matchId" ).transform(minmax)
 for i in temp_2.columns[:4]:
temp_2[i] =(temp_2[i] - min(temp_2[i])) /(max(temp_2[i])- min(temp_2[i]))<merge>
for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 print(train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean() )
X = pd.merge(temp_1, temp_2, on="Id")
 X = pd.merge(X, test.loc[:, ["matchType_1", "winPlacePerc"]], on="Id" )<feature_engineering>
for dataset in full_data:
dataset['Sex'] = dataset['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] 						 = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] 							 = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 dataset.loc[ dataset['Age'] <= 16, 'Age'] 					 = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
drop_elements = ['Name', 'Ticket', 'Cabin', 'SibSp',\
'Parch', 'FamilySize']
train = train.drop(drop_elements, axis = 1)
 train = train.drop(['CategoricalAge', 'CategoricalFare'], axis = 1)
 test = test.drop(drop_elements, axis = 1)
 print(train.head(10))
<feature_engineering>
cols=['Pclass', 'Sex', 'Age', 'Fare', 'Embarked',
'IsAlone', 'Title']
X=train[cols]
Y=train['Survived']
<drop_column>
logreg = LogisticRegression()
logreg.fit(X, Y)
 logreg.score(X, Y )
test = X
test.reset_index()<predict_on_test>
train1, test1 = train_test_split(train, test_size=0.2 )
result_1 = model_1.predict(test.loc[test.matchType_1 == "solo", list_feat])
 result_2 = model_2.predict(test.loc[test.matchType_1 == "duo", list_feat])
 result_3 = model_3.predict(test.loc[test.matchType_1 == "squad", list_feat])
 result_4 = model_4.predict(test.loc[test.matchType_1 == "etc", list_feat] )<create_dataframe>
X1=train1[cols]
Y1=train1['Survived']
logreg.fit(X1, Y1)
 logreg.score(X1, Y1 )
temp = pd.DataFrame(test.loc[test.matchType_1 == "solo", "Id"] ).append(pd.DataFrame(test.loc[test.matchType_1 == "duo", "Id"])).append(pd.DataFrame(test.loc[test.matchType_1 == "squad", "Id"])).append(pd.DataFrame(test.loc[test.matchType_1 == "etc", "Id"]))
_ = pd.DataFrame(result_1, columns = ["winPlacePerc"] ).append(pd.DataFrame(result_2, columns = ["winPlacePerc"])).append(pd.DataFrame(result_3, columns = ["winPlacePerc"])).append(pd.DataFrame(result_4, columns = ["winPlacePerc"]))<concatenate>
X1_test = test1[cols]
Y1_test = test1['Survived']
Y3test_pred = logreg.predict(X1_test)
 print('Accuracy of logistic regression classifier on test set: {:.2f}'.format(logreg.score(X1_test, Y1_test)) )
result = pd.concat([temp.reset_index(drop=True), _.reset_index(drop=True)], axis=1 )<count_missing_values>
from sklearn.ensemble import RandomForestClassifier
np.sum(result.winPlacePerc.isna() )<count_values>
model = RandomForestClassifier(random_state=42 )
np.sum(result.winPlacePerc < 0 )<count_values>
param_grid = {
'n_estimators': [200, 500],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth' : [4,5,6,7,8],
'criterion' :['gini', 'entropy']
}
np.sum(result.winPlacePerc > 1 )<save_to_csv>
CV_rfc = GridSearchCV(estimator=model, param_grid=param_grid, cv= 5)
 CV_rfc.fit(X1, Y1 )
result.loc[result.winPlacePerc.isna() , "winPlacePerc"] = 0
result.loc[result.winPlacePerc < 0, "winPlacePerc"] = 0
result.loc[result.winPlacePerc > 1, "winPlacePerc"] = 1
result.to_csv('submission.csv', index=False )<set_options>
CV_rfc.best_params_
!pip install ultimate
gc.enable()<define_variables>
rfc1=RandomForestClassifier(random_state=42, max_features='auto', n_estimators= 200, max_depth=5, criterion='gini' )
INPUT_DIR = ".. /input/"<load_from_csv>
rfc1.fit(X1, Y1 )
def feature_engineering(is_train=True):
if is_train:
print("processing train_V2.csv")
 df = pd.read_csv(INPUT_DIR + 'train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test_V2.csv")
 df = pd.read_csv(INPUT_DIR + 'test_V2.csv')
 state('totalDistance')
 s = timer()
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
e = timer()
state('totalDistance', False, e - s)
 state('rankPoints')
 s = timer()
df['rankPoints'] = np.where(df['rankPoints'] <= 0 ,0 , df['rankPoints'])
 e = timer()
state('rankPoints', False, e-s)
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchDuration")
 features.remove("matchType")
 y = None
if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = np.array(df_out, dtype=np.float64)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y<normalization>
pred=rfc1.predict(X1_test )
%%time
x_train, y = feature_engineering(True)
 scaler = preprocessing.MinMaxScaler(feature_range=(-1, 1), copy=False ).fit(x_train )<train_model>
print("Accuracy for Random Forest on CV data: ",accuracy_score(Y1_test,pred))
print("x_train", x_train.shape, x_train.max() , x_train.min())
 scaler.transform(x_train)
 print("x_train", x_train.shape, x_train.max() , x_train.min() )<choose_model_class>
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, BaggingClassifier, ExtraTreesClassifier
from sklearn.tree import DecisionTreeClassifier
from xgboost import XGBClassifier, plot_importance
from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
%%time
NN_model = Sequential()
NN_model.add(Dense(x_train.shape[1], input_dim = x_train.shape[1], activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(1, activation='linear'))
NN_model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mean_absolute_error'])
 NN_model.summary()<train_model>
clf_rf = RandomForestClassifier()
clf_et = ExtraTreesClassifier()
clf_bc = BaggingClassifier()
clf_ada = AdaBoostClassifier()
clf_dt = DecisionTreeClassifier()
clf_xg = XGBClassifier()
clf_lr = LogisticRegression()
clf_svm = SVC()
%%time
NN_model.fit(x=x_train, y=y, batch_size=1000,
epochs=30, verbose=1, callbacks=callbacks_list,
validation_split=0.15, validation_data=None, shuffle=True,
class_weight=None, sample_weight=None, initial_epoch=0,
steps_per_epoch=None, validation_steps=None)
 del x_train, y
gc.collect()<normalization>
npX = np.array(X ).copy()
npy = np.array(Y ).copy()
x_test, _ = feature_engineering(False)
 scaler.transform(x_test)
 print("x_test", x_test.shape, x_test.max() , x_test.min())
 np.clip(x_test, out=x_test, a_min=-1, a_max=1)
 print("x_test", x_test.shape, x_test.max() , x_test.min() )<predict_on_test>
Classifiers = ['RandomForest','ExtraTrees','Bagging','AdaBoost','DecisionTree','XGBoost','LogisticRegression','SVM']
scores = []
models = [clf_rf, clf_et, clf_bc, clf_ada, clf_dt, clf_xg, clf_lr, clf_svm]
for model in models:
score = cross_val_score(model, npX, npy, scoring = 'accuracy', cv = 10, n_jobs = -1 ).mean()
scores.append(score )
%%time
pred = NN_model.predict(x_test)
 del x_test
gc.collect()<prepare_output>
mode = pd.DataFrame(scores, index = Classifiers, columns = ['score'] ).sort_values(by = 'score',
ascending = False )
pred = pred.reshape(-1)
 pred =(pred + 1)/ 2<load_from_csv>
parameters_xg = {'max_depth':[3,6,7], 'learning_rate': [0.1,0.2], 'n_estimators': [300,200],
'min_child_weight': [4], 'reg_alpha': [6,0], 'reg_lambda': [1,8],'max_delta_step':[2],
'gamma':[0],'seed':[1]}
parameters_svm = {'C':[0.9,0.01],'kernel':['rbf','linear'], 'gamma':[0,0.1,'auto'], 'probability':[True,False],
'random_state':[0,7,16],'decision_function_shape':['ovo','ovr'],'degree':[3,4,10]}
df_test = pd.read_csv(INPUT_DIR + 'test_V2.csv' )<feature_engineering>
def grid(model,parameters):
grid = GridSearchCV(estimator = model, param_grid = parameters, cv = 10,
scoring = 'accuracy')
 grid.fit(npX,npy)
 return grid.best_score_, grid.best_estimator_.get_params()
%%time
print("fix winPlacePerc")
 for i in range(len(df_test)) :
winPlacePerc = pred[i]
maxPlace = int(df_test.iloc[i]['maxPlace'])
 if maxPlace == 0:
winPlacePerc = 0.0
elif maxPlace == 1:
winPlacePerc = 1.0
else:
gap = 1.0 /(maxPlace - 1)
 winPlacePerc = round(winPlacePerc / gap)* gap
if winPlacePerc < 0: winPlacePerc = 0.0
if winPlacePerc > 1: winPlacePerc = 1.0
pred[i] = winPlacePerc<feature_engineering>
best_score_svm, best_params_svm = grid(clf_svm, parameters_svm)
 print(best_score_svm )
df_test['winPlacePerc'] = pred<save_to_csv>
x = StandardScaler().fit_transform(X)
 X_train, X_test, y_train, y_test = train_test_split(x, Y, test_size =.2)
 X_train_reduced = PCA(n_components = 2 ).fit_transform(X_train)
 X_test_reduced = PCA(n_components= 2 ).fit_transform(X_test )
submission = df_test[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False )<load_from_csv>
def boundaries(model, heading, best_params):
Model = model(**best_params)
 Model.fit(X_train_reduced, y_train)
 X_set, y_set = np.concatenate([X_train_reduced, X_test_reduced], axis = 0), np.concatenate([y_train, y_test], axis = 0)
 X1, X2 = np.meshgrid(np.arange(start = X_set[:, 0].min() - 1, stop = X_set[:, 0].max() + 1, step = 0.01),
np.arange(start = X_set[:, 1].min() - 1, stop = X_set[:, 1].max() + 1, step = 0.01))
plt.contourf(X1, X2, Model.predict(np.array([X1.ravel() , X2.ravel() ] ).T ).reshape(X1.shape),
alpha = 0.5, cmap = ListedColormap(( 'k', 'blue')))
 plt.xlim(X1.min() , X1.max())
 plt.ylim(X2.min() , X2.max())
 for i, j in enumerate(np.unique(y_set)) :
plt.scatter(X_set[y_set == j, 0], X_set[y_set == j, 1], c = ListedColormap(( 'red', 'green'))(i), label = j)
 plt.xticks(fontsize = 3)
 plt.yticks(fontsize = 3 )
def import_training_data(nrows=None):
df=pd.read_csv('.. /input/train_V2.csv', sep=',', encoding='utf-8', nrows=nrows)
 df=df.drop(['rankPoints'],axis=1)
 return df
def import_test_data(nrows=None):
df_test=pd.read_csv('.. /input/test_V2.csv', sep=',', encoding='utf-8', nrows=nrows)
 df_test=df_test.drop(['rankPoints'],axis=1)
 return df_test<feature_engineering>
clf_svm = SVC(**best_params_svm)
 clf_svm.fit(npX,npy)
df=import_training_data()<drop_column>
test_df = test[cols]
df_squad=df[(df['matchType'].str.contains('squad')) |(df['matchType'].str.contains('flare')) ].drop('matchType',axis=1)
 df_duo=df[df['matchType'].str.contains('duo')].drop('matchType',axis=1)
 df_solo=df[df['matchType'].str.contains('solo')].drop(columns=['DBNOs','revives'] ).drop('matchType',axis=1)
 df_crash=df[df['matchType'].str.contains('crash')].drop('matchType',axis=1 ).drop('killPoints', axis=1 ).drop('winPoints', axis=1 )<drop_column>
nptest = np.array(test_df)
 pred = clf_svm.predict(nptest)
df_test=import_test_data()
df_test_squad=df_test[(df_test['matchType'].str.contains('squad')) |(df_test['matchType'].str.contains('flare')) ].drop('matchType',axis=1)
 df_test_duo=df_test[df_test['matchType'].str.contains('duo')].drop('matchType',axis=1)
 df_test_solo=df_test[df_test['matchType'].str.contains('solo')].drop('matchType',axis=1 ).drop(columns=['DBNOs','revives'])
 df_test_crash=df_test[df_test['matchType'].str.contains('crash')].drop('matchType',axis=1 ).drop('killPoints', axis=1 ).drop('winPoints', axis=1 )<drop_column>
predictions = pd.DataFrame(pred, index = test_df.index, columns = ['Survived'])
len_df_test=len(df_test)
 del df
del df_test<sort_values>
pred_df=pd.DataFrame(test['PassengerId'])
 pred_df['Survived']=predictions
<groupby><EOS> 
pred_df.to_csv('predictions_svm_titanic_final.csv',index=False )