Kaggle competition: Titanic - Machine Learning from Disaster
data['killsWithoutMoving'] =(( data['kills'] > 0)&(data['totalDistance'] == 0))
test_data['killsWithoutMoving'] =(( test_data['kills'] > 0)&(test_data['totalDistance'] == 0))<drop_column>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
data.drop(data[data['killsWithoutMoving'] == True].index, inplace=True)
 data.drop(data[data['roadKills'] > 10].index, inplace=True)
 data.drop(data[data['kills'] >= 40].index, inplace=True)
 data.drop(data[data['longestKill'] >= 1000].index, inplace=True)
 data.drop(data[data['heals'] >= 40].index, inplace=True )<categorify>
train_dir = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_dir = pd.read_csv('/kaggle/input/titanic/test.csv')
 print("Training data: {}".format(train_dir.shape))
print("Testing data: {}".format(test_dir.shape))
lbl = LabelEncoder()
lbl.fit(list(data['matchType'].values))
data['matchType'] = lbl.transform(list(data['matchType'].values))
lbl.fit(list(test_data['matchType'].values))
test_data['matchType'] = lbl.transform(list(test_data['matchType'].values)) 
<feature_engineering>
train_dir.isnull().any()
cols = [col for col in data.columns if col not in ['Id','matchId','groupId']]
for i, t in data.loc[:, cols].dtypes.iteritems() :
if t == object:
data[i] = pd.factorize(data[i])[0]
cols = [col for col in test_data.columns if col not in ['Id','matchId','groupId']]
for i, t in test_data.loc[:, cols].dtypes.iteritems() :
if t == object:
test_data[i] = pd.factorize(test_data[i])[0] 
<sort_values>
train_dir.isnull().sum()
total = data.isnull().sum().sort_values(ascending = False)
 percent =(data.isnull().sum() /data.isnull().count() ).sort_values(ascending = False)
 missing_data = pd.concat([total,percent], axis = 1, keys = ['Total', 'Percent'])
 missing_data.head(20 )<drop_column>
train_dir.isnull().sum()
data.drop(2744604, inplace =True)
<correct_missing_values>
updated_train_dir = train_dir.drop(['Ticket', 'PassengerId', 'Name','Cabin'], axis = 1)
 updated_train_dir.head(3 )
data= data.dropna()<drop_column>
updated_test_dir = test_dir.drop(['Ticket', 'PassengerId', 'Name','Cabin'], axis = 1)
 updated_test_dir.head(3 )
data = data.drop(columns=['groupId','matchId'], axis = 1)
 test_data = test_data.drop(columns=['groupId','matchId'], axis = 1 )<prepare_x_and_y>
Missing_age = 100 * updated_train_dir['Age'].isnull().sum() / updated_train_dir['Age'].shape[0]
print("Age missing value: {}".format(Missing_age))
y = data.winPlacePerc
X = data.drop(['winPlacePerc', 'Id'], axis=1 )<choose_model_class>
updated_train_dir['Age'].isnull().any()
def identify_zero_importance_features(X, y, iterations = 2):

 feature_importances = np.zeros(X.shape[1])
 model = lgb.LGBMRegressor(objective='regression', boosting_type = 'goss',
n_estimators =10000, class_weight = 'balanced')
 for i in range(iterations):
train_features, valid_features, train_y, valid_y = train_test_split(X, y,
test_size = 0.25,
random_state = i)
 model.fit(train_features, train_y, early_stopping_rounds=200,
eval_set = [(valid_features, valid_y)])
 feature_importances += model.feature_importances_ / iterations
feature_importances = pd.DataFrame({'feature': list(X.columns),
'importance': feature_importances} ).sort_values('importance',
ascending = False)
 zero_features = list(feature_importances[feature_importances['importance'] == 0.0]['feature'])
 print('
There are %d features with 0.0 importance' % len(zero_features))
return zero_features, feature_importances
zero_features, feature_importances = identify_zero_importance_features(X, y, iterations = 1)
 print('zero_features:',zero_features)
 print('feature_importances : ', feature_importances )<drop_column>
updated_test_dir['Age'].fillna(updated_test_dir['Age'].median() , inplace=True)
 updated_test_dir['Age'].isnull().any()
to_drop = feature_importances[feature_importances['importance'] <= pp]['feature']
X = X.drop(columns = to_drop )<split>
updated_train_dir.isnull().any()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1 )<choose_model_class>
updated_train_dir['Embarked'].value_counts()
gbm = LGBMRegressor(objective='regression',
num_leaves=40,
learning_rate=0.005,
n_estimators=20000,
max_bin=55,
bagging_fraction=0.7,
bagging_freq=9,
feature_fraction=0.7,
feature_fraction_seed=9,
bagging_seed=10,
min_data_in_leaf=7,
min_sum_hessian_in_leaf=5)
 gbm.fit(X_train, y_train,
eval_set=[(X_test, y_test)],
eval_metric='rmsle',
early_stopping_rounds=200)
 y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration_)
 print('The accuracy of the lgbm Regressor is',r2_score(y_test,y_pred))
print('RMSE is:
', mean_squared_error(y_test,y_pred))<predict_on_test>
Missing_embarked = 100 * updated_train_dir['Embarked'].isnull().sum() / updated_train_dir['Embarked'].shape[0]
print("Missing embakred info: {}".format(Missing_embarked))
feats = test_data.drop(['Id'], axis=1)
 feats = feats[X_train.columns]
final_preds = gbm.predict(feats,num_iteration=gbm.best_iteration_ )<save_to_csv>
updated_train_dir['Embarked'].fillna('S', inplace=True)
 updated_train_dir.isnull().any()
submission = pd.DataFrame()
submission['Id'] = test_data.Id
submission['winPlacePerc'] = final_preds
submission.to_csv('submission1.csv', index=False )<feature_engineering>
updated_test_dir['Embarked'].value_counts()
submission[submission['winPlacePerc'] < 0] = 0
submission[submission['winPlacePerc'] >1] = 1 
<set_options>
updated_test_dir["Embarked"].fillna(updated_test_dir['Embarked'].value_counts().idxmax() , inplace=True)
 updated_test_dir.isnull().any()
gc.enable() 
<load_from_csv>
total_survived_notsurvived = updated_train_dir['Survived'].shape[0]
num_survived = updated_train_dir[updated_train_dir['Survived'] == 1].shape[0]
not_survived = updated_train_dir[updated_train_dir['Survived'] == 0].shape[0]
print("Survived: {}".format(100 *(num_survived / total_survived_notsurvived)))
 print("Not Survived: {}".format(100 *(not_survived/total_survived_notsurvived)) )
def feature_engineering(is_train=True):
if is_train:
print("processing train.csv")
 df = pd.read_csv(".. /input/train_V2.csv")
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv(".. /input/test_V2.csv")
 df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
print("remove some columns")
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
print("get target")
 if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group sum feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('sum')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_sum", "_sum_rank"], how='left', on=['matchId', 'groupId'])
 df_out=reduce_mem_usage(df_out)
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 df_out=reduce_mem_usage(df_out)
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out=reduce_mem_usage(df_out)
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = np.array(df_out)
 feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names 
<prepare_x_and_y>
updated_test_dir['Age group']= updated_test_dir['Age'].apply(grouping_Age )
x_train, y, feature_names = feature_engineering(True )<split>
updated_train_dir['Age group'].value_counts()
X_train, X_val, y_train, y_val = train_test_split(x_train, y, test_size=0.33, random_state=42)
<set_options>
gender = {
'male': 1,
'female':0
}
updated_train_dir['Sex'] = updated_train_dir['Sex'].apply(lambda x: gender.get(x))
updated_train_dir.drop(['Age'], axis=1, inplace=True)
warnings.filterwarnings("ignore")
 color = sns.color_palette() 
<predict_on_test>
updated_test_dir['Sex'] = updated_test_dir['Sex'].apply(lambda x: gender.get(x))
train_data=lgb.Dataset(X_train, label=y_train)
 val_data= lgb.Dataset(X_val, label=y_val)
 params = {
'num_leaves': 144,
'learning_rate': 0.1,
'n_estimators': 1500,
'max_depth':12,
'max_bin':55,
'bagging_fraction':0.8,
'bagging_freq':5,
'feature_fraction':0.9,
'verbose':50,
'early_stopping_rounds':100
}
params['metric'] = 'auc'
light_reg= lgb.train(params, train_data, valid_sets=val_data, num_boost_round=5000, early_stopping_rounds=100)
 y_pred=light_reg.predict(X_val)
 print(mean_absolute_error(y_val,y_pred)) 
<predict_on_test>
updated_test_dir.drop(['Age'], axis=1, inplace=True)
y_pred=light_reg.predict(X_val)
 print(mean_absolute_error(y_val,y_pred))
del X_val
del y_val<prepare_x_and_y>
updated_train_dir.drop(['Fare'], axis=1, inplace=True )
x_test, y_test, feature_names = feature_engineering(False )<predict_on_test>
updated_test_dir.drop(['Fare'], axis=1, inplace=True )
y_test_pred=light_reg.predict(x_test )<save_to_csv>
traindf = pd.get_dummies(updated_train_dir, columns = ["Embarked","Age group", "Pclass"],
prefix=["Em_type", "Age_group", "Pclass_"] )
df=pd.read_csv(".. /input/test_V2.csv")
 var=pd.DataFrame(columns=['Id','winPlacePerc'])
 var['Id']= df['Id']
var['winPlacePerc'] = y_test_pred
submission = var[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False )<set_options>
testdf = pd.get_dummies(updated_test_dir, columns = ["Embarked","Age group", "Pclass"],
prefix=["Em_type", "Age_group", "Pclass_"])
 testdf.head(2)
warnings.filterwarnings('ignore')
 %matplotlib inline
py.init_notebook_mode(connected=True)
 print(os.listdir(".. /input")) 
<load_from_csv>
train_y = traindf['Survived']
traindf.drop(['Survived'], axis=1,inplace=True )
df_train = reduce_mem_usage(pd.read_csv('.. /input/train_V2.csv'))
df_test = reduce_mem_usage(pd.read_csv('.. /input/test_V2.csv'))<train_model>
print("Training shape: {} and Testing shape: {}
\
Training Label:{}".format(traindf.shape,
testdf.shape,
train_y.shape))
print('train : {}'.format(df_train.shape))
print('test : {}'.format(df_test.shape))<sort_values>
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
total = df_train.isnull().sum().sort_values(ascending=False)
 percent =(df_train.isnull().sum() / df_train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head()<sort_values>
X_train, X_test, Y_train, Y_test = train_test_split(traindf,
train_y,
test_size=0.3,
random_state=42)
 X_train.shape, X_test.shape, Y_train.shape, Y_test.shape
total = df_test.isnull().sum().sort_values(ascending=False)
 percent =(df_test.isnull().sum() /df_test.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head()<correct_missing_values>
LogisticRegression = LogisticRegression(max_iter=10000)
 LogisticRegression.fit(X_train, Y_train )
df_train.dropna(axis=0, inplace=True )<compute_test_metric>
predictions = LogisticRegression.predict(X_test)
 predictions
cm = np.corrcoef(df_train[cols].values.T)
<feature_engineering>
Linear_Reg_acc = accuracy_score(predictions, Y_test)* 100
headshot = df_train[['kills', 'winPlacePerc', 'headshotKills']]
headshot['headshotrate'] = headshot['headshotKills'] / headshot['kills']
headshot.corr()<feature_engineering>
model_random = RandomForestClassifier(n_estimators = 700,
max_features='auto',
oob_score=True,
random_state=1,
n_jobs=1,
min_samples_leaf=1,
min_samples_split=10
)
df_train['headshotrate'] = df_train['headshotKills']/df_train['kills']
df_test['headshotrate'] = df_test['headshotKills']/df_test['kills']
del headshot<feature_engineering>
model_random.fit(X_train, Y_train)
 predictions_random = model_random.predict(X_test )
killStreak = df_train[['kills','winPlacePerc','killStreaks']]
killStreak['killStreakrate'] = killStreak['killStreaks']/killStreak['kills']
killStreak.corr()<feature_engineering>
Random_Forest_Acc = accuracy_score(predictions_random, Y_test)* 100
print("Random FOrest acc: {}".format(Random_Forest_Acc))
df_train['killStreakrate'] = -(df_train['killStreaks'] / df_train['kills'])
 df_test['killStreakrate'] = -(df_test['killStreaks'] / df_test['kills'])
 del killStreak<feature_engineering>
svc_model = SVC()
svc_model.fit(X_train, Y_train)
 predictions_svc = svc_model.predict(X_test)
df_train['hacker_pt'] = 0
df_test['hacker_pt'] = 0<set_options>
SVC_acc = accuracy_score(predictions_svc, Y_test)* 100
print("SVC accuracy: {}".format(SVC_acc))
pd.set_option('display.max_columns', 50 )<feature_engineering>
n_model = KNeighborsClassifier(n_neighbors = 4)
 n_model.fit(X_train, Y_train )
df_train['total_Distance'] = df_train['rideDistance'] +df_train['walkDistance'] + df_train['swimDistance']
df_test['total_Distance'] = df_test['rideDistance'] + df_test['walkDistance'] + df_test['swimDistance']
df_train[(df_train['winPlacePerc'] == 1)&(df_train['total_Distance'] < 100)].head()<data_type_conversions>
predictions_knn = n_model.predict(X_test )
df_train['headshotrate'] = df_train['headshotrate'].fillna(0)
 df_train['killStreakrate'] = df_train['killStreakrate'].fillna(0)
 df_test['headshotrate'] = df_test['headshotrate'].fillna(0)
 df_test['killStreakrate'] = df_test['killStreakrate'].fillna(0)
<feature_engineering>
accuracy_score(predictions_knn, Y_test)* 100
df_train['hacker_pt'][(df_train['heals'] + df_train['boosts'] < 1)&(df_train['total_Distance'] < 100)&(df_train['kills'] > 20)] = 1
df_test['hacker_pt'][(df_test['heals'] + df_test['boosts'] < 1)&(df_test['total_Distance'] < 100)&(df_test['kills'] > 20)] = 1<filter>
knn_data = {}
for i in range(1,30):
model = KNeighborsClassifier(n_neighbors=i)
 model.fit(X_train, Y_train)
 prediction_data = model.predict(X_test)
 knn_data[i] = accuracy_score(prediction_data, Y_test)* 100
df_train['hacker_pt'][(df_train['kills'] > 10)&(df_train['weaponsAcquired'] >= 10)&(df_train['total_Distance'] == 0)] += 1
df_test['hacker_pt'][(df_test['kills'] > 10)&(df_test['weaponsAcquired'] >= 10)&(df_test['total_Distance'] == 0)] += 1<filter>
n_model = KNeighborsClassifier(n_neighbors = 14)
 n_model.fit(X_train, Y_train)
 predictions_knn = n_model.predict(X_test)
 knn_Ac = accuracy_score(predictions_knn, Y_test)* 100
print(knn_Ac )
df_train['hacker_pt'][df_train['longestKill'] >= 1000] += 1
df_test['hacker_pt'][df_train['longestKill'] >= 1000] += 1<sort_values>
knn_acc = knn_data[14]
print("KNN accuracy: {}".format(knn_acc))
df_train.sort_values('hacker_pt', ascending=False ).head()<feature_engineering>
model_naive = GaussianNB()
model_naive.fit(X_train, Y_train)
 prediction_naive = model_naive.predict(X_test )
kills = df_train[['assists','winPlacePerc','kills']]
kills['kills_assists'] =(kills['kills'] + kills['assists'])
 kills.corr()<drop_column>
gaussian_acc = accuracy_score(prediction_naive, Y_test)* 100
print("Gaussian Acc: {}".format(gaussian_acc))
df_train['kills_assists'] = df_train['kills'] + df_train['assists']
df_test['kills_assists'] = df_test['kills'] + df_test['assists']
del df_train['kills']
del df_test['kills']
del df_train['assists']
del df_test['assists']
del kills<set_options>
from sklearn.tree import DecisionTreeClassifier
df_train = reduce_mem_usage(df_train)
 df_test = reduce_mem_usage(df_test)
 gc.collect()<drop_column>
tree_data = {}
for i in range(2,40):
tree_model = DecisionTreeClassifier(criterion='gini',
min_samples_split=i,
max_features='auto',
min_samples_leaf=1)
 tree_model.fit(X_train, Y_train)
 tree_predict = tree_model.predict(X_test)
 tree_data[i]=accuracy_score(tree_predict, Y_test)
del missing_data
del percent
del total
gc.collect()<groupby>
tree_val = max(tree_data, key=tree_data.get)
 print(tree_val )
df_train_size = df_train.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_test_size = df_test.groupby(['matchId','groupId'] ).size().reset_index(name='group_size' )<groupby>
tree_model = DecisionTreeClassifier(criterion='gini',
min_samples_split=12,
max_features='auto',
min_samples_leaf=12 )
df_train_mean = df_train.groupby(['matchId','groupId'] ).mean().reset_index()
df_test_mean = df_test.groupby(['matchId','groupId'] ).mean().reset_index()<groupby>
tree_model.fit(X_train, Y_train)
 tree_predict = tree_model.predict(X_test)
 tree_acc = accuracy_score(tree_predict, Y_test)
 print("Tree accuarcy: {}".format(tree_acc))
df_train_match_mean = df_train.groupby(['matchId'] ).mean().reset_index()
df_test_match_mean = df_test.groupby(['matchId'] ).mean().reset_index()<merge>
model_ada = AdaBoostClassifier(n_estimators=1000, learning_rate=0.1)
 model_ada.fit(X_train, Y_train )
df_train = pd.merge(df_train, df_train_mean, suffixes=["", "_mean"], how='left', on=['matchId', 'groupId'])
 df_test = pd.merge(df_test, df_test_mean, suffixes=["", "_mean"], how='left', on=['matchId', 'groupId'])
 del df_train_mean
del df_test_mean<merge>
prediction_add = model_ada.predict(X_test)
 ada_accuracy = accuracy_score(prediction_add, Y_test)* 100
print('ada accuarcy: {}'.format(ada_accuracy))
df_train = pd.merge(df_train, df_train_match_mean, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 df_test = pd.merge(df_test, df_test_match_mean, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 del df_train_match_mean
del df_test_match_mean<merge>
model_linear_d= LinearDiscriminantAnalysis()
model_linear_d.fit(X_train,Y_train)
 prediction_lda=model_linear_d.predict(X_test)
 LDA_Accuracy = accuracy_score(prediction_lda, Y_test)
 print("Accuracy LDA: {}".format(LDA_Accuracy))
df_train = pd.merge(df_train, df_train_size, how='left', on=['matchId', 'groupId'])
 df_test = pd.merge(df_test, df_test_size, how='left', on=['matchId', 'groupId'])
 del df_train_size
del df_test_size<set_options>
models = {
'LinearDiscriminant': [LDA_Accuracy, prediction_lda],
'ADA': [ada_accuracy, prediction_add],
'DecisionTreeClassifier':[tree_acc,tree_predict],
'GaussianNB': [gaussian_acc,prediction_naive],
'LinearSVC':[SVC_acc,predictions_svc],
'RandomForestClassifier':[Random_Forest_Acc,predictions_random],
'LogisticRegression':[Linear_Reg_acc,predictions]
}
gc.collect()
df_train = reduce_mem_usage(df_train)
 df_test = reduce_mem_usage(df_test)
 gc.collect()<set_options>
models_acc = {
'LinearDiscriminant': LDA_Accuracy,
'ADA': ada_accuracy,
'DecisionTreeClassifier':tree_acc,
'GaussianNB': gaussian_acc,
'LinearSVC':SVC_acc,
'RandomForestClassifier':Random_Forest_Acc,
'LogisticRegression':Linear_Reg_acc}
warnings.filterwarnings("ignore")
 color = sns.color_palette() 
<drop_column>
max_acc = max(models_acc, key=models_acc.get)
 print("MAx accuracy is : {}".format(max_acc))
print(models_acc[max_acc] )
train_columns = list(df_test.columns)
 train_idx = df_train.Id
test_idx = df_test.Id
train_columns.remove("Id")
 train_columns.remove("matchId")
 train_columns.remove("groupId" )<prepare_x_and_y>
pred_test = svc_model.predict(testdf)
 pred_test
x_train = df_train[train_columns]
x_test = df_test[train_columns]
y_train = df_train["winPlacePerc"].astype('float' )<categorify>
submission = pd.DataFrame({
"PassengerId": test_dir["PassengerId"],
"Survived": pred_test} )
<merge><EOS> 
submission.to_csv("submission.csv",index=False )