Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<count_values>
%matplotlib inline
warnings.filterwarnings("ignore" )
cols = ['killPoints','rankPoints','winPoints']
querys = ['rankPoints <= 0 & killPoints != 0', 'rankPoints <= 0 & winPoints != 0',
'killPoints == 0 & rankPoints != 0', 'killPoints == 0 & winPoints != 0',
'winPoints == 0 & rankPoints != 0', 'winPoints == 0 & killPoints != 0']
for q in querys:
print('count(%s):' % q, len(train[cols].query(q)) )<count_unique_values>
df_titanic_train = pd.read_csv('.. /input/train.csv')
 df_titanic_test = pd.read_csv('.. /input/test.csv')
 PassengerId = df_titanic_test["PassengerId"]
<count_unique_values>
df_titanic_train = df_titanic_train.drop(outliers_to_drop, axis = 0 ).reset_index(drop=True )
print('match count:', train['matchId'].nunique())
 maxPlacePerc = train.groupby('matchId')['winPlacePerc'].max()
print('match [not contains 1st place]:', len(maxPlacePerc[maxPlacePerc != 1]))
del maxPlacePerc
edge = train[(train['maxPlace'] > 1)&(train['numGroups'] == 1)]
print('match [maxPlace>1 & numGroups==1]:', len(edge.groupby('matchId')))
 print(' - unique winPlacePerc:', edge['winPlacePerc'].unique() )<concatenate>
train_size = len(df_titanic_train)
 df_titanic = pd.concat(objs=[df_titanic_train, df_titanic_test], axis=0 ).reset_index(drop=True)
 df_titanic.head()
pd.concat([train[train['winPlacePerc'] == 1].head(5),
train[train['winPlacePerc'] == 0].head(5)],
keys=['winPlacePerc_1', 'winPlacePerc_0'] )<concatenate>
df_titanic = df_titanic.fillna(np.nan)
 df_titanic.isnull().sum()
all_data = train.append(test, sort=False ).reset_index(drop=True)
 del train, test
gc.collect()<groupby>
def absolute_relative_freq(variable):
absolute_frequency = variable.value_counts()
relative_frequency = round(variable.value_counts(normalize = True)*100, 2)
 df = pd.DataFrame({'Absolute Frequency':absolute_frequency, 'Relative Frequency(%)':relative_frequency})
 print('Absolute and Relative Frequency of [',variable.name,']')
 display(df )
match = all_data.groupby('matchId')
 all_data['killPlacePerc'] = match['kills'].rank(pct=True ).values<data_type_conversions>
df_titanic['Fare'] = df_titanic['Fare'].fillna(df_titanic['Fare'].median() )
distance =(all_data['rideDistance'] + all_data['walkDistance'] + all_data['swimDistance'])
 all_data['zombi'] =(( distance == 0)&(all_data['kills'] == 0)
 &(all_data['weaponsAcquired'] == 0)
 &(all_data['matchType'].str.contains('solo')) ).astype(int)
 all_data['cheater'] =(( all_data['kills'] / distance >= 1)
 |(all_data['kills'] > 30)|(all_data['roadKills'] > 10)).astype(int)
 del distance
pd.concat([all_data['zombi'].value_counts() , all_data['cheater'].value_counts() ], axis=1 ).T<feature_engineering>
df_titanic["Fare"] = df_titanic["Fare"].map(lambda i: np.log(i)if i > 0 else 0 )
all_data.drop(['rankPoints','killPoints','winPoints'], axis=1, inplace=True )<data_type_conversions>
df_titanic['Embarked'].value_counts()
def fillInf(df, val):
numcols = df.select_dtypes(include='number' ).columns
cols = numcols[numcols != 'winPlacePerc']
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
for c in cols: df[c].fillna(val, inplace=True )<feature_engineering>
labelEncoder = LabelEncoder()
df_titanic['Embarked'] = labelEncoder.fit_transform(df_titanic['Embarked'] )
all_data['_totalDistance'] = all_data['rideDistance'] + all_data["walkDistance"] + all_data["swimDistance"]
all_data["_specialKills"] = all_data["headshotKills"] + all_data["roadKills"]
all_data['_healthItems'] = all_data['heals'] + all_data['boosts']
all_data['_headshotKillRate'] = all_data['headshotKills'] / all_data['kills']
all_data['_killsOverWalkDistance'] = all_data['kills'] / all_data['walkDistance']
all_data['_killsOverDistance'] = all_data['kills'] / all_data['_totalDistance']
all_data['_killPlaceOverMaxPlace'] = all_data['killPlace'] / all_data['maxPlace']
fillInf(all_data, 0 )<drop_column>
labelEncoder = LabelEncoder()
df_titanic['Sex'] = labelEncoder.fit_transform(df_titanic['Sex'] )
agg_col = list(all_data.columns)
 exclude_agg_col = ['Id','matchId','groupId','matchType','matchDuration','maxPlace','numGroups','winPlacePerc']
for c in exclude_agg_col:
agg_col.remove(c)
 print(agg_col )<groupby>
condition = df_titanic['Age'].isnull()
age_NaN = df_titanic['Age'][condition].index
for age in age_NaN :
condition1 = df_titanic['SibSp'] == df_titanic.iloc[age]["SibSp"]
condition2 = df_titanic['Pclass'] == df_titanic.iloc[age]["Pclass"]
condition3 = df_titanic['Parch'] == df_titanic.iloc[age]["Parch"]
condition = condition1 & condition2 & condition3
new_age = df_titanic['Age'][condition].median()
df_titanic['Age'].iloc[age] = new_age if not np.isnan(new_age)else df_titanic['Age'].median()
def grouping(df):
group = df.groupby(['matchId','groupId','matchType'])
 groupCount = group.size().to_frame('member')
 groupMean = group.mean()
groupMax = group[agg_col].max().rename(columns=lambda s: '_max.' + s)
 groupMin = group[agg_col].min().rename(columns=lambda s: '_min.' + s)
 return pd.concat([groupCount, groupMean, groupMax, groupMin], axis=1 ).reset_index()
all_data = grouping(all_data)
 all_data = reduce_mem_usage(all_data)
 print(all_data.shape )<groupby>
df_titanic['Age'] =(df_titanic['Age'] - df_titanic['Age'].mean())/ df_titanic['Age'].std()
cols = np.r_[agg_col,['matchId']]
match = all_data[cols].groupby('matchId' )<merge>
df_titanic['Title'] = df_titanic['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 df_titanic['Title'].unique()
cols = agg_col
matchSum = match[cols].sum().rename(columns=lambda s: '_sum.' + s ).reset_index()
all_data = pd.merge(all_data, matchSum)
 print(all_data.shape)
 del matchSum
gc.collect()
for c in cols:
all_data['_percSum.' + c] = all_data[c] / all_data['_sum.' + c]
fillInf(all_data, 0 )<categorify>
df_titanic['Title'] = df_titanic['Title'].replace(['Don', 'Rev', 'Dr', 'Major', 'Lady', 'Sir',
'Col', 'Capt', 'Countess', 'Jonkheer', 'Dona'], 'Rare' )
cols = agg_col
matchMean = match[cols].mean().rename(columns=lambda s: '_matchMean.' + s ).reset_index()
all_data = pd.merge(all_data, reduce_mem_usage(matchMean))
print(all_data.shape)
 del matchMean
gc.collect()
for c in cols:
all_data['_percMean.' + c] = all_data[c] / all_data['_matchMean.' + c]
all_data.drop(['_matchMean.' + c], axis=1, inplace=True)
 fillInf(all_data, 0 )<merge>
df_titanic["Title"] = df_titanic['Title'].map({"Master":0, "Miss":1, "Ms" : 1, "Mme":1,
"Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 df_titanic['Title'] = df_titanic["Title"].astype(int )
killMinorRank = all_data[['matchId','_min.kills','_max.killPlace']].copy()
killMinorRank['_rank.minor.killPlace'] = killMinorRank.groupby(['matchId','_min.kills'] ).rank().values
all_data = pd.merge(all_data, killMinorRank)
 del killMinorRank
gc.collect()<sort_values>
df_titanic['Surname'] = df_titanic['Name'].map(lambda i: i.split(',')[0] )
null_cnt = all_data.isnull().sum().sort_values()
print(null_cnt[null_cnt > 0])
 all_data.head()<count_values>
del df_titanic['Name']
mapper = lambda x: 'solo' if('solo' in x)else 'duo' if('duo' in x)or('crash' in x)else 'squad'
all_data['matchTypeCat'] = all_data['matchType'].map(mapper)
 print(all_data['matchTypeCat'].value_counts() )<count_unique_values>
df_titanic['Cabin'].isnull().sum()
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )<feature_engineering>
df_titanic['Ticket'] = df_titanic['Ticket'].map(
 lambda i: i.replace(".","" ).replace("/","" ).strip().split(' ')[0] if not i.isdigit() else "TKT" )
cols = [col for col in all_data.columns if col not in ['Id','matchId','groupId']]
for i, t in all_data.loc[:, cols].dtypes.iteritems() :
if t == object:
all_data[i] = pd.factorize(all_data[i])[0]<prepare_x_and_y>
df_titanic['Family'] = df_titanic['SibSp'] + df_titanic['Parch'] + 1
X_train = all_data[all_data['winPlacePerc'].notnull() ].reset_index(drop=True)
 X_test = all_data[all_data['winPlacePerc'].isnull() ].drop(['winPlacePerc'], axis=1 ).reset_index(drop=True)
 del all_data
gc.collect()
Y_train = X_train.pop('winPlacePerc')
 X_test_grp = X_test[['matchId','groupId']].copy()
X_train.drop(['matchId','groupId'], axis=1, inplace=True)
 X_test.drop(['matchId','groupId'], axis=1, inplace=True)
 print(X_train.shape, X_test.shape )<create_dataframe>
df_titanic['Alone'] = df_titanic['Family'].map(lambda i: 1 if i == 1 else 0)
 df_titanic['Small'] = df_titanic['Family'].map(lambda i: 1 if i == 2 else 0)
 df_titanic['Medium'] = df_titanic['Family'].map(lambda i: 1 if 3 <= i <= 4 else 0)
 df_titanic['Large'] = df_titanic['Family'].map(lambda i: 1 if i >= 5 else 0 )
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10] )<choose_model_class>
df_titanic['Pclass'] = df_titanic['Pclass'].astype("category")
 columns = ['Title', 'Surname', 'Cabin', 'Ticket', 'Pclass']
for col in columns:
df_titanic = pd.get_dummies(df_titanic, columns=[col], prefix=col )
params={'learning_rate': 0.05,
'objective':'mae',
'metric':'mae',
'num_leaves': 31,
'verbose': 0,
'random_state':42,
'bagging_fraction': 0.7,
'feature_fraction': 0.7
}
mts = list()
fis = list()
pred = np.zeros(X_test.shape[0])
 for mt in X_train['matchTypeCat'].unique() :
idx = X_train[X_train['matchTypeCat'] == mt].index
reg = lgb.LGBMRegressor(**params, n_estimators=1000)
 reg.fit(X_train.loc[idx], Y_train.loc[idx])
 idx = X_test[X_test['matchTypeCat'] == mt].index
pred[idx] = reg.predict(X_test.loc[idx], num_iteration=reg.best_iteration_)
 mts.append(mt)
 fis.append(reg.feature_importances_ )<merge>
df_titanic.drop(labels = ["PassengerId"], axis = 1, inplace = True )
X_test_grp['winPlacePerc'] = pred
group = X_test_grp.groupby(['matchId'])
 X_test_grp['winPlacePerc'] = pred
X_test_grp['_rank.winPlacePerc'] = group['winPlacePerc'].rank(method='min')
 X_test = pd.concat([X_test, X_test_grp], axis=1)
 sub_match = X_test_grp[['matchId','_rank.winPlacePerc']].groupby(['matchId'])
 sub_group = group.count().reset_index() ['matchId'].to_frame()
X_test = pd.merge(X_test, sub_group )<feature_engineering>
df_titanic_train = df_titanic[:train_size]
df_titanic_test = df_titanic[train_size:]
df_titanic_train['Survived'] = df_titanic_train['Survived'].astype(int)
 del df_titanic_test['Survived']
fullgroup =(X_test['numGroups'] == X_test['maxPlace'])
 subset = X_test.loc[fullgroup]
X_test.loc[fullgroup, 'winPlacePerc'] =(subset['_rank.winPlacePerc'].values - 1)/(subset['maxPlace'].values - 1)
 subset = X_test.loc[~fullgroup]
gap = 1.0 /(subset['maxPlace'].values - 1)
 new_perc = np.around(subset['winPlacePerc'].values / gap)* gap
X_test.loc[~fullgroup, 'winPlacePerc'] = new_perc<feature_engineering>
X_train = df_titanic_train.drop(['Survived'], axis=1)
 y_train = df_titanic_train['Survived']
X_test = df_titanic_test
sc = StandardScaler()
sc.fit(X_train)
 X_train = sc.transform(X_train)
 X_test = sc.transform(X_test )
X_test.loc[~fullgroup, '_pred.winPlacePerc'] = np.around(X_test.loc[~fullgroup, 'winPlacePerc'].values / gap)+ 1
_=
X_test.loc[~fullgroup &(X_test['matchId'] == '000b598b79aa5e'),
['matchId','groupId','winPlacePerc','maxPlace','numGroups','_pred.winPlacePerc','_rank.winPlacePerc']
].sort_values(['matchId','_pred.winPlacePerc'] )<groupby>
kfold = StratifiedKFold(n_splits=10)
 seed = 20
clfs = []
clfs.append(SVC(random_state=seed))
clfs.append(DecisionTreeClassifier(random_state=seed))
clfs.append(RandomForestClassifier(random_state=seed))
clfs.append(ExtraTreesClassifier(random_state=seed))
clfs.append(GradientBoostingClassifier(random_state=seed))
clfs.append(MLPClassifier(random_state=seed))
clfs.append(KNeighborsClassifier())
 clfs.append(LogisticRegression(random_state=seed))
clfs.append(XGBClassifier(random_state = seed))
_=<feature_engineering>
clf_results = []
for clf in clfs :
clf_results.append(cross_val_score(clf, X_train, y=y_train, scoring = "accuracy", cv=kfold, n_jobs=1))
X_test.loc[X_test['maxPlace'] == 0, 'winPlacePerc'] = 0
X_test.loc[X_test['maxPlace'] == 1, 'winPlacePerc'] = 1
X_test.loc[(X_test['maxPlace'] > 1)&(X_test['numGroups'] == 1), 'winPlacePerc'] = 0
X_test['winPlacePerc'].describe()<save_to_csv>
df_result = pd.DataFrame({"Means":clf_means,
"Stds": clf_std,
"Algorithm":["SVC",
"DecisionTree",
"RandomForest",
"ExtraTrees",
"GradientBoosting",
"MLPClassifier",
"KNeighboors",
"LogisticRegression",
"XGBoost"]})
 df_result.sort_values(by=['Means'], ascending=False )
test = pd.read_csv('.. /input/test_V2.csv')
 submission = pd.merge(test, X_test[['matchId','groupId','winPlacePerc']])
 submission = submission[['Id','winPlacePerc']]
submission.to_csv("submission.csv", index=False )<import_modules>
extraTrees = ExtraTreesClassifier(random_state=seed)
 gBoosting = GradientBoostingClassifier(random_state=seed)
 randomForest = RandomForestClassifier(random_state=seed)
 logReg = LogisticRegression(random_state=seed)
 xgbc = XGBClassifier(random_state=seed )
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt<load_from_csv>
param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
grid_result = GridSearchCV(extraTrees,
param_grid = param_grid,
cv=kfold,
scoring="accuracy",
n_jobs= -1,
verbose = 1)
 grid_result.fit(X_train,y_train)
 extraTrees_best_result = grid_result.best_estimator_
print('Best score:', np.round(grid_result.best_score_*100, 2))
print('Best estimator:', extraTrees_best_result )
develop_mode = False
if develop_mode:
df_train = reduce_mem_usage(pd.read_csv('.. /input/train_V2.csv', nrows=5000))
df_test = reduce_mem_usage(pd.read_csv('.. /input/test_V2.csv'))
else:
df_train = reduce_mem_usage(pd.read_csv('.. /input/train_V2.csv'))
df_test = reduce_mem_usage(pd.read_csv('.. /input/test_V2.csv'))<train_model>
param_grid = {'learning_rate': [0.01, 0.02],
'max_depth': [4, 5, 6],
'max_features': [0.2, 0.3, 0.4],
'min_samples_split': [2, 3, 4],
'random_state':[seed]}
grid_result = GridSearchCV(gBoosting,
param_grid=param_grid,
cv=kfold,
scoring="accuracy",
n_jobs=-1,
verbose=1)
 grid_result.fit(X_train, y_train)
 gBoosting_best_result = grid_result.best_estimator_
print('Best score:', np.round(grid_result.best_score_*100, 2))
print('Best estimator:', gBoosting_best_result )
print('The sizes of the datasets are:')
 print('Training Dataset: ', df_train.shape)
 print('Testing Dataset: ', df_test.shape )<sort_values>
param_grid = {"max_depth": [None],
"max_features": [1, 2],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
grid_result = GridSearchCV(randomForest,
param_grid=param_grid,
cv=kfold,
scoring="accuracy",
n_jobs= -1,
verbose = 1)
 grid_result.fit(X_train, y_train)
 randomForest_best_result = grid_result.best_estimator_
print('Best score:', np.round(grid_result.best_score_*100, 2))
print('Best estimator:', randomForest_best_result )
group_tmp = df_train[df_train['matchId']=='df014fbee741c6']['groupId'].value_counts().sort_values(ascending=False )<set_options>
param_grid = {'penalty' : ['l1', 'l2'],
'C': np.logspace(0, 4, 10),
'solver' : ['liblinear', 'saga']
}
grid_result = GridSearchCV(logReg,
param_grid=param_grid,
cv=kfold,
scoring="accuracy",
n_jobs= -1,
verbose = 1)
 grid_result.fit(X_train, y_train)
 logReg_best_result = grid_result.best_estimator_
print('Best score:', np.round(grid_result.best_score_*100, 2))
print('Best estimator:', logReg_best_result )
warnings.filterwarnings('ignore' )<load_from_csv>
param_grid = {'n_estimators': [275, 280],
'learning_rate': [0.01, 0.03],
'subsample': [0.9, 1],
'max_depth': [3, 4],
'colsample_bytree': [0.8, 0.9],
'min_child_weight': [2, 3],
'random_state':[seed]}
grid_result = GridSearchCV(xgbc,
param_grid=param_grid,
cv=kfold,
scoring="accuracy",
n_jobs= -1,
verbose = 1)
 grid_result.fit(X_train, y_train)
 xgbc_best_result = grid_result.best_estimator_
print('Best score:', np.round(grid_result.best_score_*100, 2))
print('Best estimator:', xgbc_best_result )
def BuildFeature(is_train=True):

 y = None
test_idx = None
if is_train:
print("Reading train.csv")
 df = pd.read_csv('.. /input/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("Reading test.csv")
 df = pd.read_csv('.. /input/test_V2.csv')
 test_idx = df.Id
df = reduce_mem_usage(df)
 print("Delete Unuseful Columns")
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 if is_train:
print("Read Labels")
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("Read Group mean features")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train:
df_out = agg.reset_index() [['matchId','groupId']]
else:
df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("Read Group max features")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("Read Group min features")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("Read Group size features")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("Read Match mean features")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("Read Match size features")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names, test_idx<prepare_x_and_y>
survived_ET = pd.Series(extraTrees_best_result.predict(X_test), name="ET")
 survived_GB = pd.Series(gBoosting_best_result.predict(X_test), name="GB")
 survived_RF = pd.Series(randomForest_best_result.predict(X_test), name="RF")
 survived_LR = pd.Series(logReg_best_result.predict(X_test), name="LR")
 survived_XB = pd.Series(xgbc_best_result.predict(X_test), name="XB")
 ensemble_results = pd.concat([survived_ET,
survived_GB,
survived_RF,
survived_LR,
survived_XB],
axis=1)
 fig, ax = plt.subplots(figsize=(12,8))
g= sns.heatmap(ensemble_results.corr() ,
annot=True,
fmt = ".3f",
cmap = "Greens",
ax=ax )
X_train, y_train, train_columns, _ = BuildFeature(is_train=True)
 X_test, _, _ , test_idx = BuildFeature(is_train=False )<drop_column>
voting = VotingClassifier(estimators=[('XB', xgbc_best_result), 
('GB', gBoosting_best_result), 
('RF', randomForest_best_result), 
('LR', logReg_best_result), 
('ET', extraTrees_best_result)],
voting='soft', n_jobs=-1)
 voting.fit(X_train, y_train )
X_train =reduce_mem_usage(X_train)
 X_test = reduce_mem_usage(X_test )<train_model>
print("Score(Voting): " + str(voting.score(X_train, y_train)) )
LR_model = LinearRegression(n_jobs=4, normalize=True)
 LR_model.fit(X_train,y_train )<compute_test_metric>
y_predict = voting.predict(X_test )
<predict_on_test><EOS> 
solution = pd.DataFrame({
"PassengerId": PassengerId,
"Survived": y_predict.astype(int)
 })
 solution.to_csv('solution_final_v1.csv', index=False)
 df_solution = pd.read_csv('solution_final_v1.csv' )