Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<set_options>
if not sys.warnoptions:
warnings.simplefilter("ignore" )
gc.collect()<predict_on_test>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
%%time
df_y = pd.DataFrame(index=df_test.index)
 df_y['y'] = 0
for matchType in matchTypes:
print("----------", matchType, "----------")
 df_test_part = df_test[df_test['matchType'] == matchType]
y_part = pipeline_predict(trained_estimators[matchType], df_test_part, trained_scalers[matchType])
 df_y_part = pd.DataFrame(index=df_test_part.index)
 df_y_part['y'] = y_part
df_y.update(df_y_part)
 y = df_y['y'].values<set_options>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train['Survived'].values
passId = test['PassengerId']
data = pd.concat(( train, test))
print("data size is: {}".format(data.shape))
del df_test
del df_y
gc.collect()<save_to_csv>
print(data.isnull().sum() )
df_oof = pd.DataFrame()
df_oof['linear_oof'] = oof
df_oof.to_csv('linear_oof.csv', index_label='id' )<save_to_csv>
data['Last_Name'] = data['Name'].apply(lambda x: str.split(x, ",")[0])
 data['Fare'].fillna(data['Fare'].mean() , inplace=True)
 default_survival_chance = 0.5
data['Family_Survival'] = default_survival_chance
for grp, grp_df in data[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin == 0.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
data.loc[data['Family_Survival']!=0.5].shape[0] )
df_submission = pd.DataFrame(index=df_test_id.index)
 df_submission['winPlacePerc'] = y
df_submission.to_csv('linear_raw.csv', index_label='Id' )<load_from_csv>
for _, grp_df in data.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "
+str(data[data['Family_Survival']!=0.5].shape[0]))
df_test = pd.read_csv('.. /input/test_V2.csv')
 df_test.shape<merge>
data = data.reset_index(drop=True)
 data = data.drop('Survived', axis=1)
 data.tail()
df_submission = df_submission.merge(df_test[['Id', 'matchId', 'groupId', 'maxPlace', 'numGroups']], on='Id', how='left')
 df_submission.head()<merge>
data['Fare'] = pd.qcut(data['Fare'], 4)
 lbl = LabelEncoder()
data['Fare'] = lbl.fit_transform(data['Fare'] )
df_submission_group = df_submission.groupby(['matchId', 'groupId'] ).first().reset_index()
df_submission_group['rank'] = df_submission_group.groupby(['matchId'])['winPlacePerc'].rank()
df_submission_group = df_submission_group.merge(df_submission_group.groupby('matchId')['rank'].max().to_frame('max_rank' ).reset_index() , on='matchId', how='left')
 df_submission_group['adjusted_perc'] =(df_submission_group['rank'] - 1)/(df_submission_group['numGroups'] - 1)
 df_submission = df_submission.merge(df_submission_group[['adjusted_perc', 'matchId', 'groupId']], on=['matchId', 'groupId'], how='left')
 df_submission['winPlacePerc'] = df_submission['adjusted_perc']
df_submission.head()<feature_engineering>
titles_data = sorted(set([x for x in data['Name'].map(lambda x: get_title(x)) ]))
df_submission.loc[df_submission.maxPlace == 0, 'winPlacePerc'] = 0
df_submission.loc[df_submission.maxPlace == 1, 'winPlacePerc'] = 1<feature_engineering>
data['Title'] = data['Name'].map(lambda x: get_title(x))
data['Title'] = data.apply(set_title, axis=1 )
t = df_submission.loc[df_submission.maxPlace > 1]
gap = 1.0 /(t.maxPlace.values - 1)
 fixed_perc = np.around(t.winPlacePerc.values / gap)* gap
df_submission.loc[df_submission.maxPlace > 1, 'winPlacePerc'] = fixed_perc<feature_engineering>
print(data['Title'].value_counts() )
df_submission.loc[(df_submission.maxPlace > 1)&(df_submission.numGroups == 1), 'winPlacePerc'] = 0
assert df_submission['winPlacePerc'].isnull().sum() == 0<save_to_csv>
print('Total missing age data: ', pd.isnull(data['Age'] ).sum() )
df_submission[['Id', 'winPlacePerc']].to_csv('linear_adjusted.csv', index=False )<set_options>
data['Age'] = data.groupby('Title')['Age'].apply(lambda x: x.fillna(x.median()))
debug = False;
debug_rows = 10000;
gc.enable()
pd.options.display.float_format = '{:,.2f}'.format
pd.set_option('display.max_columns', 100)
 %matplotlib inline 
<load_from_csv>
data['Age'] = pd.qcut(data['Age'], 4)
 lbl = LabelEncoder()
data['Age'] = lbl.fit_transform(data['Age'] )
if(debug):
train = pd.read_csv('.. /input/train_V2.csv', nrows = debug_rows)
 test = pd.read_csv('.. /input/test_V2.csv')
 else:
train = pd.read_csv('.. /input/train_V2.csv', nrows = debug_rows)
 test = pd.read_csv('.. /input/test_V2.csv' )<count_missing_values>
data['Title'] = data['Title'].replace(['Mr', 'Miss', 'Mrs', 'Master'], [0, 1, 2, 3] )
train.isnull().sum()<count_missing_values>
data['Sex'] = data['Sex'].replace(['male', 'female'], [0, 1] )
train.isnull().sum()<count_missing_values>
data['Embarked'] = data['Embarked'].fillna(data['Embarked'].mode() [0])
 data['Embarked'] = data['Embarked'].replace(['S', 'C', 'Q'], [0, 1, 2] )
train.dropna(axis=0, how='all')
 print(train.isnull().any().any() )<count_values>
data['Cabin'] = data['Cabin'].map(lambda x: x[0] )
assists[assists>10].count()<count_values>
data['Cabin'].value_counts()
eda[eda>15].count()<count_values>
def unknown_cabin(cabin):
if cabin != 'U':
return 1
else:
return 0
data['Cabin'] = data['Cabin'].apply(lambda x:unknown_cabin(x))
eda[eda>10].count()<filter>
data['Family Size'] = data['SibSp'] + data['Parch']
train[train['walkDistance']>10000]<filter>
data = data.drop(['Name', 'Parch', 'SibSp', 'Ticket', 'Last_Name', 'PassengerId'], axis = 1 )
train[train['headshotKills']>10]<filter>
train = data[:ntrain]
test = data[ntrain:]
train[(train['rideDistance']==0)&(train['roadKills']>0)]<filter>
X_test = test
X_train = train
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )
train[train['teamKills']>3]<drop_column>
ran = RandomForestClassifier(random_state=1)
 knn = KNeighborsClassifier()
log = LogisticRegression()
xgb = XGBClassifier()
gbc = GradientBoostingClassifier()
svc = SVC(probability=True)
 ext = ExtraTreesClassifier()
ada = AdaBoostClassifier()
gnb = GaussianNB()
gpc = GaussianProcessClassifier()
bag = BaggingClassifier()
models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores.append(acc.mean() )
df = train
df = df.drop(df[(df['walkDistance']<10.0)&(df['damageDealt']>0)].index)
 df = df.drop(df[(df['walkDistance']<10.0)&(df['kills']>10)].index)
 df = df.drop(df[(df['walkDistance']<100.0)&(df['weaponsAcquired']>30)].index)
 df = df.drop(df[(df['walkDistance']<10.0)&(df['heals']>100)].index)
 df = df.drop(df[(df['walkDistance']<10.0)&(df['headshotKills']>5)].index)
 df = df.drop(df[(df['walkDistance']<10.0)&(df['headshotKills']>5)].index)
 df['unrated_kill'] = 0
df.loc[df['killPoints']==0, 'unrated_kill']=1
df['poor_kills'] = 0
df.loc[(df['killPoints']>10)&(df['killPoints']<800), 'poor_kills'] = 1
df.loc[(df['killPoints']>10)&(df['killPoints']<800), 'killPoints'] = 0
df['killPerDamage'] = df['kills']/df['damageDealt']
df = df.fillna(0)
 df = df.drop(df[df['killStreaks']>=10].index)
 df['unrated_rank'] = 0
df.loc[df['rankPoints']==-1, 'unrated_rank']=1
df = df.drop(df[(df['rideDistance']==0.0)&(df['roadKills']>0)].index)
 df = df.drop(df[(df['weaponsAcquired']>30)&(df['walkDistance']<100)].index)
 df['unrated_win'] = 0
df.loc[(df['winPoints']==-1)|(df['winPoints'] == 0), 'unrated_win']=1
df['poor_wins'] = 0
df.loc[(df['winPoints']>250)&(df['winPoints']<1200), 'poor_wins'] = 1
df.loc[(df['winPoints']>250)&(df['winPoints']<1200), 'killPoints'] = 0
print('removed:' + str(train['Id'].count() - df['Id'].count()))
df.head() 
<load_from_csv>
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Score': scores})
 result_df = results.sort_values(by='Score', ascending=False ).reset_index(drop=True)
 result_df.head(11 )
def feature_engineering(is_train=True,debug=True):
test_idx = None
if is_train:
print("processing train.csv")
 if debug == True:
df = pd.read_csv('.. /input/train_V2.csv', nrows=1000000)
 else:
df = pd.read_csv('.. /input/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 if debug == True:
df = pd.read_csv('.. /input/test_V2.csv')
 else:
df = pd.read_csv('.. /input/test_V2.csv')
 test_idx = df.Id
print("remove some columns")
 target = 'winPlacePerc'
if(is_train):
print("removing cheaters")
 df = df.drop(df[(df['walkDistance']<10.0)&(df['damageDealt']>0)].index)
 df = df.drop(df[(df['walkDistance']<10.0)&(df['kills']>10)].index)
 df = df.drop(df[(df['walkDistance']<100.0)&(df['weaponsAcquired']>30)].index)
 df = df.drop(df[(df['walkDistance']<10.0)&(df['heals']>100)].index)
 df = df.drop(df[(df['walkDistance']<10.0)&(df['headshotKills']>5)].index)
 df = df.drop(df[(df['walkDistance']<10.0)&(df['headshotKills']>5)].index)
 df = df.drop(df[df['killStreaks']>=10].index)
 df = df.drop(df[(df['rideDistance']==0.0)&(df['roadKills']>0)].index)
 df = df.drop(df[(df['weaponsAcquired']>30)&(df['walkDistance']<100)].index)
 df['unrated_kill'] = 0
df.loc[df['killPoints']==0, 'unrated_kill']=1
df['poor_kills'] = 0
df.loc[(df['killPoints']>10)&(df['killPoints']<800), 'poor_kills'] = 1
df.loc[(df['killPoints']>10)&(df['killPoints']<800), 'killPoints'] = 0
df['unrated_rank'] = 0
df.loc[df['rankPoints']==-1, 'unrated_rank']=1
df['unrated_win'] = 0
df.loc[(df['winPoints']==-1)|(df['winPoints'] == 0), 'unrated_win']=1
df['poor_wins'] = 0
df.loc[(df['winPoints']>250)&(df['winPoints']<1200), 'poor_wins'] = 1
df.loc[(df['winPoints']>250)&(df['winPoints']<1200), 'killPoints'] = 0
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
print("Removing Na's From DF")
 df.fillna(0, inplace=True)
 features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
if is_train:
print("get target")
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 print("Adding Features")
 df['headshotrate'] = df['kills']/df['headshotKills']
df['killStreakrate'] = df['killStreaks']/df['kills']
df['healthitems'] = df['heals'] + df['boosts']
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['headshotKills_over_kills'] = df['headshotKills'] / df['kills']
df['distance_over_weapons'] = df['totalDistance'] / df['weaponsAcquired']
df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['killsPerWalkDistance'] = df['kills'] / df['walkDistance']
df["skill"] = df["headshotKills"] + df["roadKills"]
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
print("Removing Na's From DF")
 df.fillna(0, inplace=True)
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names, test_idx<prepare_x_and_y>
fi = {'Features':train.columns.tolist() , 'Importance':xgb.feature_importances_}
importance = pd.DataFrame(fi, index=None ).sort_values('Importance', ascending=False )
x_train, y_train, train_columns, _ = feature_engineering(True, debug=debug)
 x_test, _, _ , test_idx = feature_engineering(False, debug=debug)
<train_model>
fi = {'Features':train.columns.tolist() , 'Importance':np.transpose(log.coef_[0])}
importance = pd.DataFrame(fi, index=None ).sort_values('Importance', ascending=False )
gc.collect() ;
train_index = round(int(x_train.shape[0]*0.8))
dev_X = x_train[:train_index]
val_X = x_train[train_index:]
dev_y = y_train[:train_index]
val_y = y_train[train_index:]
gc.collect() ;
def run_lgb(train_X, train_y, val_X, val_y, x_test):
params = {"objective" : "regression", "metric" : "mae", 'n_estimators':20000, 'early_stopping_rounds':200,
"num_leaves" : 31, "learning_rate" : 0.05, "bagging_fraction" : 0.7,
"bagging_seed" : 0, "num_threads" : 4,"colsample_bytree" : 0.7
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, valid_sets=[lgtrain, lgval], early_stopping_rounds=200, verbose_eval=1000)
 pred_test_y = model.predict(x_test, num_iteration=model.best_iteration)
 return pred_test_y, model
pred_test, model = run_lgb(dev_X, dev_y, val_X, val_y, x_test)
 pred_test<load_from_csv>
gbc_imp = pd.DataFrame({'Feature':train.columns, 'gbc importance':gbc.feature_importances_})
 xgb_imp = pd.DataFrame({'Feature':train.columns, 'xgb importance':xgb.feature_importances_})
 ran_imp = pd.DataFrame({'Feature':train.columns, 'ran importance':ran.feature_importances_})
 ext_imp = pd.DataFrame({'Feature':train.columns, 'ext importance':ext.feature_importances_})
 ada_imp = pd.DataFrame({'Feature':train.columns, 'ada importance':ada.feature_importances_})
 importances = gbc_imp.merge(xgb_imp, on='Feature' ).merge(ran_imp, on='Feature' ).merge(ext_imp, on='Feature' ).merge(ada_imp, on='Feature')
 importances['Average'] = importances.mean(axis=1)
 importances = importances.sort_values(by='Average', ascending=False ).reset_index(drop=True)
 importances
print(pred_test.shape[0])
 pred_test
df_sub = pd.read_csv(".. /input/sample_submission_V2.csv")
 df_sub['winPlacePerc'] = pred_test
df_sub.head() 
<load_from_csv>
fi = {'Features':importances['Feature'], 'Importance':importances['Average']}
importance = pd.DataFrame(fi, index=None ).sort_values('Importance', ascending=False )
if(debug):
df_sub = pd.read_csv(".. /input/sample_submission_V2.csv", nrows=pred_test.shape[0])
 df_test = pd.read_csv(".. /input/test_V2.csv", nrows=pred_test.shape[0])
 else:
df_sub = pd.read_csv(".. /input/sample_submission_V2.csv")
 df_test = pd.read_csv(".. /input/test_V2.csv")
 df_sub['winPlacePerc'] = pred_test
df_sub = df_sub.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 df_sub_group = df_sub.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub_group = df_sub_group.merge(
 df_sub_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 df_sub = df_sub.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub["winPlacePerc"] = df_sub["adjusted_perc"]
df_sub.loc[df_sub.maxPlace == 0, "winPlacePerc"] = 0
df_sub.loc[df_sub.maxPlace == 1, "winPlacePerc"] = 1
subset = df_sub.loc[df_sub.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub.loc[df_sub.maxPlace > 1, "winPlacePerc"] = new_perc
df_sub.loc[(df_sub.maxPlace > 1)&(df_sub.numGroups == 1), "winPlacePerc"] = 0
assert df_sub["winPlacePerc"].isnull().sum() == 0
df_sub[["Id", "winPlacePerc"]].to_csv("submission_adjusted.csv", index=False)
 df_sub<drop_column>
train = train.drop(['Embarked', 'Cabin'], axis=1)
 test = test.drop(['Embarked', 'Cabin'], axis=1)
 X_train = train
X_test = test
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )
<prepare_x_and_y>
ran = RandomForestClassifier(random_state=1)
 knn = KNeighborsClassifier()
log = LogisticRegression()
xgb = XGBClassifier(random_state=1)
 gbc = GradientBoostingClassifier(random_state=1)
 svc = SVC(probability=True)
 ext = ExtraTreesClassifier(random_state=1)
 ada = AdaBoostClassifier(random_state=1)
 gnb = GaussianNB()
gpc = GaussianProcessClassifier()
bag = BaggingClassifier(random_state=1)
 models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores_v2 = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores_v2.append(acc.mean() )
<compute_test_metric>
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Original Score': scores,
'Score with feature selection': scores_v2})
 result_df = results.sort_values(by='Score with feature selection', ascending=False ).reset_index(drop=True)
 result_df.head(11 )
<load_from_csv>
Cs = [0.001, 0.01, 0.1, 1, 5, 10, 15, 20, 50, 100]
gammas = [0.001, 0.01, 0.1, 1]
hyperparams = {'C': Cs, 'gamma' : gammas}
gd=GridSearchCV(estimator = SVC(probability=True), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
<feature_engineering>
learning_rate = [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
n_estimators = [100, 250, 500, 750, 1000, 1250, 1500]
hyperparams = {'learning_rate': learning_rate, 'n_estimators': n_estimators}
gd=GridSearchCV(estimator = GradientBoostingClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
<load_from_csv>
penalty = ['l1', 'l2']
C = np.logspace(0, 4, 10)
 hyperparams = {'penalty': penalty, 'C': C}
gd=GridSearchCV(estimator = LogisticRegression() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
warnings.filterwarnings('ignore')
 %matplotlib inline
py.init_notebook_mode(connected=True)
 debug = False
if debug == True:
df_train = pd.read_csv('.. /input/train_V2.csv', nrows=10000)
 df_test = pd.read_csv('.. /input/test_V2.csv')
 else:
df_train = pd.read_csv('.. /input/train_V2.csv')
 df_test = pd.read_csv('.. /input/test_V2.csv' )<filter>
learning_rate = [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
n_estimators = [10, 25, 50, 75, 100, 250, 500, 750, 1000]
hyperparams = {'learning_rate': learning_rate, 'n_estimators': n_estimators}
gd=GridSearchCV(estimator = XGBClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df_train[df_train['groupId']=='4d4b580de459be']<feature_engineering>
max_depth = [3, 4, 5, 6, 7, 8, 9, 10]
min_child_weight = [1, 2, 3, 4, 5, 6]
hyperparams = {'max_depth': max_depth, 'min_child_weight': min_child_weight}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df_train = df_train[df_train['Id']!='f70c74418bb064']
headshot = df_train[['kills','winPlacePerc','headshotKills']]
headshot['headshotrate'] = headshot['kills'] / headshot['headshotKills']<feature_engineering>
gamma = [i*0.1 for i in range(0,5)]
hyperparams = {'gamma': gamma}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10, max_depth=3,
min_child_weight=1), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
del headshot
df_train['headshotrate'] = df_train['kills']/df_train['headshotKills']
df_test['headshotrate'] = df_test['kills']/df_test['headshotKills']<feature_engineering>
subsample = [0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1]
colsample_bytree = [0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1]
hyperparams = {'subsample': subsample, 'colsample_bytree': colsample_bytree}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10, max_depth=3,
min_child_weight=1, gamma=0), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
killStreak = df_train[['kills','winPlacePerc','killStreaks']]
killStreak['killStreakrate'] = killStreak['killStreaks']/killStreak['kills']
killStreak.corr()<drop_column>
reg_alpha = [1e-5, 1e-2, 0.1, 1, 100]
hyperparams = {'reg_alpha': reg_alpha}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10, max_depth=3,
min_child_weight=1, gamma=0, subsample=0.6, colsample_bytree=0.9),
param_grid = hyperparams, verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
del healthitems<feature_engineering>
n_restarts_optimizer = [0, 1, 2, 3]
max_iter_predict = [1, 2, 5, 10, 20, 35, 50, 100]
warm_start = [True, False]
hyperparams = {'n_restarts_optimizer': n_restarts_optimizer, 'max_iter_predict': max_iter_predict, 'warm_start': warm_start}
gd=GridSearchCV(estimator = GaussianProcessClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
kills = df_train[['assists','winPlacePerc','kills']]
kills['kills_assists'] =(kills['kills'] + kills['assists'] )<set_options>
n_estimators = [10, 25, 50, 75, 100, 125, 150, 200]
learning_rate = [0.001, 0.01, 0.1, 0.5, 1, 1.5, 2]
hyperparams = {'n_estimators': n_estimators, 'learning_rate': learning_rate}
gd=GridSearchCV(estimator = AdaBoostClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
del df_train,df_test;
gc.collect()<load_from_csv>
n_neighbors = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20]
algorithm = ['auto']
weights = ['uniform', 'distance']
leaf_size = [1, 2, 3, 4, 5, 10, 15, 20, 25, 30]
hyperparams = {'algorithm': algorithm, 'weights': weights, 'leaf_size': leaf_size,
'n_neighbors': n_neighbors}
gd=GridSearchCV(estimator = KNeighborsClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
def feature_engineering(is_train=True,debug=True):
test_idx = None
if is_train:
print("processing train.csv")
 if debug == True:
df = pd.read_csv('.. /input/train_V2.csv', nrows=10000)
 else:
df = pd.read_csv('.. /input/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv('.. /input/test_V2.csv')
 test_idx = df.Id
print("remove some columns")
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
if is_train:
print("get target")
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names, test_idx
x_train, y_train, train_columns, _ = feature_engineering(True,False)
 x_test, _, _ , test_idx = feature_engineering(False,True )<feature_engineering>
n_estimators = [10, 25, 50, 75, 100]
max_depth = [3, None]
max_features = [1, 3, 5, 7]
min_samples_split = [2, 4, 6, 8, 10]
min_samples_leaf = [2, 4, 6, 8, 10]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features,
'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
gd=GridSearchCV(estimator = RandomForestClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
x_train['headshotrate'] = x_train['kills']/x_train['headshotKills']
x_test['headshotrate'] = x_test['kills']/x_test['headshotKills']
x_train['killStreakrate'] = x_train['killStreaks']/x_train['kills']
x_test['killStreakrate'] = x_test['killStreaks']/x_test['kills']
x_train['healthitems'] = x_train['heals'] + x_train['boosts']
x_test['healthitems'] = x_test['heals'] + x_test['boosts']
del x_train['heals'];del x_test['heals']
train_columns.append('headshotrate')
 train_columns.append('killStreakrate')
 train_columns.append('healthitems')
 train_columns.remove('heals' )<drop_column>
n_estimators = [10, 25, 50, 75, 100]
max_depth = [3, None]
max_features = [1, 3, 5, 7]
min_samples_split = [2, 4, 6, 8, 10]
min_samples_leaf = [2, 4, 6, 8, 10]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features,
'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
gd=GridSearchCV(estimator = ExtraTreesClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
x_train = reduce_mem_usage(x_train)
 x_test = reduce_mem_usage(x_test )<set_options>
n_estimators = [10, 15, 20, 25, 50, 75, 100, 150]
max_samples = [1, 2, 3, 5, 7, 10, 15, 20, 25, 30, 50]
max_features = [1, 3, 5, 7]
hyperparams = {'n_estimators': n_estimators, 'max_samples': max_samples, 'max_features': max_features}
gd=GridSearchCV(estimator = BaggingClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
warnings.filterwarnings("ignore")
<split>
ran = RandomForestClassifier(n_estimators=25,
max_depth=3,
max_features=3,
min_samples_leaf=2,
min_samples_split=8,
random_state=1)
 knn = KNeighborsClassifier(algorithm='auto',
leaf_size=1,
n_neighbors=5,
weights='uniform')
 log = LogisticRegression(C=2.7825594022071245,
penalty='l2')
 xgb = XGBClassifier(learning_rate=0.0001,
n_estimators=10,
random_state=1)
 gbc = GradientBoostingClassifier(learning_rate=0.0005,
n_estimators=1250,
random_state=1)
 svc = SVC(probability=True)
 ext = ExtraTreesClassifier(max_depth=None,
max_features=3,
min_samples_leaf=2,
min_samples_split=8,
n_estimators=10,
random_state=1)
 ada = AdaBoostClassifier(learning_rate=0.1,
n_estimators=50,
random_state=1)
 gpc = GaussianProcessClassifier()
bag = BaggingClassifier(random_state=1)
 models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores_v3 = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores_v3.append(acc.mean() )
folds = KFold(n_splits=3,random_state=6)
 oof_preds = np.zeros(x_train.shape[0])
 sub_preds = np.zeros(x_test.shape[0])
 start = time.time()
valid_score = 0
feature_importance_df = pd.DataFrame()
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(x_train, y_train)) :
trn_x, trn_y = x_train.iloc[trn_idx], y_train[trn_idx]
val_x, val_y = x_train.iloc[val_idx], y_train[val_idx]
train_data = lgb.Dataset(data=trn_x, label=trn_y)
 valid_data = lgb.Dataset(data=val_x, label=val_y)
 params = {"objective" : "regression", "metric" : "mae", 'n_estimators':15000, 'early_stopping_rounds':100,
"num_leaves" : 31, "learning_rate" : 0.05, "bagging_fraction" : 0.9,
"bagging_seed" : 0, "num_threads" : 4,"colsample_bytree" : 0.7
}
lgb_model = lgb.train(params, train_data, valid_sets=[train_data, valid_data], verbose_eval=1000)
 oof_preds[val_idx] = lgb_model.predict(val_x, num_iteration=lgb_model.best_iteration)
 oof_preds[oof_preds>1] = 1
oof_preds[oof_preds<0] = 0
sub_pred = lgb_model.predict(x_test, num_iteration=lgb_model.best_iteration)
 sub_pred[sub_pred>1] = 1
sub_pred[sub_pred<0] = 0
sub_preds += sub_pred/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = train_columns
fold_importance_df["importance"] = lgb_model.feature_importance()
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 gc.collect()
end = time.time()
print("Take Time :",(end-start))<load_from_csv>
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Original Score': scores,
'Score with feature selection': scores_v2,
'Score with tuned parameters': scores_v3})
 result_df = results.sort_values(by='Score with tuned parameters', ascending=False ).reset_index(drop=True)
 result_df.head(11 )
df_test = pd.read_csv('.. /input/' + 'test_V2.csv')
 pred = sub_preds
print("fix winPlacePerc")
 for i in range(len(df_test)) :
winPlacePerc = pred[i]
maxPlace = int(df_test.iloc[i]['maxPlace'])
 if maxPlace == 0:
winPlacePerc = 0.0
elif maxPlace == 1:
winPlacePerc = 1.0
else:
gap = 1.0 /(maxPlace - 1)
 winPlacePerc = round(winPlacePerc / gap)* gap
if winPlacePerc < 0: winPlacePerc = 0.0
if winPlacePerc > 1: winPlacePerc = 1.0
pred[i] = winPlacePerc
if(i + 1)% 100000 == 0:
print(i, flush=True, end=" ")
 df_test['winPlacePerc'] = pred
submission = df_test[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False )<import_modules>
grid_hard = VotingClassifier(estimators = [('Random Forest', ran), 
('Logistic Regression', log), 
('XGBoost', xgb), 
('Gradient Boosting', gbc), 
('Extra Trees', ext), 
('AdaBoost', ada), 
('Gaussian Process', gpc), 
('SVC', svc), 
('K Nearest Neighbour', knn), 
('Bagging Classifier', bag)], voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, X_train, y_train, cv = 10)
 grid_hard.fit(X_train, y_train)
 print("Hard voting on train set score mean: {:.2f}".format(grid_hard_cv['train_score'].mean() *100))
print("Hard voting on test set score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100))
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt<load_from_csv>
grid_soft = VotingClassifier(estimators = [('Random Forest', ran), 
('Logistic Regression', log), 
('XGBoost', xgb), 
('Gradient Boosting', gbc), 
('Extra Trees', ext), 
('AdaBoost', ada), 
('Gaussian Process', gpc), 
('SVC', svc), 
('K Nearest Neighbour', knn), 
('Bagging Classifier', bag)], voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, X_train, y_train, cv = 10)
 grid_soft.fit(X_train, y_train)
 print("Soft voting on train set score mean: {:.2f}".format(grid_soft_cv['train_score'].mean() *100))
print("Soft voting on test set score mean: {:.2f}".format(grid_soft_cv['test_score'].mean() *100))
develop_mode = False
if develop_mode:
df_train = reduce_mem_usage(pd.read_csv('.. /input/train_V2.csv', nrows=5000))
df_test = reduce_mem_usage(pd.read_csv('.. /input/test_V2.csv'))
else:
df_train = reduce_mem_usage(pd.read_csv('.. /input/train_V2.csv'))
df_test = reduce_mem_usage(pd.read_csv('.. /input/test_V2.csv'))<train_model>
predictions = grid_soft.predict(X_test)
 submission = pd.concat([pd.DataFrame(passId), pd.DataFrame(predictions)], axis = 'columns')
 submission.columns = ["PassengerId", "Survived"]
submission.to_csv('titanic_submission.csv', header = True, index = False )