Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<categorify>
%%javascript
$.getScript('https://kmahelona.github.io/ipython_notebook_goodies/ipython_notebook_toc.js' )
y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 print("Original y_train:", y_train.sum(axis=0))
print("Multilabel version:", y_train_multi.sum(axis=0))<split>
get_ipython().run_line_magic('matplotlib', 'inline')
 sns.set(style='white', context='notebook', palette='deep')
 warnings.filterwarnings('ignore')
 train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 IDtest = test["PassengerId"]
train.info()
test.info()
x_train, x_val, y_train, y_val = train_test_split(
 x_train, y_train_multi,
test_size=0.50,
random_state=2019
)<train_model>
train_na =(train.isnull().sum() / len(train)) * 100
train_na = train_na.drop(train_na[train_na == 0].index ).sort_values(ascending=False)[:30]
miss_train = pd.DataFrame({'Train Missing Ratio' :train_na})
 miss_train.head()
BATCH_SIZE = 13
def create_datagen() :
return ImageDataGenerator(
 zoom_range=0.15,
fill_mode='constant',
cval=0.,
horizontal_flip=True,
vertical_flip=True, 
)
 data_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019 )<choose_model_class>
test_na =(test.isnull().sum() / len(test)) * 100
test_na = test_na.drop(test_na[test_na == 0].index ).sort_values(ascending=False)[:30]
miss_test = pd.DataFrame({'Test Missing Ratio' :test_na})
 miss_test.head()
densenet = DenseNet121(
 weights='.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5',
include_top=False,
input_shape=(224,224,3)
 )<choose_model_class>
train = train.fillna(np.nan)
 test = test.fillna(np.nan )
def build_model() :
model = Sequential()
model.add(densenet)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.80))
model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=0.00010509613402110064),
metrics=['accuracy'] 
)
 return model<compute_train_metric>
train['source']='train'
test['source']='test'
combdata = pd.concat([train, test],ignore_index=True)
 print(train.shape, test.shape, combdata.shape )
class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val = y_val.sum(axis=1)- 1
y_pred = self.model.predict(X_val)> 0.5
y_pred = y_pred.astype(int ).sum(axis=1)- 1
_val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save('model.h5')
 return<train_model>
combdata.drop(labels = ["PassengerId"], axis = 1, inplace = True )
kappa_metrics = Metrics()
history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
epochs=15,
validation_data=(x_val, y_val)
 )<predict_on_test>
salutation = [i.split(",")[1].split(".")[0].strip() for i in combdata["Name"]]
combdata["Title"] = pd.Series(salutation)
 combdata["Title"].value_counts()
y_test = model.predict(x_test)
 y_test<filter>
combdata['Title'] = combdata['Title'].replace('Mlle', 'Miss')
 combdata['Title'] = combdata['Title'].replace(['Mme','Lady','Ms'], 'Mrs')
 combdata.Title.loc[(combdata.Title != 'Master')&(combdata.Title != 'Mr')& 
(combdata.Title != 'Miss')&(combdata.Title != 'Mrs')] = 'Others'
combdata["Title"].value_counts()
y_test = y_test > 0.37757874193797547
y_test<data_type_conversions>
combdata[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
y_test.astype(int ).sum(axis=1 )<data_type_conversions>
combdata = pd.get_dummies(combdata, columns = ["Title"] )
y_test.astype(int ).sum(axis=1)- 1<data_type_conversions>
combdata["Fare"].isnull().sum()
combdata["Fare"] = combdata["Fare"].fillna(combdata["Fare"].median())
y_test = y_test.astype(int ).sum(axis=1)- 1
y_test<save_to_csv>
combdata['Fare-bin'] = pd.qcut(combdata.Fare,5,labels=[1,2,3,4,5] ).astype(int)
 combdata[['Fare-bin', 'Survived']].groupby(['Fare-bin'], as_index=False ).mean()
test_df['diagnosis'] = y_test
test_df.to_csv('submission.csv',index=False )<set_options>
warnings.filterwarnings("ignore")
 gc.enable()
print(os.listdir(".. /input")) 
<load_from_csv>
combdata_temp = combdata[['Age','Title_Master','Title_Miss','Title_Mr','Title_Mrs','Title_Others','Fare-bin','SibSp']]
X = combdata_temp.dropna().drop('Age', axis=1)
 Y = combdata['Age'].dropna()
holdout = combdata_temp.loc[np.isnan(combdata.Age)].drop('Age', axis=1)
 regressor = RandomForestRegressor(n_estimators = 300)
 regressor.fit(X, Y)
 y_pred = np.round(regressor.predict(holdout),1)
 combdata.Age.loc[combdata.Age.isnull() ] = y_pred
combdata.Age.isnull().sum(axis=0 )
def feature_engineering(is_train=True,debug=True):
if is_train:
print("processing train.csv")
 if debug == True:
df = pd.read_csv('.. /input/train_V2.csv', nrows=10000)
 else:
df = pd.read_csv('.. /input/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv('.. /input/test_V2.csv')
 print("remove some columns")
 target = 'winPlacePerc'
print("Adding Features")
 df['headshotrate'] = df['kills']/df['headshotKills']
df['killStreakrate'] = df['killStreaks']/df['kills']
df['healthitems'] = df['heals'] + df['boosts']
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['headshotKills_over_kills'] = df['headshotKills'] / df['kills']
df['distance_over_weapons'] = df['totalDistance'] / df['weaponsAcquired']
df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['killsPerWalkDistance'] = df['kills'] / df['walkDistance']
df["skill"] = df["headshotKills"] + df["roadKills"]
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
print("Removing Na's From DF")
 df.fillna(0, inplace=True)
 features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
if is_train:
print("get target")
 y = df.groupby(['matchId','groupId'])[target].agg('mean')
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
del df, df_out, agg, agg_rank
gc.collect()
return X, y<prepare_x_and_y>
bins = [ 0, 4, 12, 18, 30, 50, 65, 100]
age_index =(1,2,3,4,5,6,7)
 combdata['Age-bin'] = pd.cut(combdata.Age, bins, labels=age_index ).astype(int)
 combdata[['Age-bin', 'Survived']].groupby(['Age-bin'],as_index=False ).mean()
x_train, y_train = feature_engineering(True,False)
 x_test, _ = feature_engineering(False,True )<drop_column>
combdata["Sex"] = combdata["Sex"].map({"male": 0, "female":1})
x_train = reduce_mem_usage(x_train)
 x_test = reduce_mem_usage(x_test )<load_from_csv>
combdata["Fsize"] = combdata["SibSp"] + combdata["Parch"] + 1
combdata[['Fsize', 'Survived']].groupby(['Fsize'], as_index=False ).mean()
def post_rst(pred_test):
df_sub = pd.read_csv(".. /input/sample_submission_V2.csv")
 df_test = pd.read_csv(".. /input/test_V2.csv")
 df_sub['winPlacePerc'] = pred_test
df_sub = df_sub.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 df_sub_group = df_sub.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub_group = df_sub_group.merge(
 df_sub_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 df_sub = df_sub.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub["winPlacePerc"] = df_sub["adjusted_perc"]
df_sub.loc[df_sub.maxPlace == 0, "winPlacePerc"] = 0
df_sub.loc[df_sub.maxPlace == 1, "winPlacePerc"] = 1
subset = df_sub.loc[df_sub.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub.loc[df_sub.maxPlace > 1, "winPlacePerc"] = new_perc
df_sub.loc[(df_sub.maxPlace > 1)&(df_sub.numGroups == 1), "winPlacePerc"] = 0
assert df_sub["winPlacePerc"].isnull().sum() == 0
df_sub[["Id", "winPlacePerc"]].to_csv("submission_adjusted.csv", index=False )<choose_model_class>
combdata = combdata.drop(labels='SibSp', axis=1 )
def nn_model(input_shape=x_train.shape[1], hidden_size=64):
model = Sequential()
model.add(Dense(hidden_size, input_dim=input_shape, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(hidden_size, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(hidden_size, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(hidden_size, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(1, kernel_initializer='normal'))
model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mae'])
 return model<split>
combdata = combdata.drop(labels='Parch', axis=1 )
def train_val_split(x_train, y_train):
train_index = round(int(x_train.shape[0]*0.8))
dev_X = x_train[:train_index]
val_X = x_train[train_index:]
dev_y = y_train[:train_index]
val_y = y_train[train_index:]
del x_train, y_train
gc.collect() ;
return dev_X, val_X, dev_y, val_y<split>
combdata.Ticket = combdata.Ticket.map(lambda x: x[0])
 combdata[['Ticket', 'Survived']].groupby(['Ticket'], as_index=False ).mean()
dev_X, val_X, dev_y, val_y = train_val_split(x_train, y_train)
 def run_lgb(train_X, train_y, val_X, val_y, x_test):
params = {"objective" : "regression", "metric" : "mae", 'n_estimators':5000, 'early_stopping_rounds':200,
"num_leaves" : 150, "learning_rate" : 0.05, "bagging_fraction" : 0.5,
"bagging_seed" : 0, "num_threads" : 4,"colsample_bytree" : 0.5
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, valid_sets=[lgtrain, lgval], early_stopping_rounds=200, verbose_eval=1000)
 pred_test_y = model.predict(x_test, num_iteration=model.best_iteration)
 return pred_test_y
pred_test_lgb = run_lgb(dev_X, dev_y, val_X, val_y, x_test)
 pred_test_lgb = pred_test_lgb.reshape(-1,1)
 del dev_X, val_X, dev_y, val_y
gc.collect()<train_model>
combdata['Ticket'].value_counts()
np.random.seed(42)
 std_scaler = StandardScaler().fit(x_train)
 x_train = std_scaler.transform(x_train)
 x_test = std_scaler.transform(x_test)
 mlp_model = nn_model(x_train.shape[1], hidden_size=64)
 mlp_model.fit(x_train, y_train, batch_size=128, epochs=8, verbose=1, validation_split=0.1, shuffle=True)
 pred_test_mlp = mlp_model.predict(x_test)
 del mlp_model
gc.collect()<drop_column>
combdata['Ticket'] = combdata['Ticket'].replace(['A','W','F','L','5','6','7','8','9'], '4')
 combdata[['Ticket', 'Survived']].groupby(['Ticket'], as_index=False ).mean()
del x_train, y_train, x_test
gc.collect()<define_variables>
combdata = pd.get_dummies(combdata, columns = ["Ticket"], prefix="T" )
pred_test =(pred_test_mlp + pred_test_lgb)/ 2.0<import_modules>
combdata["Cabin"] = pd.Series([i[0] if not pd.isnull(i)else 'U' for i in combdata['Cabin'] ] )
import numpy as np
import pandas as pd
import os 
<set_options>
combdata = combdata.drop(labels='Cabin', axis=1 )
mpl.rcParams['font.sans-serif'] = ['FangSong']
mpl.rcParams['axes.unicode_minus'] = False
%matplotlib inline
sns.set_style('darkgrid')
 sns.set_palette('bone')
 warnings.filterwarnings('ignore')
 gc.enable()
INPUT_DIR = ".. /input/"<data_type_conversions>
combdata = combdata.drop(labels='Embarked', axis=1 )
def fillInf(df, val):
numcols = df.select_dtypes(include='number' ).columns
cols = numcols[numcols != 'winPlacePerc']
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
for c in cols: df[c].fillna(val, inplace=True )<load_from_csv>
combdata =combdata.drop(labels=['Age', 'Fare', 'Name'],axis = 1 )
def feature_engineering(is_train=True):
if is_train:
print("processing train.csv")
 df = pd.read_csv(INPUT_DIR + 'train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv(INPUT_DIR + 'test_V2.csv')
 df.dropna(inplace=True)
 df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
match = df.groupby('matchId')
 df['killPlacePerc'] = match['kills'].rank(pct=True ).values
df['walkDistancePerc'] = match['walkDistance'].rank(pct=True ).values
df['_totalDistance'] = df['rideDistance'] + df['walkDistance'] + df['swimDistance']
df['zombi'] =(( df['_totalDistance'] == 0)|(df['kills'] == 0)
 |(df['weaponsAcquired'] == 0)
 |(df['matchType'].str.contains('solo')) ).astype(int)
 df['cheater'] =(( df['kills'] / df['_totalDistance'] >= 1)
 |(df['kills'] > 30)|(df['roadKills'] > 10)).astype(int)
 pd.concat([df['zombi'].value_counts() , df['cheater'].value_counts() ], axis=1 ).T
df['_healthAndBoosts'] = df['heals'] + df['boosts']
df['_killDamage'] = df['kills'] * 100 + df['damageDealt']
df['_killPlaceOverMaxPlace'] = df['killPlace'] / df['maxPlace']
df['_killsOverWalkDistance'] = df['kills'] / df['walkDistance']
df['_walkDistancePerSec'] = df['walkDistance'] / df['matchDuration']
fillInf(df, 0)
 mapper = lambda x: 'solo' if('solo' in x)else 'duo' if('duo' in x)or('crash' in x)else 'squad'
df['matchType'] = df['matchType'].map(mapper)
 df['matchType'] = df['matchType'].map(mapper)
 a = pd.get_dummies(df['matchType'], prefix='matchType')
 df = pd.concat([df, a], axis=1)
 df.drop(['headshotKills','teamKills','roadKills','vehicleDestroys'], axis=1, inplace=True)
 df.drop(['rideDistance','swimDistance','matchDuration'], axis=1, inplace=True)
 df.drop(['rankPoints','killPoints','winPoints'], axis=1, inplace=True)
 df.drop(['matchType'], axis=1, inplace=True)
 print("remove some columns")
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 y = None
print("get target")
 if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 del agg, agg_rank
gc.collect()
print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 del agg, agg_rank
gc.collect()
print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 del agg, agg_rank
gc.collect()
print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 del agg
gc.collect()
print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 gc.collect()
df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = np.array(df_out, dtype=np.float64)
 feature_names = list(df_out.columns)
 del df, df_out, agg
gc.collect()
return X, y, feature_names<normalization>
from sklearn.svm import SVC
from collections import Counter
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier
x_train, y, feature_names = feature_engineering(True)
 scaler = MinMaxScaler(feature_range=(-1, 1), copy=False ).fit(x_train)
 scaler.transform(x_train )<normalization>
train = combdata.loc[combdata['source']=="train"]
test = combdata.loc[combdata['source']=="test"]
test.drop(labels=["Survived"],axis = 1,inplace=True)
 train.drop(labels=["source"],axis = 1,inplace=True)
 test.drop(labels=["source"],axis = 1,inplace=True)
 test.shape
x_prediction, _, _ = feature_engineering(False)
 scaler = MinMaxScaler(feature_range=(-1, 1), copy=False ).fit(x_prediction)
 scaler.transform(x_prediction )<split>
train["Survived"] = train["Survived"].astype(int)
 Y_train = train["Survived"]
X_train = train.drop(labels = ["Survived"],axis = 1)
 X_train.shape
X_train,X_test, y_train, y_test =train_test_split(x_train,y,test_size=0.3, random_state=0 )<train_model>
kfold = StratifiedKFold(n_splits=10 )
%%time
linreg = LinearRegression()
linreg.fit(X_train, y_train)
 print(linreg.intercept_)
 print(linreg.coef_ )<compute_test_metric>
random_state = 2
classifiers = []
classifiers.append(KNeighborsClassifier())
 classifiers.append(LinearDiscriminantAnalysis())
 classifiers.append(SVC(random_state=random_state))
classifiers.append(MLPClassifier(random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))
classifiers.append(LogisticRegression(random_state = random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),random_state=random_state,learning_rate=0.1))
cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, X_train, y = Y_train, scoring = "accuracy", cv = kfold, n_jobs=4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,
"Algorithm":["SVC",
"AdaBoost",
"ExtraTrees",
"KNeighboors",
"DecisionTree",
"RandomForest",
"GradientBoosting",
"LogisticRegression",
"MultipleLayerPerceptron",
"LinearDiscriminantAnalysis"]})
 g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores" )
<compute_test_metric>
DTC = DecisionTreeClassifier()
adaDTC = AdaBoostClassifier(DTC, random_state=7)
 ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
"base_estimator__splitter" : ["best", "random"],
"algorithm" : ["SAMME","SAMME.R"],
"n_estimators" :[1,2],
"learning_rate": [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3,1.5]}
gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsadaDTC.fit(X_train,Y_train)
 ada_best = gsadaDTC.best_estimator_
gsadaDTC.best_score_
<predict_on_test>
ExtC = ExtraTreesClassifier()
ex_param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsExtC.fit(X_train,Y_train)
 ExtC_best = gsExtC.best_estimator_
gsExtC.best_score_
%%time
result = linreg.predict(x_prediction )<load_from_csv>
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsRFC.fit(X_train,Y_train)
 RFC_best = gsRFC.best_estimator_
gsRFC.best_score_
%%time
test_data = pd.read_csv(INPUT_DIR+'test_V2.csv')
 print("fix winPlacePerc")
 for i in range(len(test_data)) :
winPlacePerc = result[i]
maxPlace = int(test_data.iloc[i]['maxPlace'])
 if maxPlace == 0:
winPlacePerc = 0.0
elif maxPlace == 1:
winPlacePerc = 1.0
else:
gap = 1.0 /(maxPlace - 1)
 winPlacePerc = round(winPlacePerc / gap)* gap
if winPlacePerc < 0: winPlacePerc = 0.0
if winPlacePerc > 1: winPlacePerc = 1.0
result[i] = winPlacePerc<save_to_csv>
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsGBC.fit(X_train,Y_train)
 GBC_best = gsGBC.best_estimator_
gsGBC.best_score_
f3=open(INPUT_DIR+'sample_submission_V2.csv')
 submit=pd.read_csv(f3)
 sample_result = pd.DataFrame(result,columns = ['winPlacePerc'])
 submit['winPlacePerc'] = sample_result
submit.to_csv(r'sample_submission_lineregression.csv', index=False)
 del f3,result,submit
gc.collect()<train_model>
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'],
'gamma': [ 0.001, 0.01, 0.1, 1],
'C': [1, 10, 50, 100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsSVMC.fit(X_train,Y_train)
 SVMC_best = gsSVMC.best_estimator_
gsSVMC.best_score_
%%time
model_lasso = Lasso(alpha=0.001)
 model_lasso.fit(X_train, y_train)
 print(model_lasso.intercept_)
 print(model_lasso.coef_ )<predict_on_test>
votingC = VotingClassifier(estimators=[('rfc', RFC_best),('extc', ExtC_best), 
('svc', SVMC_best),('adac',ada_best),('gbc',GBC_best)], voting='soft', n_jobs=4)
 votingC = votingC.fit(X_train, Y_train )
%%time
predicted_lasso = model_lasso.predict(x_prediction )<load_from_csv>
test_Survived = pd.Series(votingC.predict(test), name="Survived")
 results = pd.concat([IDtest,test_Survived],axis=1)
 results.to_csv("Final Submission File.csv",index=False )