Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<concatenate>
print("Python version: {}".format(sys.version))
print("pandas version: {}".format(pd.__version__))
print("matplotlib version: {}".format(matplotlib.__version__))
print("NumPy version: {}".format(np.__version__))
print("SciPy version: {}".format(sp.__version__))
print("IPython version: {}".format(IPython.__version__))
print("scikit-learn version: {}".format(sklearn.__version__))
warnings.filterwarnings('ignore')
 print('-'*25)
 print(check_output(["ls", ".. /input"] ).decode("utf8"))
cartesian_test = []
cartesian_test.append(np.array(jan))
cartesian_test.append(np.array(feb))<concatenate>
%matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('white')
 pylab.rcParams['figure.figsize'] = 12,8
cartesian_test = np.vstack(cartesian_test )<create_dataframe>
data_raw = pd.read_csv('.. /input/train.csv')
 data_val = pd.read_csv('.. /input/test.csv')
 data1 = data_raw.copy(deep = True)
 data_cleaner = [data1, data_val]
print(data_raw.info())
 data_raw.sample(10 )
cartesian_test_df = pd.DataFrame(cartesian_test, columns = ['shop_id', 'item_id', 'date_block_num'] )<data_type_conversions>
for dataset in data_cleaner:
dataset['Age'].fillna(dataset['Age'].median() , inplace = True)
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace = True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 drop_column = ['PassengerId','Cabin', 'Ticket']
data1.drop(drop_column, axis=1, inplace = True)
 print(data1.isnull().sum())
 print("-"*10)
 print(data_val.isnull().sum() )
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float16)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df<create_dataframe>
for dataset in data_cleaner:
dataset['FamilySize'] = dataset ['SibSp'] + dataset['Parch'] + 1
dataset['IsAlone'] = 1
dataset['IsAlone'].loc[dataset['FamilySize'] > 1] = 0
dataset['Title'] = dataset['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]
dataset['FareBin'] = pd.qcut(dataset['Fare'], 4)
 dataset['AgeBin'] = pd.cut(dataset['Age'].astype(int), 5)
 stat_min = 10
title_names =(data1['Title'].value_counts() < stat_min)
 data1['Title'] = data1['Title'].apply(lambda x: 'Misc' if title_names.loc[x] == True else x)
 print(data1['Title'].value_counts())
 print("-"*10)
 data1.info()
data_val.info()
data1.sample(10 )
cartesian_df = pd.DataFrame(np.vstack(cartesian), columns = ['shop_id', 'item_id', 'date_block_num'], dtype=np.int32 )<groupby>
label = LabelEncoder()
for dataset in data_cleaner:
dataset['Sex_Code'] = label.fit_transform(dataset['Sex'])
 dataset['Embarked_Code'] = label.fit_transform(dataset['Embarked'])
 dataset['Title_Code'] = label.fit_transform(dataset['Title'])
 dataset['AgeBin_Code'] = label.fit_transform(dataset['AgeBin'])
 dataset['FareBin_Code'] = label.fit_transform(dataset['FareBin'])
 Target = ['Survived']
data1_x = ['Sex','Pclass', 'Embarked', 'Title','SibSp', 'Parch', 'Age', 'Fare', 'FamilySize', 'IsAlone']
data1_x_calc = ['Sex_Code','Pclass', 'Embarked_Code', 'Title_Code','SibSp', 'Parch', 'Age', 'Fare']
data1_xy = Target + data1_x
print('Original X Y: ', data1_xy, '
')
 data1_x_bin = ['Sex_Code','Pclass', 'Embarked_Code', 'Title_Code', 'FamilySize', 'AgeBin_Code', 'FareBin_Code']
data1_xy_bin = Target + data1_x_bin
print('Bin X Y: ', data1_xy_bin, '
')
 data1_dummy = pd.get_dummies(data1[data1_x])
 data1_x_dummy = data1_dummy.columns.tolist()
data1_xy_dummy = Target + data1_x_dummy
print('Dummy X Y: ', data1_xy_dummy, '
')
 data1_dummy.head()
x = train.groupby(['shop_id', 'item_id', 'date_block_num'])['item_cnt_day'].sum().rename('item_cnt_month' ).reset_index()<merge>
train1_x, test1_x, train1_y, test1_y = model_selection.train_test_split(data1[data1_x_calc], data1[Target], random_state = 0)
 train1_x_bin, test1_x_bin, train1_y_bin, test1_y_bin = model_selection.train_test_split(data1[data1_x_bin], data1[Target] , random_state = 0)
 train1_x_dummy, test1_x_dummy, train1_y_dummy, test1_y_dummy = model_selection.train_test_split(data1_dummy[data1_x_dummy], data1[Target], random_state = 0)
 print("Data1 Shape: {}".format(data1.shape))
print("Train1 Shape: {}".format(train1_x.shape))
print("Test1 Shape: {}".format(test1_x.shape))
train1_x_bin.head()
new_train = pd.merge(cartesian_df, x, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0 )<feature_engineering>
plt.figure(figsize=[16,12])
 plt.subplot(231)
 plt.boxplot(x=data1['Fare'], showmeans = True, meanline = True)
 plt.title('Fare Boxplot')
 plt.ylabel('Fare($)')
 plt.subplot(232)
 plt.boxplot(data1['Age'], showmeans = True, meanline = True)
 plt.title('Age Boxplot')
 plt.ylabel('Age(Years)')
 plt.subplot(233)
 plt.boxplot(data1['FamilySize'], showmeans = True, meanline = True)
 plt.title('Family Size Boxplot')
 plt.ylabel('Family Size(
 plt.subplot(234)
 plt.hist(x = [data1[data1['Survived']==1]['Fare'], data1[data1['Survived']==0]['Fare']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Fare Histogram by Survival')
 plt.xlabel('Fare($)')
 plt.ylabel('
plt.legend()
plt.subplot(235)
 plt.hist(x = [data1[data1['Survived']==1]['Age'], data1[data1['Survived']==0]['Age']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Age Histogram by Survival')
 plt.xlabel('Age(Years)')
 plt.ylabel('
plt.legend()
plt.subplot(236)
 plt.hist(x = [data1[data1['Survived']==1]['FamilySize'], data1[data1['Survived']==0]['FamilySize']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Family Size Histogram by Survival')
 plt.xlabel('Family Size(
 plt.ylabel('
plt.legend()
new_train['item_cnt_month'] = np.clip(new_train['item_cnt_month'], 0, 20 )<sort_values>
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
gaussian_process.GaussianProcessClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
svm.SVC(probability=True),
svm.NuSVC(probability=True),
svm.LinearSVC() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
discriminant_analysis.LinearDiscriminantAnalysis() ,
discriminant_analysis.QuadraticDiscriminantAnalysis() ,
XGBClassifier()
]
cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size =.3, train_size =.6, random_state = 0)
 MLA_columns = ['MLA Name', 'MLA Parameters','MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Test Accuracy 3*STD' ,'MLA Time']
MLA_compare = pd.DataFrame(columns = MLA_columns)
 MLA_predict = data1[Target]
row_index = 0
for alg in MLA:
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
 cv_results = model_selection.cross_validate(alg, data1[data1_x_bin], data1[Target], cv = cv_split)
 MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA Train Accuracy Mean'] = cv_results['train_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy 3*STD'] = cv_results['test_score'].std() *3
alg.fit(data1[data1_x_bin], data1[Target])
 MLA_predict[MLA_name] = alg.predict(data1[data1_x_bin])
 row_index+=1
MLA_compare.sort_values(by = ['MLA Test Accuracy Mean'], ascending = False, inplace = True)
 MLA_compare
new_train.sort_values(['date_block_num','shop_id','item_id'], inplace = True)
 new_train.head()<prepare_output>
for index, row in data1.iterrows() :
if random.random() >.5:
data1.set_value(index, 'Random_Predict', 1)
 else:
data1.set_value(index, 'Random_Predict', 0)
 data1['Random_Score'] = 0
data1.loc[(data1['Survived'] == data1['Random_Predict']), 'Random_Score'] = 1
print('Coin Flip Model Accuracy: {:.2f}%'.format(data1['Random_Score'].mean() *100))
print('Coin Flip Model Accuracy w/SciKit: {:.2f}%'.format(metrics.accuracy_score(data1['Survived'], data1['Random_Predict'])*100))
test.insert(loc=3, column='date_block_num', value=34 )<feature_engineering>
pivot_female = data1[data1.Sex=='female'].groupby(['Sex','Pclass', 'Embarked','FareBin'])['Survived'].mean()
print('Survival Decision Tree w/Female Node:
',pivot_female)
 pivot_male = data1[data1.Sex=='male'].groupby(['Sex','Title'])['Survived'].mean()
print('
Survival Decision Tree w/Male Node:
',pivot_male )
test['item_cnt_month'] = 0<concatenate>
def mytree(df):
Model = pd.DataFrame(data = {'Predict':[]})
 male_title = ['Master']
for index, row in df.iterrows() :
Model.loc[index, 'Predict'] = 0
if(df.loc[index, 'Sex'] == 'female'):
Model.loc[index, 'Predict'] = 1
if(( df.loc[index, 'Sex'] == 'female')& 
(df.loc[index, 'Pclass'] == 3)& 
(df.loc[index, 'Embarked'] == 'S')& 
(df.loc[index, 'Fare'] > 8)
):
Model.loc[index, 'Predict'] = 0
if(( df.loc[index, 'Sex'] == 'male')& 
(df.loc[index, 'Title'] in male_title)
):
Model.loc[index, 'Predict'] = 1
return Model
Tree_Predict = mytree(data1)
 print('Decision Tree Model Accuracy/Precision Score: {:.2f}%
'.format(metrics.accuracy_score(data1['Survived'], Tree_Predict)*100))
print(metrics.classification_report(data1['Survived'], Tree_Predict))
new_train = new_train.append(test.drop(['ID'], axis=1))<merge>
dtree = tree.DecisionTreeClassifier(random_state = 0)
 base_results = model_selection.cross_validate(dtree, data1[data1_x_bin], data1[Target], cv = cv_split)
 dtree.fit(data1[data1_x_bin], data1[Target])
 print('BEFORE DT Parameters: ', dtree.get_params())
 print("BEFORE DT Training w/bin score mean: {:.2f}".format(base_results['train_score'].mean() *100))
print("BEFORE DT Test w/bin score mean: {:.2f}".format(base_results['test_score'].mean() *100))
print("BEFORE DT Test w/bin score 3*std: +/- {:.2f}".format(base_results['test_score'].std() *100*3))
print('-'*10)
 param_grid = {'criterion': ['gini', 'entropy'],
'max_depth': [2,4,6,8,10,None],
'random_state': [0]
}
tune_model = model_selection.GridSearchCV(tree.DecisionTreeClassifier() , param_grid=param_grid, scoring = 'roc_auc', cv = cv_split)
 tune_model.fit(data1[data1_x_bin], data1[Target])
 print('AFTER DT Parameters: ', tune_model.best_params_)
 print("AFTER DT Training w/bin score mean: {:.2f}".format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100))
print("AFTER DT Test w/bin score mean: {:.2f}".format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT Test w/bin score 3*std: +/- {:.2f}".format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
new_train = pd.merge(new_train, shop, on=['shop_id'], how='left')
 new_train.head()<merge>
print('BEFORE DT RFE Training Shape Old: ', data1[data1_x_bin].shape)
 print('BEFORE DT RFE Training Columns Old: ', data1[data1_x_bin].columns.values)
 print("BEFORE DT RFE Training w/bin score mean: {:.2f}".format(base_results['train_score'].mean() *100))
print("BEFORE DT RFE Test w/bin score mean: {:.2f}".format(base_results['test_score'].mean() *100))
print("BEFORE DT RFE Test w/bin score 3*std: +/- {:.2f}".format(base_results['test_score'].std() *100*3))
print('-'*10)
 dtree_rfe = feature_selection.RFECV(dtree, step = 1, scoring = 'accuracy', cv = cv_split)
 dtree_rfe.fit(data1[data1_x_bin], data1[Target])
 X_rfe = data1[data1_x_bin].columns.values[dtree_rfe.get_support() ]
rfe_results = model_selection.cross_validate(dtree, data1[X_rfe], data1[Target], cv = cv_split)
 print('AFTER DT RFE Training Shape New: ', data1[X_rfe].shape)
 print('AFTER DT RFE Training Columns New: ', X_rfe)
 print("AFTER DT RFE Training w/bin score mean: {:.2f}".format(rfe_results['train_score'].mean() *100))
print("AFTER DT RFE Test w/bin score mean: {:.2f}".format(rfe_results['test_score'].mean() *100))
print("AFTER DT RFE Test w/bin score 3*std: +/- {:.2f}".format(rfe_results['test_score'].std() *100*3))
print('-'*10)
 rfe_tune_model = model_selection.GridSearchCV(tree.DecisionTreeClassifier() , param_grid=param_grid, scoring = 'roc_auc', cv = cv_split)
 rfe_tune_model.fit(data1[X_rfe], data1[Target])
 print('AFTER DT RFE Tuned Parameters: ', rfe_tune_model.best_params_)
 print("AFTER DT RFE Tuned Training w/bin score mean: {:.2f}".format(rfe_tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100))
print("AFTER DT RFE Tuned Test w/bin score mean: {:.2f}".format(rfe_tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT RFE Tuned Test w/bin score 3*std: +/- {:.2f}".format(rfe_tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10 )
new_train = pd.merge(new_train, items.drop('item_name', axis = 1), on=['item_id'], how='left')
 new_train.head()<merge>
dot_data = tree.export_graphviz(dtree, out_file=None,
feature_names = data1_x_bin, class_names = True,
filled = True, rounded = True)
 graph = graphviz.Source(dot_data)
 graph
new_train = pd.merge(new_train, item_cat.drop('item_category_name', axis = 1), on=['item_category_id'], how='left')
 new_train.head()<merge>
vote_est = [ 
('ada', ensemble.AdaBoostClassifier()), 
('bc', ensemble.BaggingClassifier()), 
('etc',ensemble.ExtraTreesClassifier()), 
('gbc', ensemble.GradientBoostingClassifier()), 
('rfc', ensemble.RandomForestClassifier()), 
('gpc', gaussian_process.GaussianProcessClassifier()), 
('lr', linear_model.LogisticRegressionCV()), 
('bnb', naive_bayes.BernoulliNB()), 
('gnb', naive_bayes.GaussianNB()), 
('knn', neighbors.KNeighborsClassifier()), 
('svc', svm.SVC(probability=True)) , 
('xgb', XGBClassifier())
 ]
vote_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 vote_hard_cv = model_selection.cross_validate(vote_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_hard.fit(data1[data1_x_bin], data1[Target])
 print("Hard Voting Training w/bin score mean: {:.2f}".format(vote_hard_cv['train_score'].mean() *100))
print("Hard Voting Test w/bin score mean: {:.2f}".format(vote_hard_cv['test_score'].mean() *100))
print("Hard Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_hard_cv['test_score'].std() *100*3))
print('-'*10)
 vote_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 vote_soft_cv = model_selection.cross_validate(vote_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_soft.fit(data1[data1_x_bin], data1[Target])
 print("Soft Voting Training w/bin score mean: {:.2f}".format(vote_soft_cv['train_score'].mean() *100))
print("Soft Voting Test w/bin score mean: {:.2f}".format(vote_soft_cv['test_score'].mean() *100))
print("Soft Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_soft_cv['test_score'].std() *100*3))
print('-'*10)
def generate_lag(train, months, lag_column):
for month in months:
train_shift = train[['date_block_num', 'shop_id', 'item_id', lag_column]].copy()
train_shift.columns = ['date_block_num', 'shop_id', 'item_id', lag_column+'_lag_'+ str(month)]
train_shift['date_block_num'] += month
train = pd.merge(train, train_shift, on=['date_block_num', 'shop_id', 'item_id'], how='left')
 return train<categorify>
grid_n_estimator = [50,100,300]
grid_ratio = [.1,.25,.5,.75,1.0]
grid_learn = [.01,.03,.05,.1,.25]
grid_max_depth = [2,4,6,None]
grid_min_samples = [5,10,.03,.05,.10]
grid_criterion = ['gini', 'entropy']
grid_bool = [True, False]
grid_seed = [0]
vote_param = [{
'ada__n_estimators': grid_n_estimator,
'ada__learning_rate': grid_ratio,
'ada__algorithm': ['SAMME', 'SAMME.R'],
'ada__random_state': grid_seed,
'bc__n_estimators': grid_n_estimator,
'bc__max_samples': grid_ratio,
'bc__oob_score': grid_bool,
'bc__random_state': grid_seed,
'etc__n_estimators': grid_n_estimator,
'etc__criterion': grid_criterion,
'etc__max_depth': grid_max_depth,
'etc__random_state': grid_seed,
'gbc__loss': ['deviance', 'exponential'],
'gbc__learning_rate': grid_ratio,
'gbc__n_estimators': grid_n_estimator,
'gbc__criterion': ['friedman_mse', 'mse', 'mae'],
'gbc__max_depth': grid_max_depth,
'gbc__min_samples_split': grid_min_samples,
'gbc__min_samples_leaf': grid_min_samples,
'gbc__random_state': grid_seed,
'rfc__n_estimators': grid_n_estimator,
'rfc__criterion': grid_criterion,
'rfc__max_depth': grid_max_depth,
'rfc__min_samples_split': grid_min_samples,
'rfc__min_samples_leaf': grid_min_samples,
'rfc__bootstrap': grid_bool,
'rfc__oob_score': grid_bool,
'rfc__random_state': grid_seed,
'lr__fit_intercept': grid_bool,
'lr__penalty': ['l1','l2'],
'lr__solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],
'lr__random_state': grid_seed,
'bnb__alpha': grid_ratio,
'bnb__prior': grid_bool,
'bnb__random_state': grid_seed,
'knn__n_neighbors': [1,2,3,4,5,6,7],
'knn__weights': ['uniform', 'distance'],
'knn__algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'],
'knn__random_state': grid_seed,
'svc__kernel': ['linear', 'poly', 'rbf', 'sigmoid'],
'svc__C': grid_max_depth,
'svc__gamma': grid_ratio,
'svc__decision_function_shape': ['ovo', 'ovr'],
'svc__probability': [True],
'svc__random_state': grid_seed,
'xgb__learning_rate': grid_ratio,
'xgb__max_depth': [2,4,6,8,10],
'xgb__tree_method': ['exact', 'approx', 'hist'],
'xgb__objective': ['reg:linear', 'reg:logistic', 'binary:logistic'],
'xgb__seed': grid_seed
}]
new_train = downcast_dtypes(new_train )<set_options>
grid_n_estimator = [10, 50, 100, 300]
grid_ratio = [.1,.25,.5,.75, 1.0]
grid_learn = [.01,.03,.05,.1,.25]
grid_max_depth = [2, 4, 6, 8, 10, None]
grid_min_samples = [5, 10,.03,.05,.10]
grid_criterion = ['gini', 'entropy']
grid_bool = [True, False]
grid_seed = [0]
grid_param = [
[{
'n_estimators': grid_n_estimator,
'learning_rate': grid_learn,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'max_samples': grid_ratio,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'criterion': grid_criterion,
'max_depth': grid_max_depth,
'random_state': grid_seed
}],
[{
'learning_rate': [.05],
'n_estimators': [300],
'max_depth': grid_max_depth,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'criterion': grid_criterion,
'max_depth': grid_max_depth,
'oob_score': [True],
'random_state': grid_seed
}],
[{
'max_iter_predict': grid_n_estimator,
'random_state': grid_seed
}],
[{
'fit_intercept': grid_bool,
'solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],
'random_state': grid_seed
}],
[{
'alpha': grid_ratio,
}],
[{}],
[{
'n_neighbors': [1,2,3,4,5,6,7],
'weights': ['uniform', 'distance'],
'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute']
}],
[{
'C': [1,2,3,4,5],
'gamma': grid_ratio,
'decision_function_shape': ['ovo', 'ovr'],
'probability': [True],
'random_state': grid_seed
}],
[{
'learning_rate': grid_learn,
'max_depth': [1,2,4,6,8,10],
'n_estimators': grid_n_estimator,
'seed': grid_seed
}]
]
start_total = time.perf_counter()
for clf, param in zip(vote_est, grid_param):
start = time.perf_counter()
best_search = model_selection.GridSearchCV(estimator = clf[1], param_grid = param, cv = cv_split, scoring = 'roc_auc')
 best_search.fit(data1[data1_x_bin], data1[Target])
 run = time.perf_counter() - start
best_param = best_search.best_params_
print('The best parameter for {} is {} with a runtime of {:.2f} seconds.'.format(clf[1].__class__.__name__, best_param, run))
clf[1].set_params(**best_param)
 run_total = time.perf_counter() - start_total
print('Total optimization time was {:.2f} minutes.'.format(run_total/60))
print('-'*10 )
gc.collect() 
<categorify>
grid_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_hard.fit(data1[data1_x_bin], data1[Target])
 print("Hard Voting w/Tuned Hyperparameters Training w/bin score mean: {:.2f}".format(grid_hard_cv['train_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_hard_cv['test_score'].std() *100*3))
print('-'*10)
 grid_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_soft.fit(data1[data1_x_bin], data1[Target])
 print("Soft Voting w/Tuned Hyperparameters Training w/bin score mean: {:.2f}".format(grid_soft_cv['train_score'].mean() *100))
print("Soft Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_soft_cv['test_score'].mean() *100))
print("Soft Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_soft_cv['test_score'].std() *100*3))
print('-'*10)
%%time
new_train = generate_lag(new_train, [1, 2, 3, 4, 5, 6, 12], 'item_cnt_month' )<merge>
print(data_val.info())
 print("-"*10)
 data_val['Survived'] = mytree(data_val ).astype(int)
 data_val['Survived'] = grid_hard.predict(data_val[data1_x_bin])
 submit = data_val[['PassengerId','Survived']]
submit.to_csv(".. /working/submit.csv", index=False)
 print('Validation Data Distribution:
', data_val['Survived'].value_counts(normalize = True))
submit.sample(10)