Kaggle competition: Dogs vs. Cats Redux: Kernels Edition
data_raw = pd.read_csv('.. /input/train.csv')
 data_val = pd.read_csv('.. /input/test.csv')
 data1 = data_raw.copy(deep = True)
 data_cleaner = [data1, data_val]
print(data_raw.info())
 data_raw.sample(10 )<data_type_conversions>
from fastai.imports import *
from fastai.transforms import *
from fastai.conv_learner import *
from fastai.model import *
from fastai.dataset import *
from fastai.sgdr import *
from fastai.plots import *
for dataset in data_cleaner:
dataset['Age'].fillna(dataset['Age'].median() , inplace = True)
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace = True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 drop_column = ['PassengerId','Cabin', 'Ticket']
data1.drop(drop_column, axis=1, inplace = True)
 print(data1.isnull().sum())
 print("-"*10)
 print(data_val.isnull().sum() )<feature_engineering>
torch.cuda.is_available()
for dataset in data_cleaner:
dataset['FamilySize'] = dataset ['SibSp'] + dataset['Parch'] + 1
dataset['IsAlone'] = 1
dataset['IsAlone'].loc[dataset['FamilySize'] > 1] = 0
dataset['Title'] = dataset['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]
dataset['FareBin'] = pd.qcut(dataset['Fare'], 4)
 dataset['AgeBin'] = pd.cut(dataset['Age'].astype(int), 5)
 stat_min = 10
title_names =(data1['Title'].value_counts() < stat_min)
 data1['Title'] = data1['Title'].apply(lambda x: 'Misc' if title_names.loc[x] == True else x)
 print(data1['Title'].value_counts())
 print("-"*10)
 data1.info()
data_val.info()
data1.sample(10 )<categorify>
torch.backends.cudnn.enabled
label = LabelEncoder()
for dataset in data_cleaner:
dataset['Sex_Code'] = label.fit_transform(dataset['Sex'])
 dataset['Embarked_Code'] = label.fit_transform(dataset['Embarked'])
 dataset['Title_Code'] = label.fit_transform(dataset['Title'])
 dataset['AgeBin_Code'] = label.fit_transform(dataset['AgeBin'])
 dataset['FareBin_Code'] = label.fit_transform(dataset['FareBin'])
 Target = ['Survived']
data1_x = ['Sex','Pclass', 'Embarked', 'Title','SibSp', 'Parch', 'Age', 'Fare', 'FamilySize', 'IsAlone']
data1_x_calc = ['Sex_Code','Pclass', 'Embarked_Code', 'Title_Code','SibSp', 'Parch', 'Age', 'Fare']
data1_xy = Target + data1_x
print('Original X Y: ', data1_xy, '
')
 data1_x_bin = ['Sex_Code','Pclass', 'Embarked_Code', 'Title_Code', 'FamilySize', 'AgeBin_Code', 'FareBin_Code']
data1_xy_bin = Target + data1_x_bin
print('Bin X Y: ', data1_xy_bin, '
')
 data1_dummy = pd.get_dummies(data1[data1_x])
 data1_x_dummy = data1_dummy.columns.tolist()
data1_xy_dummy = Target + data1_x_dummy
print('Dummy X Y: ', data1_xy_dummy, '
')
 data1_dummy.head() 
<split>
dog_indexes = []
cat_indexes = []
count = 0
cat_count = 1
dog_count = 1
train1_x, test1_x, train1_y, test1_y = model_selection.train_test_split(data1[data1_x_calc], data1[Target], random_state = 0)
 train1_x_bin, test1_x_bin, train1_y_bin, test1_y_bin = model_selection.train_test_split(data1[data1_x_bin], data1[Target] , random_state = 0)
 train1_x_dummy, test1_x_dummy, train1_y_dummy, test1_y_dummy = model_selection.train_test_split(data1_dummy[data1_x_dummy], data1[Target], random_state = 0)
 print("Data1 Shape: {}".format(data1.shape))
print("Train1 Shape: {}".format(train1_x.shape))
print("Test1 Shape: {}".format(test1_x.shape))
train1_x_bin.head()<set_options>
for name in os.listdir('.. /input/dogs-vs-cats-redux-kernels-edition/train/'):
if 'cat' in name and cat_count <= 8000:
cat_indexes.append(name.split('.')[1])
 cat_count += 1
if 'dog' in name and dog_count <= 8000:
dog_indexes.append(name.split('.')[1])
 dog_count += 1
if dog_count > 8000 and cat_count > 8000:
break
plt.figure(figsize=[16,12])
 plt.subplot(231)
 plt.boxplot(x=data1['Fare'], showmeans = True, meanline = True)
 plt.title('Fare Boxplot')
 plt.ylabel('Fare($)')
 plt.subplot(232)
 plt.boxplot(data1['Age'], showmeans = True, meanline = True)
 plt.title('Age Boxplot')
 plt.ylabel('Age(Years)')
 plt.subplot(233)
 plt.boxplot(data1['FamilySize'], showmeans = True, meanline = True)
 plt.title('Family Size Boxplot')
 plt.ylabel('Family Size(
 plt.subplot(234)
 plt.hist(x = [data1[data1['Survived']==1]['Fare'], data1[data1['Survived']==0]['Fare']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Fare Histogram by Survival')
 plt.xlabel('Fare($)')
 plt.ylabel('
plt.legend()
plt.subplot(235)
 plt.hist(x = [data1[data1['Survived']==1]['Age'], data1[data1['Survived']==0]['Age']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Age Histogram by Survival')
 plt.xlabel('Age(Years)')
 plt.ylabel('
plt.legend()
plt.subplot(236)
 plt.hist(x = [data1[data1['Survived']==1]['FamilySize'], data1[data1['Survived']==0]['FamilySize']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Family Size Histogram by Survival')
 plt.xlabel('Family Size(
 plt.ylabel('
plt.legend() 
<choose_model_class>
train_dir = ".. /input/dogs-vs-cats-redux-kernels-edition/train/"
test_dir = ".. /input/dogs-vs-cats-redux-kernels-edition/test/"
cat_train_dir = ".. /working/dogcats/train/cats/"
cat_valid_dir = ".. /working/dogcats/valid/cats/"
dog_train_dir = ".. /working/dogcats/train/dogs/"
dog_valid_dir = ".. /working/dogcats/valid/dogs/"
dogcats_test = ".. /working/dogcats/test/"
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
gaussian_process.GaussianProcessClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
svm.SVC(probability=True),
svm.NuSVC(probability=True),
svm.LinearSVC() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
discriminant_analysis.LinearDiscriminantAnalysis() ,
discriminant_analysis.QuadraticDiscriminantAnalysis() ,
XGBClassifier()
]
cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size =.3, train_size =.6, random_state = 0)
 MLA_columns = ['MLA Name', 'MLA Parameters','MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Test Accuracy 3*STD' ,'MLA Time']
MLA_compare = pd.DataFrame(columns = MLA_columns)
 MLA_predict = data1[Target]
row_index = 0
for alg in MLA:
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
 cv_results = model_selection.cross_validate(alg, data1[data1_x_bin], data1[Target], cv = cv_split)
 MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA Train Accuracy Mean'] = cv_results['train_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy 3*STD'] = cv_results['test_score'].std() *3
alg.fit(data1[data1_x_bin], data1[Target])
 MLA_predict[MLA_name] = alg.predict(data1[data1_x_bin])
 row_index+=1
MLA_compare.sort_values(by = ['MLA Test Accuracy Mean'], ascending = False, inplace = True)
 MLA_compare 
<define_variables>
PATH = ".. /working/dogcats/"
sz=224
for index, row in data1.iterrows() :
if random.random() >.5:
data1.set_value(index, 'Random_Predict', 1)
 else:
data1.set_value(index, 'Random_Predict', 0)
 data1['Random_Score'] = 0
data1.loc[(data1['Survived'] == data1['Random_Predict']), 'Random_Score'] = 1
print('Coin Flip Model Accuracy: {:.2f}%'.format(data1['Random_Score'].mean() *100))
print('Coin Flip Model Accuracy w/SciKit: {:.2f}%'.format(metrics.accuracy_score(data1['Survived'], data1['Random_Predict'])*100)) 
<groupby>
for jpgfile in iglob(os.path.join(test_dir, "*.jpg")) :
shutil.copy(jpgfile, dogcats_test )
pivot_female = data1[data1.Sex=='female'].groupby(['Sex','Pclass', 'Embarked','FareBin'])['Survived'].mean()
print('Survival Decision Tree w/Female Node:
',pivot_female)
 pivot_male = data1[data1.Sex=='male'].groupby(['Sex','Title'])['Survived'].mean()
print('
Survival Decision Tree w/Male Node:
',pivot_male )<create_dataframe>
gc.collect()
def mytree(df):
Model = pd.DataFrame(data = {'Predict':[]})
 male_title = ['Master']
for index, row in df.iterrows() :
Model.loc[index, 'Predict'] = 0
if(df.loc[index, 'Sex'] == 'female'):
Model.loc[index, 'Predict'] = 1
if(( df.loc[index, 'Sex'] == 'female')& 
(df.loc[index, 'Pclass'] == 3)& 
(df.loc[index, 'Embarked'] == 'S')& 
(df.loc[index, 'Fare'] > 8)
):
Model.loc[index, 'Predict'] = 0
if(( df.loc[index, 'Sex'] == 'male')& 
(df.loc[index, 'Title'] in male_title)
):
Model.loc[index, 'Predict'] = 1
return Model
Tree_Predict = mytree(data1)
 print('Decision Tree Model Accuracy/Precision Score: {:.2f}%
'.format(metrics.accuracy_score(data1['Survived'], Tree_Predict)*100))
print(metrics.classification_report(data1['Survived'], Tree_Predict)) 
<train_on_grid>
tfms = tfms_from_model(resnet34, sz, aug_tfms=transforms_side_on, max_zoom=1.1 )
dtree = tree.DecisionTreeClassifier(random_state = 0)
 base_results = model_selection.cross_validate(dtree, data1[data1_x_bin], data1[Target], cv = cv_split)
 dtree.fit(data1[data1_x_bin], data1[Target])
 print('BEFORE DT Parameters: ', dtree.get_params())
 print("BEFORE DT Training w/bin score mean: {:.2f}".format(base_results['train_score'].mean() *100))
print("BEFORE DT Test w/bin score mean: {:.2f}".format(base_results['test_score'].mean() *100))
print("BEFORE DT Test w/bin score 3*std: +/- {:.2f}".format(base_results['test_score'].std() *100*3))
print('-'*10)
 param_grid = {'criterion': ['gini', 'entropy'],
'max_depth': [2,4,6,8,10,None],
'random_state': [0]
}
tune_model = model_selection.GridSearchCV(tree.DecisionTreeClassifier() , param_grid=param_grid, scoring = 'roc_auc', cv = cv_split)
 tune_model.fit(data1[data1_x_bin], data1[Target])
 print('AFTER DT Parameters: ', tune_model.best_params_)
 print("AFTER DT Training w/bin score mean: {:.2f}".format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100))
print("AFTER DT Test w/bin score mean: {:.2f}".format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT Test w/bin score 3*std: +/- {:.2f}".format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
<train_model>
arch=resnet34
print('BEFORE DT RFE Training Shape Old: ', data1[data1_x_bin].shape)
 print('BEFORE DT RFE Training Columns Old: ', data1[data1_x_bin].columns.values)
 print("BEFORE DT RFE Training w/bin score mean: {:.2f}".format(base_results['train_score'].mean() *100))
print("BEFORE DT RFE Test w/bin score mean: {:.2f}".format(base_results['test_score'].mean() *100))
print("BEFORE DT RFE Test w/bin score 3*std: +/- {:.2f}".format(base_results['test_score'].std() *100*3))
print('-'*10)
 dtree_rfe = feature_selection.RFECV(dtree, step = 1, scoring = 'accuracy', cv = cv_split)
 dtree_rfe.fit(data1[data1_x_bin], data1[Target])
 X_rfe = data1[data1_x_bin].columns.values[dtree_rfe.get_support() ]
rfe_results = model_selection.cross_validate(dtree, data1[X_rfe], data1[Target], cv = cv_split)
 print('AFTER DT RFE Training Shape New: ', data1[X_rfe].shape)
 print('AFTER DT RFE Training Columns New: ', X_rfe)
 print("AFTER DT RFE Training w/bin score mean: {:.2f}".format(rfe_results['train_score'].mean() *100))
print("AFTER DT RFE Test w/bin score mean: {:.2f}".format(rfe_results['test_score'].mean() *100))
print("AFTER DT RFE Test w/bin score 3*std: +/- {:.2f}".format(rfe_results['test_score'].std() *100*3))
print('-'*10)
 rfe_tune_model = model_selection.GridSearchCV(tree.DecisionTreeClassifier() , param_grid=param_grid, scoring = 'roc_auc', cv = cv_split)
 rfe_tune_model.fit(data1[X_rfe], data1[Target])
 print('AFTER DT RFE Tuned Parameters: ', rfe_tune_model.best_params_)
 print("AFTER DT RFE Tuned Training w/bin score mean: {:.2f}".format(rfe_tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100))
print("AFTER DT RFE Tuned Test w/bin score mean: {:.2f}".format(rfe_tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT RFE Tuned Test w/bin score 3*std: +/- {:.2f}".format(rfe_tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10 )<save_to_csv>
data = ImageClassifierData.from_paths(PATH, tfms=tfms, test_name='test')
 learn = ConvLearner.pretrained(arch, data, precompute=True )
dot_data = tree.export_graphviz(dtree, out_file=None,
feature_names = data1_x_bin, class_names = True,
filled = True, rounded = True)
 graph = graphviz.Source(dot_data)
 graph<choose_model_class>
learn.fit(1e-2, 1 )
vote_est = [ 
('ada', ensemble.AdaBoostClassifier()), 
('bc', ensemble.BaggingClassifier()), 
('etc',ensemble.ExtraTreesClassifier()), 
('gbc', ensemble.GradientBoostingClassifier()), 
('rfc', ensemble.RandomForestClassifier()), 
('gpc', gaussian_process.GaussianProcessClassifier()), 
('lr', linear_model.LogisticRegressionCV()), 
('bnb', naive_bayes.BernoulliNB()), 
('gnb', naive_bayes.GaussianNB()), 
('knn', neighbors.KNeighborsClassifier()), 
('svc', svm.SVC(probability=True)) , 
('xgb', XGBClassifier())
 ]
vote_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 vote_hard_cv = model_selection.cross_validate(vote_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_hard.fit(data1[data1_x_bin], data1[Target])
 print("Hard Voting Training w/bin score mean: {:.2f}".format(vote_hard_cv['train_score'].mean() *100))
print("Hard Voting Test w/bin score mean: {:.2f}".format(vote_hard_cv['test_score'].mean() *100))
print("Hard Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_hard_cv['test_score'].std() *100*3))
print('-'*10)
 vote_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 vote_soft_cv = model_selection.cross_validate(vote_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_soft.fit(data1[data1_x_bin], data1[Target])
 print("Soft Voting Training w/bin score mean: {:.2f}".format(vote_soft_cv['train_score'].mean() *100))
print("Soft Voting Test w/bin score mean: {:.2f}".format(vote_soft_cv['test_score'].mean() *100))
print("Soft Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_soft_cv['test_score'].std() *100*3))
print('-'*10)
<define_search_space>
learn.precompute=False
grid_n_estimator = [50,100,300]
grid_ratio = [.1,.25,.5,.75,1.0]
grid_learn = [.01,.03,.05,.1,.25]
grid_max_depth = [2,4,6,None]
grid_min_samples = [5,10,.03,.05,.10]
grid_criterion = ['gini', 'entropy']
grid_bool = [True, False]
grid_seed = [0]
vote_param = [{
'ada__n_estimators': grid_n_estimator,
'ada__learning_rate': grid_ratio,
'ada__algorithm': ['SAMME', 'SAMME.R'],
'ada__random_state': grid_seed,
'bc__n_estimators': grid_n_estimator,
'bc__max_samples': grid_ratio,
'bc__oob_score': grid_bool,
'bc__random_state': grid_seed,
'etc__n_estimators': grid_n_estimator,
'etc__criterion': grid_criterion,
'etc__max_depth': grid_max_depth,
'etc__random_state': grid_seed,
'gbc__loss': ['deviance', 'exponential'],
'gbc__learning_rate': grid_ratio,
'gbc__n_estimators': grid_n_estimator,
'gbc__criterion': ['friedman_mse', 'mse', 'mae'],
'gbc__max_depth': grid_max_depth,
'gbc__min_samples_split': grid_min_samples,
'gbc__min_samples_leaf': grid_min_samples,
'gbc__random_state': grid_seed,
'rfc__n_estimators': grid_n_estimator,
'rfc__criterion': grid_criterion,
'rfc__max_depth': grid_max_depth,
'rfc__min_samples_split': grid_min_samples,
'rfc__min_samples_leaf': grid_min_samples,
'rfc__bootstrap': grid_bool,
'rfc__oob_score': grid_bool,
'rfc__random_state': grid_seed,
'lr__fit_intercept': grid_bool,
'lr__penalty': ['l1','l2'],
'lr__solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],
'lr__random_state': grid_seed,
'bnb__alpha': grid_ratio,
'bnb__prior': grid_bool,
'bnb__random_state': grid_seed,
'knn__n_neighbors': [1,2,3,4,5,6,7],
'knn__weights': ['uniform', 'distance'],
'knn__algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'],
'knn__random_state': grid_seed,
'svc__kernel': ['linear', 'poly', 'rbf', 'sigmoid'],
'svc__C': grid_max_depth,
'svc__gamma': grid_ratio,
'svc__decision_function_shape': ['ovo', 'ovr'],
'svc__probability': [True],
'svc__random_state': grid_seed,
'xgb__learning_rate': grid_ratio,
'xgb__max_depth': [2,4,6,8,10],
'xgb__tree_method': ['exact', 'approx', 'hist'],
'xgb__objective': ['reg:linear', 'reg:logistic', 'binary:logistic'],
'xgb__seed': grid_seed
}] 
<define_search_space>
learn.fit(1e-2, 3, cycle_len=1 )
grid_n_estimator = [10, 50, 100, 300]
grid_ratio = [.1,.25,.5,.75, 1.0]
grid_learn = [.01,.03,.05,.1,.25]
grid_max_depth = [2, 4, 6, 8, 10, None]
grid_min_samples = [5, 10,.03,.05,.10]
grid_criterion = ['gini', 'entropy']
grid_bool = [True, False]
grid_seed = [0]
grid_param = [
[{
'n_estimators': grid_n_estimator,
'learning_rate': grid_learn,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'max_samples': grid_ratio,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'criterion': grid_criterion,
'max_depth': grid_max_depth,
'random_state': grid_seed
}],
[{
'learning_rate': [.05],
'n_estimators': [300],
'max_depth': grid_max_depth,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'criterion': grid_criterion,
'max_depth': grid_max_depth,
'oob_score': [True],
'random_state': grid_seed
}],
[{
'max_iter_predict': grid_n_estimator,
'random_state': grid_seed
}],
[{
'fit_intercept': grid_bool,
'solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],
'random_state': grid_seed
}],
[{
'alpha': grid_ratio,
}],
[{}],
[{
'n_neighbors': [1,2,3,4,5,6,7],
'weights': ['uniform', 'distance'],
'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute']
}],
[{
'C': [1,2,3,4,5],
'gamma': grid_ratio,
'decision_function_shape': ['ovo', 'ovr'],
'probability': [True],
'random_state': grid_seed
}],
[{
'learning_rate': grid_learn,
'max_depth': [1,2,4,6,8,10],
'n_estimators': grid_n_estimator,
'seed': grid_seed
}]
]
start_total = time.perf_counter()
for clf, param in zip(vote_est, grid_param):
start = time.perf_counter()
best_search = model_selection.GridSearchCV(estimator = clf[1], param_grid = param, cv = cv_split, scoring = 'roc_auc')
 best_search.fit(data1[data1_x_bin], data1[Target])
 run = time.perf_counter() - start
best_param = best_search.best_params_
print('The best parameter for {} is {} with a runtime of {:.2f} seconds.'.format(clf[1].__class__.__name__, best_param, run))
clf[1].set_params(**best_param)
 run_total = time.perf_counter() - start_total
print('Total optimization time was {:.2f} minutes.'.format(run_total/60))
print('-'*10 )<train_on_grid>
gc.collect()
grid_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_hard.fit(data1[data1_x_bin], data1[Target])
 print("Hard Voting w/Tuned Hyperparameters Training w/bin score mean: {:.2f}".format(grid_hard_cv['train_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_hard_cv['test_score'].std() *100*3))
print('-'*10)
 grid_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_soft.fit(data1[data1_x_bin], data1[Target])
 print("Soft Voting w/Tuned Hyperparameters Training w/bin score mean: {:.2f}".format(grid_soft_cv['train_score'].mean() *100))
print("Soft Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_soft_cv['test_score'].mean() *100))
print("Soft Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_soft_cv['test_score'].std() *100*3))
print('-'*10)
<train_on_grid>
lr=np.array([1e-4,1e-3,1e-2] )
print(data_val.info())
 print("-"*10)
 data_val['Survived'] = mytree(data_val ).astype(int)
 data_val['Survived'] = grid_hard.predict(data_val[data1_x_bin])
 submit = data_val[['PassengerId','Survived']]
submit.to_csv(".. /working/submit.csv", index=False)
 print('Validation Data Distribution:
', data_val['Survived'].value_counts(normalize = True))
submit.sample(10)
<import_modules>
learn.fit(lr, 3, cycle_len=1, cycle_mult=2 )
Embedding,
BatchNormalization,
Dense,
TimeDistributed,
Dropout,
Bidirectional,
Flatten,
GlobalMaxPool1D)
 precision_score,
recall_score,
f1_score,
classification_report,
accuracy_score 
)
<load_from_csv>
log_preds,y = learn.TTA()
probs = np.mean(np.exp(log_preds),0 )
dataset = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )<prepare_x_and_y>
accuracy_np(probs, y )
train = dataset.text.values
test = test.text.values
sentiments = dataset.target.values<train_model>
preds = np.argmax(probs, axis=1)
 probs = probs[:,1]
word_tokenizer = Tokenizer()
word_tokenizer.fit_on_texts(train)
 vocab_length = len(word_tokenizer.word_index)+ 1<compute_test_metric>
temp = learn.predict(is_test=True )
def metrics(pred_tag, y_test):
print("F1-score: ", f1_score(pred_tag, y_test))
print("Precision: ", precision_score(pred_tag, y_test))
print("Recall: ", recall_score(pred_tag, y_test))
print("Acuracy: ", accuracy_score(pred_tag, y_test))
print("-"*50)
 print(classification_report(pred_tag, y_test))
def embed(corpus):
return word_tokenizer.texts_to_sequences(corpus )<categorify>
pred_test = np.argmax(temp, axis=1 )
longest_train = max(train, key=lambda sentence: len(word_tokenize(sentence)))
 length_long_sentence = len(word_tokenize(longest_train))
padded_sentences = pad_sequences(embed(train), length_long_sentence, padding='post')
 test_sentences = pad_sequences(
 embed(test),
length_long_sentence,
padding='post'
)<feature_engineering>
probs = np.exp(temp[:,1] )
embeddings_dictionary = dict()
embedding_dim = 200
glove_file = open('.. /input/glove-twitter/glove.twitter.27B.' + str(embedding_dim)+ 'd.txt', encoding="utf8")
 for line in glove_file:
records = line.split()
word = records[0]
vector_dimensions = np.asarray(records[1:], dtype='float32')
 embeddings_dictionary [word] = vector_dimensions
glove_file.close()<feature_engineering>
submission = pd.DataFrame({'id':os.listdir(f'{PATH}test'), 'label':probs} )
embedding_matrix = np.zeros(( vocab_length, embedding_dim))
for word, index in word_tokenizer.word_index.items() :
if index >= vocab_length:
continue
embedding_vector = embeddings_dictionary.get(word)
 if embedding_vector is not None:
embedding_matrix[index] = embedding_vector<choose_model_class>
submission['id'] = submission['id'].map(lambda x: x.split('.')[0] )
def BLSTM() :
model = Sequential()
model.add(Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights = [embedding_matrix],
input_length=length_long_sentence,
trainable=False))
model.add(Bidirectional(LSTM(16)))
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
 model.summary()
return model<choose_model_class>
submission['id'] = submission['id'].astype(int )
def BLSTM() :
model = Sequential()
model.add(Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights = [embedding_matrix],
input_length=length_long_sentence,
trainable=False))
model.add(Bidirectional(LSTM(length_long_sentence, return_sequences = True, recurrent_dropout=0.2)))
 model.add(GlobalMaxPool1D())
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
 return model<choose_model_class>
submission = submission.sort_values('id' )
reduce_lr = ReduceLROnPlateau(
 monitor='val_loss',
factor=0.2,
verbose =1,
patience=5,
min_lr=0.001 
)
 for idx in range(5):
print("*"*20 + '
Modelo: ' + str(idx)+ '
')
 reduce_lr = ReduceLROnPlateau(
 monitor='val_loss',
factor=0.2,
verbose =1,
patience=5,
min_lr=0.001 
)
 checkpoint = ModelCheckpoint(
 'model_' + str(idx)+ '.h5',
monitor='val_loss',
mode='auto',
verbose=1,
save_weights_only = True,
save_best_only=True 
)
 X_train, X_test, y_train, y_test = train_test_split(
 padded_sentences,
sentiments,
test_size=0.5 
)
 model = BLSTM()
model.fit(X_train,
y_train,
batch_size=32,
epochs=15,
validation_data=[X_test, y_test],
callbacks = [reduce_lr, checkpoint],
verbose=1 )<load_pretrained>
submission.to_csv('.. /working/output.csv', index=False )