Kaggle competition: Aerial Cactus Identification
def write_model_results(name_str, accuracy, crossvalscores, predictions, y_test, accumulate=False):
num_type1, pct_type1, num_type2, pct_type2, pct_errors = calculate_error_types(predictions, y_test)
 tot_errors = num_type1+num_type2
crossvalscore = crossvalscores.mean()
crossval_std = crossvalscores.std()
global model_results
if 'model_results' in globals() :
if(( name_str in model_results['Model'].unique())&(not accumulate)) :
this_model = model_results.Model==name_str
column_list = ['Crossval','CV_std', 'TestAccuracy','Errors','ErrorPct','Type1','Type1Pct','Type2','Type2Pct']
model_results.loc[this_model, column_list] = [crossvalscore, crossval_std, accuracy, tot_errors,
pct_errors, num_type1, pct_type1, num_type2, pct_type2]
else:
model_results = model_results.append(
 {'Model': name_str, 'Crossval': crossvalscore, 'CV_std': crossval_std, 'TestAccuracy': accuracy,
'Errors': tot_errors,'ErrorPct': pct_errors,
'Type1': num_type1, 'Type1Pct': pct_type1, 'Type2': num_type2, 'Type2Pct': pct_type2},
ignore_index=True)
 else:
model_results = pd.DataFrame(
 data=[[name_str, crossvalscore, crossval_std, accuracy, tot_errors,
pct_errors, num_type1, pct_type1, num_type2, pct_type2]],
columns=['Model', 'Crossval', 'CV_std', 'TestAccuracy', 'Errors', 'ErrorPct',
'Type1', 'Type1Pct', 'Type2', 'Type2Pct'] )<predict_on_test>
train_dir = ".. /input/aerial-cactus-identification/train/train/"
test_dir = ".. /input/aerial-cactus-identification/test/test/"
train_df = pd.read_csv('.. /input/aerial-cactus-identification/train.csv')
 train_df.head()
def create_voted_predictions(model_predictions, survived_tilt=0):
voted_predictions = sum(model_predictions)
 num_models = len(model_predictions)
 cutoff = int(np.ceil(( num_models+1)/2)) - survived_tilt
voted_predictions[voted_predictions<cutoff] = 0
voted_predictions[voted_predictions>=cutoff] = 1
return voted_predictions<groupby>
im = cv2.imread(".. /input/aerial-cactus-identification/train/train/01e30c0ba6e91343a12d2126fcafc0dd.jpg")
 plt.imshow(im )
def get_ticket_survival_arrays(test_indexes=np.array(['none'])) :
if test_indexes[0]=='none':
train_df = train_copy.loc[:, ['Ticket','Survived','Name']]
test_df = test.loc[:, ['Ticket','Name']]
else:
train_df = train_copy.loc[~test_indexes, ['Ticket','Survived','Name']]
test_df = train_copy.loc[test_indexes, ['Ticket','Name']]
grouped = train_df.groupby(['Ticket'])
 for ticket, group in grouped:
for idx, row in group.iterrows() :
group_others = group.drop(idx)
 tot = group_others['Name'].count()
lived = group_others.loc[group_others.Survived==1,'Name'].count()
pct = lived/tot if tot>0 else -1
train_df.loc[idx, 'PctLived'] = pct
tot = group['Name'].count()
lived = group.loc[group.Survived==1,'Name'].count()
pct = lived/tot if tot>0 else -1
test_df.loc[test_df.Ticket==ticket, 'PctLived'] = pct
mean, std = train_df.loc[train_df.PctLived>-1 ,'PctLived'].agg([np.mean,np.std])
 train_df.loc[train_df.PctLived==-1,'PctLived'] = mean
test_df.loc[test_df.PctLived==-1,'PctLived'] = mean
test_df.loc[:,'PctLived'] = test_df.loc[:,'PctLived'].fillna(mean)
 train_df.loc[:,'PctLived'] = train_df.loc[:,'PctLived'].transform(lambda x:(x-mean)/std)
 test_df.loc[:,'PctLived'] = test_df.loc[:,'PctLived'].transform(lambda x:(x-mean)/std)
 return train_df.PctLived.values, test_df.PctLived.values<normalization>
X_tr = []
Y_tr = []
imges = train_df['id'].values
for img_id in tqdm_notebook(imges):
image = np.array(cv2.imread(train_dir + img_id))
X_tr.append(image)
 Y_tr.append(train_df[train_df['id'] == img_id]['has_cactus'].values[0])
 X_tr.append(np.flip(image))
Y_tr.append(train_df[train_df['id'] == img_id]['has_cactus'].values[0])
 X_tr.append(np.flipud(image))
Y_tr.append(train_df[train_df['id'] == img_id]['has_cactus'].values[0])
 X_tr.append(np.fliplr(image))
Y_tr.append(train_df[train_df['id'] == img_id]['has_cactus'].values[0])
 X_tr = np.asarray(X_tr)
 X_tr = X_tr.astype('float32')
 X_tr /= 255
Y_tr = np.asarray(Y_tr)
def drop_column_feature_importances(X_train, y_train, random_state = 0):
feature_importances = pd.DataFrame(index=train.loc[:,train.columns!='Survived'].columns)
 for model in [adaboost, logitmodel, randomforest]:
model_clone = clone(model)
 model_clone.random_state = random_state
train_with_dummies=pd.get_dummies(X_train, drop_first=False)
 X, y = np.array(train_with_dummies), np.array(y_train)
 full_scaler = preprocessing.StandardScaler().fit(X)
 X_standardized = full_scaler.transform(X)
 model_clone.fit(X_standardized, y)
 benchmark_score = model_clone.score(X_standardized, y)
 importances = []
for col in X_train.columns:
model_clone = clone(model)
 model_clone.random_state = random_state
train_with_dummies=pd.get_dummies(X_train.drop(col, axis = 1),drop_first=False)
 X = np.array(train_with_dummies)
 full_scaler = preprocessing.StandardScaler().fit(X)
 X_standardized = full_scaler.transform(X)
 model_clone.fit(X_standardized, y)
 drop_col_score = model_clone.score(X_standardized, y)
 importances.append(benchmark_score - drop_col_score)
 modelname = model.__str__()
modelname = modelname[0: modelname.find('(')]
feature_importances[modelname] = importances
feature_importances = feature_importances.transpose()
col_order = feature_importances.mean().sort_values(ascending=False ).index
chart = sns.boxplot(data=feature_importances, color='lightgrey', width=.5, order=col_order,)
 _=plt.gcf().set_size_inches(16,4)
 _=plt.gca().set_title('Feature effect on score, including variance among 3 scikit-learn models.')
 _=plt.gca().axhline(0, linestyle='--', linewidth=.7, color='indianred')
 return feature_importances 
<prepare_x_and_y>
X_tr_2 = X_tr
Y_tr_2 = Y_tr
def cross_validate_entire_process(k=10, ticket_survival_feature=False):
rows = len(X_full)
 cv_indexes = np.linspace(rows//k, rows-(rows//k), k-1 ).astype('int')
 cv_indexes = np.append(cv_indexes, rows)
 start_idx = 0
end_idx = 0
for index in cv_indexes:
end_idx = index
test_indexes = np.isin(np.arange(len(X_full)) , np.r_[start_idx:end_idx])
 X_test = X_full[test_indexes]
X_train = X_full[~test_indexes]
y_test = y_full[test_indexes]
y_train = y_full[~test_indexes]
scaler = preprocessing.StandardScaler().fit(X_train)
 X_train_standardized = scaler.transform(X_train)
 X_test_standardized = scaler.transform(X_test)
 if ticket_survival_feature:
train_survival, test_survival = get_ticket_survival_arrays(test_indexes)
 train_survival = train_survival.reshape(-1,1)
 X_train_standardized = np.append(X_train_standardized,train_survival,axis=1)
 test_survival = test_survival.reshape(-1,1)
 X_test_standardized = np.append(X_test_standardized,test_survival,axis=1)
 _=logitmodel.fit(X_train_standardized,y_train)
 logit_accuracy=logitmodel.score(X_test_standardized,y_test)
 logit_predictions = logitmodel.predict(X_test_standardized)
 _=adaboost.fit(X_train_standardized,y_train)
 adaboost_accuracy=adaboost.score(X_test_standardized,y_test)
 adaboost_predictions = adaboost.predict(X_test_standardized)
 _=randomforest.fit(X_train_standardized,y_train)
 randomforest_accuracy=randomforest.score(X_test_standardized,y_test)
 randomforest_predictions = randomforest.predict(X_test_standardized)
 deep_model_temp = make_deep_model(num_features=X_train_standardized.shape[1])
 _=deep_model_temp.fit(X_train_standardized,y_train,epochs=40,verbose=0)
 deep_accuracy = deep_model_temp.evaluate(x=X_test_standardized, y=y_test, verbose=0)[1]
deep_predictions = deep_model_temp.predict(X_test_standardized)[:, 1]
deep_predictions[deep_predictions<.5] = 0
deep_predictions[deep_predictions>=.5] = 1
deep_predictions=deep_predictions.astype('int')
 all_model_predictions = [logit_predictions, adaboost_predictions, randomforest_predictions, deep_predictions]
ada_rf_deep_predictions = [adaboost_predictions, randomforest_predictions, deep_predictions]
all_no_deep_predictions = [adaboost_predictions, logit_predictions, randomforest_predictions]
voted_allmodels_predictions = create_voted_predictions(all_model_predictions)
 voted_ada_rf_deep_predictions = create_voted_predictions(ada_rf_deep_predictions)
 voted_all_no_deep_predictions = create_voted_predictions(all_no_deep_predictions)
 voted_allmodels_tilted_predictions = create_voted_predictions(all_model_predictions, survived_tilt=1)
 voted_ada_rf_deep_tilted_predictions = create_voted_predictions(ada_rf_deep_predictions, survived_tilt=1)
 voted_all_no_deep_tilted_predictions = create_voted_predictions(all_no_deep_predictions, survived_tilt=1)
 voted_allmodels_accuracy = len(np.where(voted_allmodels_predictions==y_test)[0])/y_test.size
voted_ada_rf_deep_accuracy = len(np.where(voted_ada_rf_deep_predictions==y_test)[0])/y_test.size
voted_all_no_deep_accuracy = len(np.where(voted_all_no_deep_predictions==y_test)[0])/y_test.size
voted_allmodels_tilted_accuracy = len(np.where(voted_allmodels_tilted_predictions==y_test)[0])/y_test.size
voted_ada_rf_deep_tilted_accuracy = len(np.where(voted_ada_rf_deep_tilted_predictions==y_test)[0])/y_test.size
voted_all_no_deep_tilted_accuracy = len(np.where(voted_all_no_deep_tilted_predictions==y_test)[0])/y_test.size
for model_prediction in all_model_predictions:
success = np.array(pd.Series(model_prediction==y_test ).replace({True:1,False:0}))
model_successes[start_idx:end_idx] = model_successes[start_idx:end_idx] + success
write_model_results('LogisticRegression', logit_accuracy, np.array(0), logit_predictions, y_test,accumulate=True)
 write_model_results('AdaBoost', adaboost_accuracy, np.array(0), adaboost_predictions, y_test,accumulate=True)
 write_model_results('RandomForest', randomforest_accuracy, np.array(0), randomforest_predictions, y_test,accumulate=True)
 write_model_results('DeepModel', deep_accuracy, np.array(0), deep_predictions, y_test,accumulate=True)
 write_model_results('Voted_AllModels', voted_allmodels_accuracy, np.array(0), voted_allmodels_predictions, y_test,accumulate=True)
 write_model_results('Voted_Ada_Rf_Deep', voted_ada_rf_deep_accuracy, np.array(0), voted_ada_rf_deep_predictions, y_test,accumulate=True)
 write_model_results('Voted_All_No_Deep', voted_all_no_deep_accuracy, np.array(0), voted_all_no_deep_predictions, y_test,accumulate=True)
 write_model_results('Voted_AllModels_Tilted', voted_allmodels_tilted_accuracy, np.array(0), voted_allmodels_tilted_predictions, y_test,accumulate=True)
 write_model_results('Voted_Ada_Rf_Deep_Tilted', voted_ada_rf_deep_tilted_accuracy, np.array(0), voted_ada_rf_deep_tilted_predictions, y_test,accumulate=True)
 write_model_results('Voted_All_No_Deep_Tilted', voted_all_no_deep_tilted_accuracy, np.array(0), voted_all_no_deep_tilted_predictions, y_test,accumulate=True)
 start_idx = end_idx
aggregated_model_results = model_results.groupby('Model')\ 
.agg({'TestAccuracy':[np.mean,np.std],'Errors':np.mean,'Type1':np.mean,'Type2':np.mean})\ 
.sort_values(( 'TestAccuracy','mean'),ascending=False)\ 
.style.format({('TestAccuracy','mean'):'{:.1%}',('TestAccuracy','std'):'{:.1%}', 
('Errors','mean'):'{:.1f}',('Type1','mean'):'{:.1f}',('Type2','mean'):'{:.1f}'})\ 
.apply(lambda x: ['background: lightgreen' if x.name in ['Voted_AllModels'] else '' for i in x], axis=1)
 print('Cross-validation of models and voted models complete.Updated 'model_results' and 'model_successes'.')
 return aggregated_model_results<set_options>
X_tr = X_tr_2
Y_tr = Y_tr_2
train.head(3)
 summary = [[train[column].dtype, train[column].unique().size, train[column].isna().sum() ] for column in list(train)]
summary_df = pd.DataFrame(summary, index=list(train), columns=['data_type','unique_values','nan_values'])
 summary_df['memory_usage'] = train.memory_usage(deep=True)
 def highlight(df):
fontstyle='color:white; font-weight:bold;'
stylemap = pd.DataFrame().reindex_like(df)
 stylemap.loc[:,:] = 'background-color: '
stylemap.loc[['Age','Cabin','Embarked'],'nan_values']= 'background-color: red; '+ fontstyle
stylemap.loc[['PassengerId','Name','Ticket','Cabin'],'unique_values'] = 'background-color: orange; '+ fontstyle
stylemap.loc[['Age','Fare'],['data_type','unique_values']] = 'background-color: lightseagreen; '+ fontstyle
return stylemap
summary_df.style.apply(highlight, axis=None )<data_type_conversions>
test_image_names = []
for filename in os.listdir(test_dir):
test_image_names.append(filename)
 test_image_names.sort()
X_ts = []
for img_id in tqdm_notebook(test_image_names):
X_ts.append(cv2.imread(test_dir + img_id))
X_ts = np.asarray(X_ts)
 X_ts = X_ts.astype('float32')
 X_ts /= 255
train['HadCabin'] = train['Cabin'].notna().replace({False:0,True:1})
 train['HadAge'] = train['Age'].notna().replace({False:0,True:1})
 train['HadEmbarked'] = train['Embarked'].notna().replace({False:0,True:1} )<feature_engineering>
x_train,x_test,y_train,y_test = train_test_split(X_tr, Y_tr, test_size = 0.2 , stratify = Y_tr )
train['CabinLetter'] = train['CabinLetter_Fillna']<filter>
base=keras.applications.vgg16.VGG16(include_top=False, weights='.. /input/weights/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5',input_shape=(32,32,3))
train_ages = train.loc[:,['Title','Age']]<feature_engineering>
print("Current train size:",X_tr.shape)
 model = Sequential()
model.add(base)
 model.add(Flatten())
 model.add(Dense(256, use_bias=True))
model.add(BatchNormalization())
 model.add(Activation("relu"))
model.add(Dropout(0.5))
model.add(Dense(256,activation='relu'))
model.add(BatchNormalization())
 model.add(Dense(16, activation='tanh'))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
 model.summary()
callback=[keras.callbacks.EarlyStopping(monitor='val_acc', patience=20, verbose=1, mode='auto', restore_best_weights=True),
keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=1, mode='auto')]
model.fit(X_tr,Y_tr,batch_size=64, epochs=80, verbose=1, validation_split=0.1,callbacks=callback)
<feature_engineering><EOS> 
test_df = pd.read_csv('.. /input/aerial-cactus-identification/sample_submission.csv')
 X_test = []
imges = test_df['id'].values
for img_id in tqdm_notebook(imges):
X_test.append(cv2.imread(test_dir + img_id))
X_test = np.asarray(X_test)
 X_test = X_test.astype('float32')
 X_test /= 255
y_test_pred = model.predict_proba(X_test)
 test_df['has_cactus'] = y_test_pred
test_df.to_csv('tf_learning_vgg16_aug2_80epoch.csv', index=False )