Kaggle competition: Aerial Cactus Identification
<SOS> metric: AUC Kaggle data source: aerial-cactus-identification<choose_model_class>
%matplotlib inline
model = build_minimal_model(bert_layer, max_len=140)
 model.summary()<split>
from keras.preprocessing.image import load_img, img_to_array
from os import listdir
import pandas as pd
import numpy as np
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.8, random_state=42)
 for train_index, test_index in sss.split(df_model, df_model['target']):
df_model = df_model.iloc[test_index.tolist() + train_index.tolist() , :]<define_variables>
def loadImages(path):
count = 1
images = []
imageList = sorted(listdir(path))
for i in imageList:
image = load_img(path + i)
 image = img_to_array(image)
 images.append(image)
 if(count % 1000 == 0):
print("Processing image", count)
 count += 1
print("Done.")
 return np.asarray(images, dtype='float')
def extract_inputs(df_in, max_len=140):
return np.array(df_in['masked_id_token'].tolist()),\
np.array(df_in['masked_list'].tolist()),\
np.array([[0] * max_len for i in range(0, df_in.shape[0])])
 train_input = extract_inputs(df_model)
 test_input = extract_inputs(df_submission)
 train_labels = df_model['target']<define_variables>
path = ".. /input/aerial-cactus-identification/train/train/"
X_train = loadImages(path )
flag_fit_models = False<train_model>
path = ".. /input/aerial-cactus-identification/test/test/"
X_test = loadImages(path )
if flag_fit_models:
train_history_minimal_model = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
batch_size=16,
callbacks=[EarlyStopping(restore_best_weights=True, patience=1)])
 else:
train_history_minimal_model = pd.read_csv(checkpoint_dir +
'train_history_minimal_model_no_label.csv' )<save_to_csv>
Y_train = pd.read_csv(".. /input/aerial-cactus-identification/train.csv" )
if flag_fit_models:
pd.DataFrame.from_dict(train_history_minimal_model.history ).\
to_csv(output_dir + 'train_history_minimal_model_no_label.csv', index=False)
 pd.DataFrame({
'id': df_submission['id'],
'target': [X[0] for X in list(model.predict(test_input)) ]
} ).to_csv(output_dir + 'predictions_model_no_sentence_labels.csv', index=False)
 else:
df_out = pd.read_csv(checkpoint_dir + 'predictions_model_no_sentence_labels.csv' ).loc[:, ['id', 'target']]
df_out['target'] = df_out['target'].apply(lambda X: 1 if X >= 0.5 else 0)
 df_out.to_csv(output_dir + 'predictions_model_no_sentence_labels.csv', index=False )<split>
Y_train = Y_train.sort_values("id", ascending=True ).has_cactus
df_eda = pd.DataFrame(dict(all_texts=df_model['text'].str.lower().str.split() , target=df_model['target'].tolist()))
df_eda.head()<feature_engineering>
epochs = 3
batch_size = 32
df_eda['hashtag_count'] = df_eda['all_texts'].apply(lambda X: sum([1.for Y in X if Y[0] == '
df_eda['at_count'] = df_eda['all_texts'].apply(lambda X: sum([1.for Y in X if Y[0] == '@']))
df_eda['exclamation_count'] = df_eda['all_texts'].apply(lambda X: sum([1.for Y in ''.join(X)if Y == '!']))
df_eda['interrogation_count'] = df_eda['all_texts'].apply(lambda X: sum([1.for Y in ''.join(X)if Y == '?']))
df_eda.head()<categorify>
from keras.applications.xception import Xception
from keras.applications.vgg16 import VGG16
from keras.applications.vgg19 import VGG19
from keras.applications.mobilenet import MobileNet
from keras.applications.densenet import DenseNet201
from keras.applications.nasnet import NASNetLarge
df_melt_eda = pd.melt(df_eda, id_vars=['target'], value_vars=['hashtag_count', 'at_count', 'exclamation_count'])
 df_melt_eda['bigger_than_0'] = df_melt_eda['value'].apply(lambda X: 1.if X > 0 else 0)
 df_melt_eda.head()<feature_engineering>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, \
factor=0.7, min_lr=0.00001 )
df_bigger_than_0_prob = df_melt_eda.groupby(['target', 'variable'] ).\
agg({'bigger_than_0': ['mean', 'count']})
 df_bigger_than_0_prob = df_bigger_than_0_prob['bigger_than_0'].reset_index()
df_bigger_than_0_prob['target'] = df_bigger_than_0_prob['target'].apply(lambda X: 'Disaster!' if X > 0 else 'Fake...')
 df_bigger_than_0_prob['std'] = np.sqrt(( df_bigger_than_0_prob['mean'] *\ 
(1 - df_bigger_than_0_prob['mean'])) /\
df_bigger_than_0_prob['count'])
 df_bigger_than_0_prob['mean']
df_bigger_than_0_prob<feature_engineering>
datagen = ImageDataGenerator(rotation_range=30, width_shift_range=0.2,
height_shift_range=0.2, zoom_range=0.2,
horizontal_flip=True, vertical_flip=True,
validation_split=0.1 )
df_model['n_characters'] = df_model['text'].apply(lambda X: len(''.join(X.split())))
 df_model['n_']<load_from_csv>
train_generator = datagen.flow(X_train, Y_train, batch_size=batch_size, subset='training')
 val_generator = datagen.flow(X_train, Y_train, batch_size=batch_size, subset='validation' )
def index_to_row(df_in):
df_in['Epoch'] = df_in.index
return df_in
history_extra_features = index_to_row(pd.read_csv(checkpoint_dir + 'train_history_extra_features.csv'))
history_minimal_model_no_label = index_to_row(pd.read_csv(checkpoint_dir + 'train_history_minimal_model_no_label.csv'))
history_minimal_model_with_label = index_to_row(pd.read_csv(checkpoint_dir + 'train_history_minimal_model_with_label.csv'))
history_with_correction = index_to_row(pd.read_csv(checkpoint_dir + 'train_history_with_correction.csv'))<feature_engineering>
def buildModel(base_model, freeze=0.8):
threshold = int(len(base_model.layers)* freeze)
 for i in base_model.layers[:threshold]:
i.trainable = False
for i in base_model.layers[threshold:]:
i.trainable = True
X = base_model.output
X = Flatten()(X)
 X = Dense(512, activation='relu', kernel_regularizer='l2' )(X)
 X = Dense(1, activation='sigmoid' )(X)
 model = Model(inputs=base_model.input, outputs=X)
 return model
history_extra_features['Model'] = 'Extra Features'
history_minimal_model_no_label['Model'] = 'Minimal'
history_minimal_model_with_label['Model'] = 'Label'
history_with_correction['Model'] = 'Auto Correction'
df_histories = pd.concat([
history_extra_features,
history_minimal_model_no_label,
history_minimal_model_with_label,
history_with_correction
], axis=0, ignore_index=True)
 df_histories.head()<drop_column>
def fitModel(model, lr=0.0005, cp=False):
model.compile(optimizer=Adam(lr=lr), loss='binary_crossentropy', metrics=['accuracy'])
 cb = [learning_rate_reduction, checkpoint] if cp else [learning_rate_reduction]
history.append(model.fit_generator(generator=train_generator, epochs=epochs,
steps_per_epoch=int(X_train.shape[0] // batch_size * 1.5),
validation_data=val_generator,
validation_steps=int(X_train.shape[0] // batch_size * 0.4),
callbacks=cb, verbose=2))
return model
train=train.drop(train[(train['OpenPorchSF']>500)&(train['SalePrice']<100000)].index)
 train=train.drop(train[(train['BsmtFinSF1']>5000)&(train['SalePrice']<300000)].index)
 train=train.drop(train[(train['YearBuilt']<1900)&(train['SalePrice']>400000)].index)
 train=train.drop(train[(train['OverallCond']==2)&(train['SalePrice']>300000)].index)
 train=train.drop(train[(train['TotalBsmtSF']>3000)&(train['SalePrice']<300000)].index)
 train=train.drop(train[(train['GrLivArea']>3000)&(train['SalePrice']<300000)].index )<prepare_x_and_y>
history = []
ytrain=train.SalePrice.values<concatenate>
X_input = Input(( 32, 32, 3))
base_model = Xception(weights=".. /input/pretrained-models/xception_weights_tf_dim_ordering_tf_kernels_notop.h5",
include_top=False, input_tensor=X_input)
 model = buildModel(base_model)
 fitModel(model )
all_data=pd.concat(( train,test)).reset_index(drop=True )<drop_column>
X_input = Input(( 32, 32, 3))
base_model = VGG16(weights=".. /input/pretrained-models/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5",
include_top=False, input_tensor=X_input)
 model = buildModel(base_model)
 fitModel(model )
all_data=all_data.drop('SalePrice',axis=1)
<sort_values>
X_input = Input(( 32, 32, 3))
base_model = VGG19(weights=".. /input/pretrained-models/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5",
include_top=False, input_tensor=X_input)
 model = buildModel(base_model)
 fitModel(model )
miss=all_data.isnull().sum()
miss=miss[miss>0]
miss=miss.sort_values(ascending=False)
 print(miss )<categorify>
X_input = Input(( 32, 32, 3))
base_model = DenseNet201(weights=".. /input/pretrained-models/densenet201_weights_tf_dim_ordering_tf_kernels_notop.h5",
include_top=False, input_tensor=X_input)
 model = buildModel(base_model)
 fitModel(model )
all_data['PoolQC']=all_data['PoolQC'].fillna("None")
 all_data['MiscFeature']=all_data['MiscFeature'].fillna("None")
 all_data['Alley']=all_data['Alley'].fillna("None")
 all_data['Fence']=all_data['Fence'].fillna("None")
 all_data['FireplaceQu']=all_data['FireplaceQu'].fillna("None")
 all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
for col in('GarageQual','GarageCond','GarageFinish','GarageType'):
all_data[col]=all_data[col].fillna("None")
 for col in('GarageYrBlt','GarageArea','GarageCars'):
all_data[col]=all_data[col].fillna(0)
<categorify>
X_input = Input(( 32, 32, 3))
base_model = NASNetLarge(weights=".. /input/pretrained-models/NASNet-large-no-top.h5",
include_top=False, input_tensor=X_input)
 model = buildModel(base_model)
 fitModel(model )
for col in('BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2'):
all_data[col]=all_data[col].fillna("None")
 for col in('BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','BsmtFullBath','BsmtHalfBath'):
all_data[col]=all_data[col].fillna(0)
<sort_values>
epochs = 5
history = []
miss=all_data.isnull().sum()
miss=miss[miss>0]
miss=miss.sort_values(ascending=False)
 print(miss )<data_type_conversions>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=100, verbose=1, \
factor=0.7, min_lr=0.00001 )
all_data['MasVnrType']=all_data['MasVnrType'].fillna("None")
 all_data['MasVnrArea']=all_data['MasVnrArea'].fillna(0)
 all_data['MSZoning']=all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0])
 all_data=all_data.drop(['Utilities'],axis=1)
 all_data['Functional']=all_data['Functional'].fillna("Typ")
 all_data['SaleType']=all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['KitchenQual']=all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0])
 all_data['Exterior2nd']=all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['Exterior1st']=all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Electrical']=all_data['Electrical'].fillna(all_data['Electrical'].mode() [0])
<sort_values>
X_input = Input(( 32, 32, 3))
lrs = [0.00001, 0.00003, 0.0001, 0.0003, 0.001, 0.003, 0.01]
for lr in lrs:
print("=============================")
 print("Fitting model with learning_rate =", lr)
 base_model = VGG19(weights=".. /input/pretrained-models/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5",
include_top=False, input_tensor=X_input)
 model = buildModel(base_model)
 fitModel(model, lr=lr )
miss=all_data.isnull().sum()
miss=miss[miss>0]
miss=miss.sort_values(ascending=False)
 print(miss )<data_type_conversions>
epochs = 5
history = []
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None" )<drop_column>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, \
factor=0.5, min_lr=1e-5 )
all_data=all_data.drop(['Street'],axis=1 )<drop_column>
X_input = Input(( 32, 32, 3))
frs = [0, 0.25, 0.5, 0.75, 1]
for fr in frs:
print("=============================")
 print("Fitting model with freeze fraction =", fr, "and learning_rate = 1e-4")
 base_model = VGG19(weights=".. /input/pretrained-models/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5",
include_top=False, input_tensor=X_input)
 model = buildModel(base_model, freeze=fr)
 fitModel(model, lr=1e-4 )
all_data=all_data.drop(['PoolQC'],axis=1 )<data_type_conversions>
def buildModel(base_model, freeze=0.8):
threshold = int(len(base_model.layers)* freeze)
 for i in base_model.layers[:threshold]:
i.trainable = False
for i in base_model.layers[threshold:]:
i.trainable = True
X = base_model.output
X = Flatten()(X)
 X = Dense(512, use_bias=True )(X)
 X = BatchNormalization()(X)
 X = Activation('relu' )(X)
 X = Dropout(0.3 )(X)
 X = Dense(256, use_bias=True )(X)
 X = BatchNormalization()(X)
 X = Activation('relu' )(X)
 X = Dropout(0.3 )(X)
 X = Dense(1, activation='sigmoid' )(X)
 model = Model(inputs=base_model.input, outputs=X)
 return model
for col in('MSSubClass' , 'OverallCond' ,'OverallQual', 'BedroomAbvGr', 'KitchenAbvGr', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath' , 'TotRmsAbvGrd' , 'Fireplaces', 'GarageYrBlt', 'GarageCars' , 'MoSold' , 'YrSold'):
all_data[col]=all_data[col].astype(str )<define_variables>
epochs = 30
c=('MSSubClass', 'OverallQual', 'OverallCond' , 'YearRemodAdd', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath' , 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd' , 'Fireplaces', 'GarageYrBlt', 'GarageCars' , 'MoSold' , 'YrSold','MSZoning','Alley','LotShape','LandContour','LotConfig','LandSlope','Neighborhood','Condition1','Condition2','BldgType','HouseStyle','RoofStyle','RoofMatl','Exterior1st','Exterior2nd','MasVnrType','ExterQual','ExterCond','Foundation','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','Heating','HeatingQC','CentralAir','Electrical','KitchenQual','Functional','FireplaceQu','GarageType','GarageFinish','GarageQual','GarageCond','PavedDrive','Fence','MiscFeature','SaleType','SaleCondition', 'YearBuilt')
<define_variables>
X_input = Input(( 32, 32, 3))
base_model = VGG19(weights=".. /input/pretrained-models/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5",
include_top=False, input_tensor=X_input)
 model = buildModel(base_model, freeze=0.25)
 model = fitModel(model, lr=1e-4, cp=True )
cat_list=list(c)
<categorify>
submission = test_images = pd.read_csv(".. /input/aerial-cactus-identification/sample_submission.csv")
 submission = pd.DataFrame(test_images.iloc[:, 0], columns=["id"] )
for i in cat_list:
lbl = LabelEncoder()
lbl.fit(list(all_data[i].values))
all_data[i] = lbl.transform(list(all_data[i].values))<define_variables>
pred = model.predict(X_test ).reshape(-1 )
numeric_col=[]
for col in all_data.columns:
if(col not in cat_list):
numeric_col.append(col )<filter>
submission["has_cactus"] = pd.Series(pred, index=None )
skew=skew[abs(skew)> 0.75]
print(len(skew))<feature_engineering>
submission.to_csv("submission.csv", index=False )
<feature_engineering><EOS> 
submission.to_csv("submission.csv", index=False )