Kaggle competition: Aerial Cactus Identification
house_train.isnull().sum().sort_values(ascending=False ).head(20 )<sort_values>
%matplotlib inline
house_test.isnull().sum().sort_values(ascending=False ).head(20 )<sort_values>
import tensorflow as tf
from tensorflow import keras
total = house_train.isnull().sum().sort_values(ascending=False)
 percent =(house_train.isnull().sum() /house_train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )<sort_values>
random.seed(0 )
total = house_test.isnull().sum().sort_values(ascending=False)
 percent =(house_test.isnull().sum() /house_test.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )<categorify>
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import InputLayer, Input
from tensorflow.keras.layers import Conv2D, Dense, Flatten, Dropout, Activation
from tensorflow.keras.layers import BatchNormalization, Reshape, MaxPooling2D, GlobalAveragePooling2D
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
to_null = ['Alley', 'MasVnrType', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'FireplaceQu',
'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'GarageYrBlt', 'BsmtFullBath', 'BsmtHalfBath',
'PoolQC', 'Fence', 'MiscFeature']
for col in to_null:
if house_train[col].dtype == 'object':
house_train[col].fillna('None',inplace=True)
 house_test[col].fillna('None',inplace=True)
 else:
house_train[col].fillna(0,inplace=True)
 house_test[col].fillna(0,inplace=True )<feature_engineering>
batch_size = 64
target_size =(32,32)
 class_mode = 'binary'
epochs = 100
input_shape =(32,32,3)
 num_classes = 2
data_dir = ".. /input/train/train/"
validation_split = 0.8
color_mode = 'rgb'
x_col = 'id'
y_col = 'has_cactus'
dropout_dense_layer = 0.5
house_test.loc[house_test.KitchenQual.isnull() , 'KitchenQual'] = 'TA'
house_test.loc[house_test.MSZoning.isnull() , 'MSZoning'] = 'RL'
house_test.loc[house_test.Utilities.isnull() , 'Utilities'] = 'AllPub'
house_test.loc[house_test.Exterior1st.isnull() , 'Exterior1st'] = 'VinylSd'
house_test.loc[house_test.Exterior2nd.isnull() , 'Exterior2nd'] = 'VinylSd'
house_test.loc[house_test.Functional.isnull() , 'Functional'] = 'Typ'
house_test.loc[house_test.SaleType.isnull() , 'SaleType'] = 'WD'
house_train.loc[house_train['Electrical'].isnull() , 'Electrical'] = 'SBrkr'
house_train.loc[house_train['LotFrontage'].isnull() , 'LotFrontage'] = house_train['LotFrontage'].mean()
house_test.loc[house_test['LotFrontage'].isnull() , 'LotFrontage'] = house_test['LotFrontage'].mean()<feature_engineering>
df = pd.read_csv(".. /input/train.csv" )
house_train.loc[house_train.MasVnrType == 'None', 'MasVnrArea'] = 0
house_test.loc[house_test.MasVnrType == 'None', 'MasVnrArea'] = 0
house_test.loc[house_test.BsmtFinType1=='None', 'BsmtFinSF1'] = 0
house_test.loc[house_test.BsmtFinType2=='None', 'BsmtFinSF2'] = 0
house_test.loc[house_test.BsmtQual=='None', 'BsmtUnfSF'] = 0
house_test.loc[house_test.BsmtQual=='None', 'TotalBsmtSF'] = 0<feature_engineering>
df.has_cactus = df.has_cactus.astype(str )
house_test.loc[house_test.GarageCars.isnull() , 'GarageCars'] = 0
house_test.loc[house_test.GarageArea.isnull() , 'GarageArea'] = 0<sort_values>
part = np.random.rand(len(df)) <validation_split
train = df[part]
validation = df[~part]
print(validation_split)
 len(df )
total = house_test.isnull().sum().sort_values(ascending=False)
 percent =(house_test.isnull().sum() /house_test.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(30 )<compute_test_metric>
train_datagen = ImageDataGenerator(rescale=1/255, horizontal_flip=True, vertical_flip=True)
 train_generator = train_datagen.flow_from_dataframe(train, directory=data_dir, x_col=x_col, y_col=y_col, target_size=target_size, color_mode=color_mode, class_mode=class_mode, batch_size=batch_size, shuffle=True)
 validation_generator = train_datagen.flow_from_dataframe(validation, directory=data_dir, x_col=x_col, y_col=y_col, target_size=target_size, color_mode=color_mode, class_mode=class_mode, batch_size=batch_size, shuffle=True )
threshold = 0.8
def correlation() :
for i in house_train.columns:
for j in house_train.columns[list(house_train.columns ).index(i)+ 1:]:
if house_train[i].dtype != 'object' and house_train[j].dtype != 'object':
if abs(pearsonr(house_train[i], house_train[j])[0])>= threshold:
yield(pearsonr(house_train[i], house_train[j])[0], i, j)
 else:
if abs(spearmanr(house_train[i], house_train[j])[0])>= threshold:
yield(spearmanr(house_train[i], house_train[j])[0], i, j )<filter>
class_weights = class_weight.compute_class_weight('balanced', np.unique(train_generator.classes), train_generator.classes)
 print(class_weights )
house_train.loc[house_train.PoolArea == 555]<data_type_conversions>
model = Sequential()
model.add(Conv2D(32,(3, 3), padding='same', activation='relu', input_shape=input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(32,(3, 3), padding='same', activation='relu', input_shape=input_shape))
model.add(BatchNormalization())
 model.add(MaxPooling2D())
 model.add(Conv2D(64,(3, 3),padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D())
 model.add(Conv2D(128,(3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D())
 model.add(Conv2D(256,(3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D())
 model.add(GlobalAveragePooling2D())
 model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(dropout_dense_layer))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss=keras.losses.binary_crossentropy, optimizer='adam', metrics=['accuracy'])
 model.summary()
house_train['MSSubClass'] = house_train['MSSubClass'].astype(str)
 house_test['MSSubClass'] = house_test['MSSubClass'].astype(str )<categorify>
callbacks = [EarlyStopping(monitor='val_loss', patience=20),
ReduceLROnPlateau(patience=10, verbose=1),
ModelCheckpoint(filepath='best_model.h5', monitor='val_loss',
verbose=0, save_best_only=True)]
for col in house_train.columns:
if house_train[col].dtype != 'object':
if skew(house_train[col])> 0.75:
house_train[col] = np.log1p(house_train[col])
 pass
else:
dummies = pd.get_dummies(house_train[col], drop_first=False)
 dummies = dummies.add_prefix("{}_".format(col))
house_train.drop(col, axis=1, inplace=True)
 house_train = house_train.join(dummies)
 for col in house_test.columns:
if house_test[col].dtype != 'object':
if skew(house_test[col])> 0.75:
house_test[col] = np.log1p(house_test[col])
 pass
else:
dummies = pd.get_dummies(house_test[col], drop_first=False)
 dummies = dummies.add_prefix("{}_".format(col))
house_test.drop(col, axis=1, inplace=True)
 house_test = house_test.join(dummies )<prepare_x_and_y>
history = model.fit_generator(train_generator,
validation_data=validation_generator,
epochs=100,
verbose=1,
shuffle=True,
callbacks=callbacks,
class_weight=class_weights )
X_train = house_train.drop('SalePrice',axis=1)
 Y_train = house_train['SalePrice']
X_test = house_test<compute_test_metric>
model.load_weights("best_model.h5")
 history.history['val_acc'][np.argmin(history.history['val_loss'])]
def rmlse(val, target):
return np.sqrt(np.sum(((np.log1p(val)- np.log1p(np.expm1(target)))**2)/ len(target)) )<split>
test_folder = ".. /input/test/"
test_datagen = ImageDataGenerator(
 rescale=1./ 255)
 test_generator = test_datagen.flow_from_directory(
 directory=test_folder,
target_size=target_size,
batch_size=1,
class_mode=None,
shuffle=False )
<predict_on_test><EOS> 
submission_file = pd.read_csv('.. /input/sample_submission.csv')
 filenames = [path.split('/')[-1] for path in test_generator.filenames]
probabilities = list(model.predict_generator(test_generator)[:,0])
 submission_file.id = filenames
submission_file.has_cactus = probabilities
submission_file.to_csv('submission_file.csv', index=False )