Kaggle competition: Aerial Cactus Identification
<SOS> metric: AUC Kaggle data source: aerial-cactus-identification<compute_test_metric>
train_dir = '.. /input/train/train/'
test_dir = '.. /input/test/test/'
train_df = pd.read_csv('.. /input/train.csv')
 test_images = os.listdir(test_dir)
 test_df = pd.DataFrame(
 list(zip(test_images, [0] * len(test_images))),
columns=['id', 'has_cactus'] 
)
eda(df_train, df_test )<categorify>
X = train_df.id
y = train_df.has_cactus
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)
 train_gen_df = pd.concat([X_train, y_train], axis=1 ).reset_index(drop=True)
 valid_gen_df = pd.concat([X_test, y_test], axis=1 ).reset_index(drop=True )
def clean_data(df):
df = df.replace({'Ex': 5, 'Gd': 4, 'TA': 3, 'Fa': 2, 'Po': 1})
 df = df.replace({'Grvl': 1, 'Pave': 2})
 df['Alley'].fillna(0, inplace = True)
 df['GarageYrBlt'].fillna(0, inplace = True)
 df['LotFrontage'] = df.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
df = df.replace({'LotShape':{'IR3': 1, 'IR2': 2, 'IR1': 3, 'Reg': 4}})
 df['LotShape'].fillna(df['LotShape'].value_counts().idxmax() , inplace=True)
 df['MSZoning'].fillna(df['MSZoning'].value_counts().idxmax() , inplace=True)
 df['Utilities'].fillna(df['Utilities'].value_counts().idxmax() , inplace=True)
 df = df.replace({'Utilities':{'ELO': 1, 'NoSeWa': 2, 'NoSewr': 3, 'AllPub': 4}})
 df['LandSlope'].fillna(df['LandSlope'].value_counts().idxmax() , inplace = True)
 df = df.replace({'LandSlope':{'Gtl': 1, 'Mod': 2, 'Sev': 3}})
 df['Exterior1st'].fillna(df['Exterior1st'].value_counts().idxmax() , inplace = True)
 df['Exterior2nd'].fillna(df['Exterior2nd'].value_counts().idxmax() , inplace = True)
 df['MasVnrType'].fillna(df['MasVnrType'].value_counts().idxmax() , inplace = True)
 df['MasVnrArea'].fillna(0, inplace = True)
 df['BsmtFinType1'].fillna('NoBsmt', inplace = True)
 df['BsmtFinType2'].fillna('NoBsmt', inplace = True)
 df['BsmtCond'].fillna(0, inplace = True)
 df['BsmtQual'].fillna(0, inplace = True)
 df['BsmtExposure'].fillna(0, inplace = True)
 df = df.replace({'BsmtExposure':{'Av': 3, 'Mn': 2, 'No': 1}})
 df['BsmtFinSF1'].fillna(0, inplace = True)
 df['BsmtFinSF2'].fillna(0, inplace = True)
 df['BsmtUnfSF'].fillna(0, inplace = True)
 df['TotalBsmtSF'].fillna(0, inplace = True)
 df = df.replace({'CentralAir':{'Y': 1, 'N': 0}})
 df['1stFlrSF'].fillna(0, inplace = True)
 df['2ndFlrSF'].fillna(0, inplace = True)
 df['LowQualFinSF'].fillna(0, inplace = True)
 df['BsmtFullBath'].fillna(0, inplace = True)
 df['BsmtHalfBath'].fillna(0, inplace = True)
 df['FullBath'].fillna(0, inplace = True)
 df['HalfBath'].fillna(0, inplace = True)
 df['KitchenQual'].fillna(df['KitchenQual'].value_counts().idxmax() ,inplace = True)
 df['Functional'].fillna('Typ',inplace = True)
 df = df.replace({'Functional':{'Typ': 7, 'Min1': 6, 'Maj1': 5, 'Min2': 4, 'Mod': 3, 'Maj2': 2, 'Sev': 1, 'Sal': 0}})
 df['FireplaceQu'].fillna(0, inplace = True)
 df['GarageType'].fillna('NoGarage', inplace = True)
 df['GarageFinish'].fillna(0, inplace = True)
 df = df.replace({'GarageFinish':{'Fin': 3, 'RFn': 2, 'Unf': 1}})
 df['GarageArea'].fillna(0, inplace = True)
 df['GarageCars'].fillna(0, inplace = True)
 df['GarageQual'].fillna(0, inplace = True)
 df['GarageCond'].fillna(0, inplace = True)
 df = df.replace({'PavedDrive':{'Y': 2, 'P': 1, 'N': 0}})
 df['Electrical'].fillna(df['Electrical'].value_counts().idxmax() , inplace = True)
 df['PoolQC'].fillna(0, inplace = True)
 df['Fence'].fillna('NoFence', inplace = True)
 df['MiscFeature'].fillna('NoMiF', inplace = True)
 df['SaleType'].fillna(df['SaleType'].value_counts().idxmax() , inplace = True)
 return df<data_type_conversions>
batch_size = 64
train_datagen = ImageDataGenerator(
 rescale=1.0 / 255.0,
horizontal_flip=True,
vertical_flip=True 
)
 train_gen = train_datagen.flow_from_dataframe(
 dataframe=train_gen_df,
directory=train_dir,
x_col='id',
y_col='has_cactus',
class_mode='other',
batch_size=batch_size,
target_size=(32, 32)
)
 valid_datagen = ImageDataGenerator(
 rescale=1.0 / 255.0 
)
 valid_gen = valid_datagen.flow_from_dataframe(
 dataframe=valid_gen_df,
directory=train_dir,
x_col='id',
y_col='has_cactus',
class_mode='other',
batch_size=batch_size,
target_size=(32, 32)
 )
def num_to_cat(df):
df['YrSold'] = df['YrSold'].astype(str)
 return df<drop_column>
input_shape =(32, 32, 3)
 model = Sequential()
model.add(Conv2D(8,(3,3), input_shape=input_shape))
model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(Conv2D(16,(3,3)))
 model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Conv2D(32,(3,3)))
 model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32,(3,3)))
 model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Flatten())
 model.add(Dense(1024))
model.add(Activation('relu'))
model.add(Dropout(0.4))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.4))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'] )
def remove_columns(df):
col = ['Id', 'Utilities', 'Street', 'PoolArea' ,'LowQualFinSF', 'Alley', 'EnclosedPorch', '3SsnPorch', 'PoolQC',
'KitchenAbvGr']
df.drop(columns = col, inplace = True)
 return df<feature_engineering>
callbacks = [
EarlyStopping(monitor='val_loss', patience=10),
ModelCheckpoint(filepath='model.h5', monitor='val_loss', save_best_only=True)
 ]
history = model.fit_generator(
 train_gen,
validation_data=valid_gen,
validation_steps=len(train_gen),
steps_per_epoch=len(train_gen_df)/ batch_size,
epochs=100,
verbose=True,
shuffle=True,
callbacks=callbacks
)
def add_features(df):
df['YrBltAndRemod'] = df['YearBuilt'] + df['YearRemodAdd']
df['TotalSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df['Total_sqr_footage'] =(df['BsmtFinSF1'] + df['BsmtFinSF2'] + df['1stFlrSF'] + df['2ndFlrSF'])
 df['Total_Bathrooms'] =(df['FullBath'] +(0.5 * df['HalfBath'])+ df['BsmtFullBath'] +(0.5 * df['BsmtHalfBath']))
df['Total_porch_sf'] =(df['OpenPorchSF'] + df['3SsnPorch'] + df['EnclosedPorch'] + df['ScreenPorch'] + df['WoodDeckSF'])
 df['haspool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df['has2ndfloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasgarage'] = df['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df['hasbsmt'] = df['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasfireplace'] = df['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 return df<filter>
model.load_weights('model.h5')
 test_datagen = ImageDataGenerator(
 rescale=1.0 / 255.0 
)
 test_gen = valid_datagen.flow_from_dataframe(
 dataframe=test_df,
directory=test_dir,
x_col='id',
class_mode=None,
batch_size=1,
target_size=(32, 32),
shuffle=False
)
def find_outliers(df):
outliers = df[df['GrLivArea'] > 4000].index
return outliers<feature_engineering>
predictions = model.predict_generator(test_gen, steps=len(test_gen), verbose=True)
<feature_engineering><EOS> 
test_df['has_cactus'] = predictions
test_df.to_csv('submission.csv', index=False)
 print('Done!' )