Kaggle competition: Plant Seedlings Classification
<SOS> metric: MeanFScore Kaggle data source: plant-seedlings-classification<feature_engineering>
import numpy as np
import pandas as pd
from PIL import Image
import os
from random import shuffle
import matplotlib.pyplot as plt
import random
def impute_BsmtQual_train(df,column):
df.loc[(df[column].isnull())&(df["SalePrice"] < 120000.00),column] = "Fa"
df.loc[(df[column].isnull())&(df["SalePrice"] >= 120000.00)&(df["SalePrice"] < 180000.00),column] = "TA"
df.loc[(df[column].isnull())&(df["SalePrice"] >= 180000.00),column] = "TA"
return "Basement Quality's missing value is imputed"<train_model>
CLASS = ['Black-grass', 'Charlock', 'Cleavers', 'Common Chickweed', 'Common wheat', 'Fat Hen', 'Loose Silky-bent',
'Maize', 'Scentless Mayweed', 'Shepherds Purse', 'Small-flowered Cranesbill', 'Sugar beet']
impute_BsmtQual_train(training_set,"BsmtQual" )<count_values>
train_dir = '.. /input/train'
test_dir = '.. /input/test'
train_Blackgrass = ['.. /input/train/Black-grass/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Black-grass')) ]
train_Charlock = ['.. /input/train/Charlock/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Charlock')) ]
train_Cleavers = ['.. /input/train/Cleavers/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Cleavers')) ]
train_CommonChickweed = ['.. /input/train/Common Chickweed/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Common Chickweed')) ]
train_Commonwheat = ['.. /input/train/Common wheat/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Common wheat')) ]
train_FatHen = ['.. /input/train/Fat Hen/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Fat Hen')) ]
train_LooseSilkybent = ['.. /input/train/Loose Silky-bent/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Loose Silky-bent')) ]
train_Maize = ['.. /input/train/Maize/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Maize')) ]
train_ScentlessMayweed = ['.. /input/train/Scentless Mayweed/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Scentless Mayweed')) ]
train_ShepherdsPurse = ['.. /input/train/Shepherds Purse/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Shepherds Purse')) ]
train_SmallfloweredCranesbill = ['.. /input/train/Small-flowered Cranesbill/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Small-flowered Cranesbill')) ]
train_Sugarbeet = ['.. /input/train/Sugar beet/{}'.format(i)for i in os.listdir(os.path.join(train_dir, 'Sugar beet')) ]
test_imgs = ['.. /input/test/{}'.format(i)for i in os.listdir(test_dir)]
train_imgs = train_Blackgrass + train_Charlock + train_Cleavers + train_CommonChickweed + train_Commonwheat + train_FatHen + train_LooseSilkybent + \
train_Maize + train_ScentlessMayweed + train_ShepherdsPurse + train_SmallfloweredCranesbill+ train_Sugarbeet
random.shuffle(train_imgs)
 test_imgs
training_set[columns_impute_train[3]].value_counts()<groupby>
nrows = 150
ncolumns = 150
channels = 3
def read_and_process_image(list_of_images):

 X = []
y = []
for image in list_of_images:
X.append(cv2.resize(cv2.imread(image, cv2.IMREAD_COLOR),(nrows,ncolumns), interpolation=cv2.INTER_CUBIC))
if image.split('/')[3] == 'Black-grass':
y.append(0)
 elif image.split('/')[3] == 'Charlock':
y.append(1)
 elif image.split('/')[3] == 'Cleavers':
y.append(2)
 elif image.split('/')[3] == 'Common Chickweed':
y.append(3)
 elif image.split('/')[3] == 'Common wheat':
y.append(4)
 elif image.split('/')[3] == 'Fat Hen':
y.append(5)
 elif image.split('/')[3] == 'Loose Silky-bent':
y.append(6)
 elif image.split('/')[3] == 'Maize':
y.append(7)
 elif image.split('/')[3] == 'Scentless Mayweed':
y.append(8)
 elif image.split('/')[3] == 'Shepherds Purse':
y.append(9)
 elif image.split('/')[3] == 'Small-flowered Cranesbill':
y.append(10)
 elif image.split('/')[3] == 'Sugar beet':
y.append(11)
 return X, y
print(training_set.groupby(columns_impute_train[3])["SalePrice"].mean() )<feature_engineering>
X, y = read_and_process_image(train_imgs )
training_set.loc[(training_set[columns_impute_train[3]].isnull())&(training_set["SalePrice"] < 75000.00),"BsmtCond"] = "Po"
training_set.loc[(training_set[columns_impute_train[3]].isnull())&(training_set["SalePrice"] >= 75000.00)&(training_set["SalePrice"] < 140000.00),"BsmtCond"] = "Fa"
training_set.loc[(training_set[columns_impute_train[3]].isnull())&(training_set["SalePrice"] >= 140000.00)&(training_set["SalePrice"] < 190000.00),"BsmtCond"] = "TA"
training_set.loc[(training_set[columns_impute_train[3]].isnull())&(training_set["SalePrice"] >= 190000.00),"BsmtCond"] = "Gd"<count_values>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.20, random_state=2)
 print("Shape of train images is:", X_train.shape)
 print("Shape of validation images is:", X_val.shape)
 print("Shape of labels is:", y_train.shape)
 print("Shape of labels is:", y_val.shape)
 y_train = to_categorical(y_train, num_classes=12)
 y_val = to_categorical(y_val, num_classes=12)
 print(y_train)
 print(y_val.shape )
training_set[columns_impute_train[4]].value_counts()<groupby>
ntrain = len(X_train)
 nval = len(X_val)
 batch_size = 32
print(training_set.groupby(columns_impute_train[4])["SalePrice"].mean() )<feature_engineering>
model = Sequential()
model.add(Conv2D(32, kernel_size =(3, 3), activation='relu', input_shape=(150, 150, 3)))
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(96, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(96, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(128, activation='relu'))
model.add(Dense(12, activation = 'softmax'))
training_set.loc[(training_set[columns_impute_train[4]].isnull())&(training_set["SalePrice"] < 150000.00),"BsmtExposure"] = "No"
training_set.loc[(training_set[columns_impute_train[4]].isnull())&(training_set["SalePrice"] >= 150000.00)&(training_set["SalePrice"] < 200000.00),"BsmtExposure"] = "Mn"
training_set.loc[(training_set[columns_impute_train[4]].isnull())&(training_set["SalePrice"] >= 200000.00)&(training_set["SalePrice"] < 230000.00),"BsmtExposure"] = "Av"
training_set.loc[(training_set[columns_impute_train[4]].isnull())&(training_set["SalePrice"] >= 210000.00),"BsmtExposure"] = "Gd"<count_values>
model.compile(loss='binary_crossentropy', optimizer=optimizers.RMSprop(lr=1e-4), metrics=['acc'] )
training_set[columns_impute_train[5]].value_counts()<groupby>
train_datagen = ImageDataGenerator(rescale=1./255,
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,)
 val_datagen = ImageDataGenerator(rescale=1./255 )
print(training_set.groupby(columns_impute_train[5])["SalePrice"].mean() )<feature_engineering>
train_generator = train_datagen.flow(X_train, y_train, batch_size=batch_size)
 val_generator = val_datagen.flow(X_val, y_val, batch_size=batch_size)
 print(ntrain)
 print(nval )
training_set.loc[(training_set[columns_impute_train[5]].isnull())&(training_set["BsmtQual"] == "Fa"),"BsmtFinType1"] = "Unf"
training_set.loc[(training_set[columns_impute_train[5]].isnull())&(training_set["BsmtQual"] == "TA"),"BsmtFinType1"] = "LwQ"
training_set.loc[(training_set[columns_impute_train[5]].isnull())&(training_set["BsmtQual"] == "Gd"),"BsmtFinType1"] = "GLQ"<count_values>
history = model.fit_generator(train_generator,
steps_per_epoch=ntrain // batch_size,
epochs=1024,
validation_data=val_generator,
validation_steps=nval // batch_size )
training_set[columns_impute_train[6]].value_counts()<groupby>
model.save_weights('model_wieghts.h5')
 model.save('model_keras.h5' )
print(training_set.groupby(columns_impute_train[6])["SalePrice"].mean() )<count_values>
X_test, y_test = read_and_process_image(test_imgs[10:20])
 x = np.array(X_test)
 test_datagen = ImageDataGenerator(rescale=1./255 )
training_set[training_set["BsmtQual"]=="Ex"]["BsmtFinType2"].value_counts()<feature_engineering>
i = 0
text_labels = []
plt.figure(figsize=(30,20))
for batch in test_datagen.flow(x, batch_size=1):
pred = model.predict(batch)
 result = np.argmax(pred)
 if result == 0:
text_labels.append('Black-grass')
 elif result == 1:
text_labels.append('Charlock')
 elif result == 2:
text_labels.append('Cleavers')
 elif result == 3:
text_labels.append('Common Chickweed')
 elif result == 4:
text_labels.append('Common wheat')
 elif result == 5:
text_labels.append('Fat Hen')
 elif result == 6:
text_labels.append('Loose Silky-bent')
 elif result == 7:
text_labels.append('Maize')
 elif result == 8:
text_labels.append('Scentless Mayweed')
 elif result == 9:
text_labels.append('Shepherds Purse')
 elif result == 10:
text_labels.append('Small-flowered Cranesbill')
 elif result == 11:
text_labels.append('Sugar beet')
 plt.subplot(5 / columns + 1, columns, i + 1)
 plt.title('This is a ' + text_labels[i])
 imgplot = plt.imshow(batch[0])
 i += 1
if i % 10 == 0:
break
plt.show()
training_set.loc[training_set[columns_impute_train[6]].isnull() ,"BsmtFinType2"] = "Unf"<count_values>
def read_and_process_test_image(list_of_images):

 X = []
y = []
for image in list_of_images:
X.append(cv2.resize(cv2.imread(image, cv2.IMREAD_COLOR),(nrows,ncolumns), interpolation=cv2.INTER_CUBIC))
y.append(image.split('/')[3])
 return X, y
X_test, y_test = read_and_process_test_image(test_imgs)
 x = np.array(X_test)
 test_datagen = ImageDataGenerator(rescale=1./255 )
training_set[columns_impute_train[7]].value_counts()<data_type_conversions>
n = 0
dic = {
'file': [],
'species': [],
}
for batch in test_datagen.flow(x, batch_size=1, shuffle=False):
pred = model.predict(batch)
 result = np.argmax(pred)
 print(result)
 print(np.array(y_test)[n])
 if result == 0:
dic['species'].append('Black-grass')
 dic['file'].append(np.array(y_test)[n])
 elif result == 1:
dic['species'].append('Charlock')
 dic['file'].append(np.array(y_test)[n])
 elif result == 2:
dic['species'].append('Cleavers')
 dic['file'].append(np.array(y_test)[n])
 elif result == 3:
dic['species'].append('Common Chickweed')
 dic['file'].append(np.array(y_test)[n])
 elif result == 4:
dic['species'].append('Common wheat')
 dic['file'].append(np.array(y_test)[n])
 elif result == 5:
dic['species'].append('Fat Hen')
 dic['file'].append(np.array(y_test)[n])
 elif result == 6:
dic['species'].append('Loose Silky-bent')
 dic['file'].append(np.array(y_test)[n])
 elif result == 7:
dic['species'].append('Maize')
 dic['file'].append(np.array(y_test)[n])
 elif result == 8:
dic['species'].append('Scentless Mayweed')
 dic['file'].append(np.array(y_test)[n])
 elif result == 9:
dic['species'].append('Shepherds Purse')
 dic['file'].append(np.array(y_test)[n])
 elif result == 10:
dic['species'].append('Small-flowered Cranesbill')
 dic['file'].append(np.array(y_test)[n])
 elif result == 11:
dic['species'].append('Sugar beet')
 dic['file'].append(np.array(y_test)[n])
 n += 1
if n == len(x):
break
submission = pd.DataFrame(dic)
 submission.to_csv("submission.csv", index=False, header=True )