Kaggle competition: Costa Rican Household Poverty Level Prediction
def get_test_images(directory):
Images = []
Image_names = []
for image_file in os.listdir(directory):
Image_names.append(image_file)
 image = cv2.imread(directory+r'/'+image_file)
 if image is not None:
image = cv2.resize(image,(300,300),)
 Images.append(image)
 return Images, Image_names<choose_model_class>
pd.options.display.max_rows = 999
pd.options.display.max_columns = 999
pd.set_option('max_colwidth',1000)
 pd.options.display.max_info_columns = 999
pd.set_option('expand_frame_repr',True)
vgg = VGG16(include_top=
False, weights='imagenet', input_shape =(300,300,3))
vgg.trainable=False
for layer in vgg.layers:
layer.trainable = False
fc1 = Concatenate(axis=-1 )([GlobalAvgPool2D()(vgg.output), GlobalMaxPooling2D()(vgg.output)])
 fc1 = Dense(400, activation='relu' )(fc1)
 fc1_dropout = Dropout(0.3 )(fc1)
 fc2 = Dense(200, activation='relu' )(fc1_dropout)
 fc2_dropout = Dropout(0.3 )(fc2)
 fc2 = Dense(75, activation='relu' )(fc1_dropout)
 output = Dense(12, activation='softmax' )(fc2_dropout)
 model = Model(vgg.input, output )<load_pretrained>
train = pd.read_csv('.. /input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('.. /input/costa-rican-household-poverty-prediction/test.csv')
 meta = pd.read_csv('.. /input/metadata/Metadata.csv',index_col='variable', header=0)
 print(f'Train data:{train.shape}, Test data: {test.shape}, Metadata: {meta.shape}' )
model.load_weights('saved_model.hdf5' )<save_model>
print(f'Train duplicates: {len(train[train.duplicated() ])}
Test Duplicates: {len(test[test.duplicated() ])}' )
tensorflow.keras.models.save_model(
 model,
'tf_model.hdf5',
overwrite=True,
include_optimizer=True
)<load_pretrained>
test.loc[test['rez_esc'] == 99.0 , 'rez_esc'] = 5
model = load_model('tf_model.hdf5' )<predict_on_test>
print(f'Train:
{train.isnull().sum() [train.isnull().sum() >0]}
')
 print(f'Test:
{test.isnull().sum() [test.isnull().sum() >0]}
')
 pd.testing.assert_series_equal(train.isnull().any() [train.isnull().any() ==True],test.isnull().any() [test.isnull().any() ==True])
 null_list = list(train.isnull().any() [train.isnull().any() ==True].index)
 print(f'Columns with Null values: {null_list}
')
 print('Definitions:')
 for item in null_list:
print(f'{item}
{meta.loc[item]}
' )
predictions = model.predict(test_images)
 predictions = np.argmax(predictions, axis = 1 )<define_variables>
train_n = train
test_n = test
name = [item + '_null' for item in null_list[:3]]
i = 0
for item in null_list[:3]:
train_n[item].fillna(0,inplace=True)
 test_n[item].fillna(0,inplace=True)
 i+=1
pd.concat([train_n.head() , test_n.head() ], sort=False )
labelled_predictions = []
mapping = {0: 'Sugar beet',1:'Fat Hen' ,2: 'Scentless Mayweed',3: 'Charlock',
4:'Small-flowered Cranesbill', 5:'Maize' ,
6: 'Shepherds Purse' ,7:'Common wheat' ,8:'Common Chickweed' ,
9:'Cleavers' ,10:'Loose Silky-bent' ,11: 'Black-grass'}
for pred in predictions:
labelled_predictions.append(mapping[pred] )<save_to_csv>
def shift_cols_last(df, col_index):

 cols = df.columns.tolist()
i=0
for item in col_index:
if i==0:
new_cols = cols[:item] + cols[item+1:] + [cols[item]]
elif i>=1:
new_cols = new_cols[:item] + new_cols[item+1:] + [cols[item]]
i+=1
return df[new_cols]
d = []
i=0
for pred in labelled_predictions:
d.append({'file': image_names[i], 'species': pred})
 i=i+1
output = pd.DataFrame(d)
 output.to_csv('submission.csv',index=False )<import_modules>
pd.concat([train_n.head() , test_n.head() ], sort=False )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from glob import glob
import cv2
import keras.backend as k
import tensorflow as tf
import os
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from keras.utils import np_utils<prepare_x_and_y>
train_n = train_n.astype('float64', errors= 'ignore')
 test_n = test_n.astype('float64', errors= 'ignore' )
images_path = '.. /input/train/*/*.png'
images = glob(images_path)
 train_images = []
train_labels = []
for img in images:
train_images.append(cv2.resize(cv2.imread(img),(70, 70)))
 train_labels.append(img.split('/')[-2])
 train_X = np.asarray(train_images)
 train_Y = pd.DataFrame(train_labels )<categorify>
print('Train')
 print(train_n.info() ,'
')
 print('Test')
 print(test_n.info() )
encoder = LabelEncoder()
encoder.fit(train_Y[0])
 encoded_labels = encoder.transform(train_Y[0])
 categorical_labels = np_utils.to_categorical(encoded_labels )<split>
sqb_null_trls = train_n[train_n['SQBmeaned'].isnull() ].index.tolist()
sqb_null_tels = test_n[test_n['SQBmeaned'].isnull() ].index.tolist()
print(f'Train SQBmeaned Null rows
{sqb_null_trls}
Test SQBmeaned Null rows
{sqb_null_tels}' )
x_train,x_test,y_train,y_test=train_test_split(train_X,categorical_labels,test_size=0.25,random_state=7)
<import_modules>
for item in sqb_null_trls:
train_n.loc[item,'SQBmeaned']=train_n.loc[item,'meaneduc']**2
for item in sqb_null_tels:
test_n.loc[item,'SQBmeaned']=test_n.loc[item,'meaneduc']**2
pd.concat([train_n.loc[sqb_null_trls, ['Id','meaneduc', 'SQBmeaned', 'Target']],
test_n.loc[sqb_null_tels, ['Id','meaneduc', 'SQBmeaned']]], sort=False )
import keras
from keras import layers
from keras.layers import Input, Dense, Activation,ZeroPadding2D, BatchNormalization, Flatten, Conv2D
from keras.layers import AveragePooling2D, MaxPooling2D, Dropout, GlobalMaxPooling2D, GlobalAveragePooling2D
from keras.models import Sequential
from keras.preprocessing.image import ImageDataGenerator
from keras.applications.vgg16 import VGG16<choose_model_class>
print(f'Train data remaining NAs: {len(train_n[train_n.isnull().any(axis=1)])}')
 print(f'Test data remaining NAs: {len(test_n[test_n.isnull().any(axis=1)])}' )
base_model = VGG16(include_top=False, weights='imagenet', input_shape=(70, 70, 3))<choose_model_class>
pared = []
piso = []
techo = []
abasta = []
sanitario = []
energcocinar = []
elimbasu = []
epared = []
etecho = []
eviv = []
gender = []
estadocivil = []
parentesco = []
instlevel = []
tipovivi = []
lugar = []
area = []
elec = []
for item in train_n.columns:
if item.startswith('pared'):
pared.append(item)
 elif item.startswith('piso'):
piso.append(item)
 elif item.startswith('techo'):
techo.append(item)
 elif item.startswith('abasta'):
abasta.append(item)
 elif item.startswith('sanitario'):
sanitario.append(item)
 elif item.startswith('energcocinar'):
energcocinar.append(item)
 elif item.startswith('elimbasu'):
elimbasu.append(item)
 elif item.startswith('epared'):
epared.append(item)
 elif item.startswith('etecho'):
etecho.append(item)
 elif item.startswith('eviv'):
eviv.append(item)
 elif item== 'male' or item =='female':
gender.append(item)
 elif item.startswith('estadocivil'):
estadocivil.append(item)
 elif item.startswith('parentesco'):
parentesco.append(item)
 elif item.startswith('instlevel'):
instlevel.append(item)
 elif item.startswith('tipovivi'):
tipovivi.append(item)
 elif item.startswith('lugar'):
lugar.append(item)
 elif item.startswith('area'):
area.append(item)
 elif item == 'noelec'or item =='coopele' or item == 'planpri' or item =='public':
elec.append(item )
model = Sequential()
model.add(base_model)
 model.add(layers.Flatten())
 model.add(layers.Dense(256, activation='relu'))
model.add(layers.Dense(12, activation='sigmoid'))<choose_model_class>
lbl_ls = [pared, piso, techo, abasta, sanitario, energcocinar, elimbasu, epared, etecho, eviv, gender, estadocivil, parentesco,
instlevel, tipovivi, lugar, area, elec]
lbl_strls = ['pared', 'piso', 'techo', 'abasta', 'sanitario', 'energcocinar', 'elimbasu', 'epared', 'etecho', 'eviv', 'gender', 'estadocivil',
'parentesco', 'instlevel', 'tipovivi', 'lugar', 'area', 'elec']
print(len(lbl_ls), len(lbl_strls))
opt = keras.optimizers.adam(lr=0.0001, decay=1e-6)
 model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'] )<define_variables>
i = 0
for item in lbl_ls:
trdf = train_n[item]
cols = trdf.columns.to_series().values
if i==0:
lbl_trdf = pd.DataFrame(np.repeat(cols[None, :], len(trdf), 0)[trdf.astype(bool ).values], trdf.index[trdf.any(1)], columns = [lbl_strls[i]])
 else:
lbl_trdf = pd.concat([lbl_trdf, pd.DataFrame(np.repeat(cols[None, :], len(trdf), 0)[trdf.astype(bool ).values],
trdf.index[trdf.any(1)], columns = [lbl_strls[i]])], axis = 1)
 i+=1
j = 0
for item in lbl_ls:
tedf = test_n[item]
cols = tedf.columns.to_series().values
if j==0:
lbl_tedf = pd.DataFrame(np.repeat(cols[None, :], len(tedf), 0)[tedf.astype(bool ).values],
tedf.index[tedf.any(1)], columns = [lbl_strls[j]])
 else:
lbl_tedf = pd.concat([lbl_tedf, pd.DataFrame(np.repeat(cols[None, :], len(tedf), 0)[tedf.astype(bool ).values],
tedf.index[tedf.any(1)], columns = [lbl_strls[j]])], axis = 1)
 j+=1
print(lbl_trdf.shape, lbl_tedf.shape)
 pd.concat([lbl_trdf.head() , lbl_tedf.head() ], sort=False )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
rotation_range=0,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
vertical_flip=False )<train_model>
for i in range(0, len(lbl_strls)) :
if i%10==0:
print(i)
 tr_ls = lbl_trdf.iloc[:,i].tolist()
lbl_trdf.iloc[:,i] = LabelEncoder().fit_transform(tr_ls)
 te_ls = lbl_tedf.iloc[:,i].tolist()
lbl_tedf.iloc[:,i] = LabelEncoder().fit_transform(te_ls )
datagen.fit(x_train )<train_model>
pd.concat([lbl_trdf.head() , lbl_tedf.head() ], sort=False )
model.fit_generator(datagen.flow(x_train, y_train,
batch_size=50),
steps_per_epoch=x_train.shape[0],
epochs=1,
validation_data=(x_test, y_test),
verbose=1)
<compute_test_metric>
train_n = pd.concat([train_n, lbl_trdf], axis = 1)
 test_n = pd.concat([test_n, lbl_tedf], axis = 1 )
[loss, accuracy] = model.evaluate(x_test, y_test )<compute_test_metric>
for item in lbl_ls:
train_n = train_n.drop(labels=item, axis=1)
 test_n = test_n.drop(labels=item, axis=1 )
print('Test Set Accuracy: '+str(accuracy*100)+"%");<define_variables>
pd.concat([train_n.head() , test_n.head() ], sort = False )
test_images_path = '.. /input/test/*.png'
test_images = glob(test_images_path)
 test_images_arr = []
test_files = []
for img in test_images:
test_images_arr.append(cv2.resize(cv2.imread(img),(70, 70)))
 test_files.append(img.split('/')[-1])
 test_X = np.asarray(test_images_arr )<predict_on_test>
train_n = shift_cols_last(train_n, col_index)
 train_n.head()
predictions = model.predict(test_X )<categorify>
def shift_cols_front(df, col_index):

 cols = df.columns.tolist()
i=0
for item in col_index:
if i==0:
new_cols = [cols[0]] + [cols[item]] + cols[1:item] + cols[item+1:]
elif item < col_index[i-1]:
new_cols = new_cols[0:i+1] + [new_cols[item+1]] + new_cols[1+i:item+1] + new_cols[item+2:]
elif item > col_index[i-1]:
new_cols = new_cols[0:i+1] + [new_cols[item]] + new_cols[1+i:item] + new_cols[item+1:]
i+=1
return df[new_cols]
preds = np.argmax(predictions, axis=1)
 pred_str = encoder.classes_[preds]<save_to_csv>
train_n = shift_cols_front(train_n, col_index)
 test_n = shift_cols_front(test_n, col_index )
final_predictions = {'file':test_files, 'species':pred_str}
final_predictions = pd.DataFrame(final_predictions)
 final_predictions.to_csv("submission.csv", index=False )<install_modules>
tr_corr = df.corr()
sorted_cor = tr_corr['Target'][tr_corr['Target'].isna() ==False].sort_values(ascending=False)
 print(f'Most Positive Correlations:
{sorted_cor.head(20)}
Most Negative Correlations:
{sorted_cor.tail(20)}' )
!pip install pytorchcv<install_modules>
train_class1_3 = train_n[train_n.Target!=4]
train_n.drop(index = train_class1_3.index, inplace=True)
 train_class1_3 = pd.concat([train_class1_3, train_class1_3, train_class1_3], sort=False, ignore_index = True)
 train_n = pd.concat([train_n, train_class1_3], sort=False, ignore_index = True)
 print(len(train_n))
!pip install fastai==1.0.47<import_modules>
X_arr = train_n.iloc[:,2:-1].values
y = train_n.iloc[:,-1].values
te_X = test_n.iloc[:,2:].values
from pathlib import Path
import fastai
from fastai.vision import *
from pytorchcv.model_provider import get_model as ptcv_get_model<define_variables>
scaler = MinMaxScaler(feature_range=(0, 1))
X_arr = scaler.fit_transform(X_arr)
 te_X = scaler.fit_transform(te_X )
PATH = Path('.. /input/train/')
 np.random.seed(42 )<define_variables>
input_dim = X_arr.shape[1]
print(f'Train features
{input_dim}' )
batch_size = 128<feature_engineering>
assert not np.any(np.isnan(X_arr))
tfms = get_transforms(flip_vert=True,max_lighting=0.5,max_rotate=360.,max_warp=0.3 )<categorify>
model3 = LogisticRegression(penalty='l2', solver = 'newton-cg', multi_class='multinomial', verbose = 1)
 scores3 = cross_validate(model3, X_arr, y, cv=10, n_jobs = 4, return_train_score = True, verbose=1, scoring = 'f1_macro')
 model3.fit(X_arr,y )
data =(src.transform(tfms,size=224)
.databunch(bs=batch_size)
.normalize(imagenet_stats))<choose_model_class>
scores3
def md(f=None):
mdl = ptcv_get_model('condensenet74_c4_g4', pretrained=True)
 mdl.features.final_pool = nn.AvgPool2d(kernel_size=7, stride=1, padding=3)
 return mdl<choose_model_class>
pred3 = model3.predict(te_X )
learn = cnn_learner(data, md, metrics=accuracy, path='./' )<train_model>
pred3s = pd.DataFrame(pred3, columns = ['Target'])
 pred3s = pred3s.assign(Id=test_n.Id)
 pred3s = pred3s[['Id','Target']]
pred3s.Target = pd.to_numeric(pred3s.Target, downcast='signed')
 pred3s.to_csv('log_reg.csv', header = True, index=False)
 pred3s.head()
learn.fit_one_cycle(5,slice(1e-2))<train_model>
pd.DataFrame.from_dict(model3_dict, orient = 'Index', columns=['Value'] )
learn.fit_one_cycle(5,slice(1e-6))<define_variables>
pd.DataFrame.from_dict(classification_report(y, model3.predict(X_arr), output_dict=True), orient='index' )
test = ImageList.from_folder('.. /input/test/' )<predict_on_test>
model7 = BaggingClassifier()
scores7 = cross_validate(model7, X_arr, y, cv=10, verbose=1, return_train_score = True, n_jobs=4, scoring = 'f1_macro')
 model7.fit(X_arr,y )
learn = load_learner(path='./', test=test)
 preds, _ = learn.get_preds(ds_type=DatasetType.Test )<define_variables>
pred7 = model7.predict(te_X )
labelled_preds = [learn.data.classes[np.argmax(pred)] for pred in preds]<define_variables>
pred7s = pd.DataFrame(pred7, columns = ['Target'])
 pred7s = pred7s.assign(Id=test_n.Id)
 pred7s = pred7s[['Id','Target']]
pred7s.Target = pd.to_numeric(pred7s.Target, downcast='signed')
 pred7s.to_csv('bagging.csv', header = True, index=False)
 pred7s.head()
fnames = [f.name for f in learn.data.test_ds.items]<create_dataframe>
model10 = GradientBoostingClassifier()
scores10 = cross_validate(model10, X_arr, y, cv=10, verbose=1, return_train_score = True, n_jobs=4, scoring = 'f1_macro')
 model10.fit(X_arr,y )
df = pd.DataFrame({'file':fnames, 'species':labelled_preds}, columns=['file', 'species'] )<save_to_csv>
pred10 = model10.predict(te_X )
df.to_csv('submission.csv', index=False )<import_modules>
pred10s = pd.DataFrame(pred10, columns = ['Target'])
 pred10s = pred10s.assign(Id=test_n.Id)
 pred10s = pred10s[['Id','Target']]
pred10s.Target = pd.to_numeric(pred10s.Target, downcast='signed')
 pred10s.to_csv('grad_boosting.csv', header = True, index=False)
 pred10s.head()
from fastai import *
from fastai.vision import *
import matplotlib.pyplot as plt<define_variables>
final_pred = np.array([])
 for i in range(0,len(te_X)) :
final_pred = np.append(final_pred, mode([pred3[i], pred7[i], pred10[i]])[0] )
<load_pretrained><EOS> 
pred = pd.DataFrame(final_pred, columns = ['Target'])
 pred = pred.assign(Id=test_n.Id)
 pred = pred[['Id','Target']]
pred.Target = pd.to_numeric(pred.Target, downcast='signed')
 pred.to_csv('prediction.csv', header = True, index=False)
 pred.head()