Kaggle competition: Aesthetic Visual Analysis
train['Embarked'] = train['Embarked'].fillna(train['Embarked'].mode().values[0] )<count_missing_values>
import pandas as pd
import numpy as np
import cv2
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sys import getsizeof
train.isnull().sum()<count_missing_values>
train = pd.read_csv('.. /input/ava/train.csv', index_col=0)
 test = pd.read_csv('.. /input/ava/test.csv', index_col=0)
 train
train.isnull().sum()<define_variables>
X_train = train['image'].values
y_train = train['label'].values
X_test = test['image'].values
num_cols = ['Age','Fare']
cat_cols = ['Pclass','Sex','Embarked','Cabin','had_Cabin','Salutation','Family_Size']<categorify>
shapes = []
for i in range(len(X_train)) :
path = '.. /input/ava/dataset/dataset/'+str(X_train[i])+'.jpg'
img = cv2.imread(path)
 shapes.append(img.shape)
 shapes = np.array(shapes[:])
 print(np.mean(shapes[:,0]), np.mean(shapes[:,1]), np.mean(shapes[:,2]))
train_data = pd.get_dummies(train,columns=cat_cols,drop_first=True)
 train_data.head(2 )<split>
def get_feature(img):
img = cv2.resize(img,(32, 32), interpolation = cv2.INTER_AREA)
 return np.array(img ).flatten()
explore_data, validation_data = train_test_split(train_data, test_size = 0.2, random_state=RANDOM_SEED, stratify=train[target] )<split>
def create_features_data(image_names):
FEATURES = []
for i in range(len(image_names)) :
path = '.. /input/ava/dataset/dataset/'+str(image_names[i])+'.jpg'
img = cv2.imread(path)
 FEATURES.append(get_feature(img))
return np.array(FEATURES)
 features_train = create_features_data(X_train)
 features_test = create_features_data(X_test )
train_data, test_data = train_test_split(explore_data, test_size = 0.2, random_state=RANDOM_SEED )<feature_engineering>
%%time
lr = LogisticRegression()
lr.fit(features_train,y_train )
def handle_outliers_per_target_class(df,var,target,tol):
gdf = df[df[target] == 1]
var_data = gdf[var].values
q25, q75 = np.percentile(var_data, 25), np.percentile(var_data, 75)
 print('Outliers handling for {}'.format(var))
print('Quartile 25: {} | Quartile 75: {}'.format(q25, q75))
iqr = q75 - q25
print('IQR {}'.format(iqr))
cut_off = iqr * tol
lower, upper = q25 - cut_off, q75 + cut_off
print('Cut Off: {}'.format(cut_off))
print('{} Lower: {}'.format(var,lower))
print('{} Upper: {}'.format(var,upper))
outliers = [x for x in var_data if x < lower or x > upper]
print('Number of Outliers in feature {} in {}: {}'.format(var,key,len(outliers)))
 print('{} outliers:{}'.format(var,outliers))
print('----' * 25)
 print('
')
 print('
')
 return list(df[(df[var] > upper)|(df[var] < lower)].index )<merge>
prediction = pd.DataFrame()
prediction['labels'] = lr.predict(features_test )
outliers_wrt_target = []
for num_col in num_cols:
outliers_wrt_target.extend(handle_outliers_per_target_class(train_data,num_col,target,1.5))
outliers_wrt_target = list(set(outliers_wrt_target))
train_data = train_data.drop(outliers_wrt_target )<feature_engineering>
prediction.to_csv("submittion.csv", index_label='id' )