Kaggle competition: Costa Rican Household Poverty Level Prediction
train_features = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train_nonscored = pd.read_csv("/kaggle/input/lish-moa/train_targets_nonscored.csv")
 train_scored = pd.read_csv("/kaggle/input/lish-moa/train_targets_scored.csv")
 test_features = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 submission = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv" )<count_values>
import os,sys
import numpy as np
import pandas as pd
import seaborn as sb
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
from sklearn.utils.class_weight import compute_sample_weight
from sklearn import metrics
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score, GridSearchCV
import xgboost as xgb
from xgboost.sklearn import XGBClassifier
train_features['cp_type'].value_counts()<feature_engineering>
train_data = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 train_target = train_data['Target']
train_data.drop(['Target'],axis=1,inplace=True)
remove_vehicle = True
if remove_vehicle:
kept_index = train_features['cp_type']=='trt_cp'
train = train_features.loc[kept_index].reset_index(drop=True)
 train_targets_scored = train_scored.loc[kept_index].reset_index(drop=True)
 train_features["cp_type"] =(train_features["cp_type"]=="trt_cp")+ 0
train_features["cp_dose"] =(train_features["cp_dose"]=="D1")+ 0
test_features["cp_type"] =(test_features["cp_type"]=="trt_cp")+ 0
test_features["cp_dose"] =(test_features["cp_dose"]=="D1")+ 0
X_test = test_features.values<filter>
def findNanCol(orig_df,print_flag=False):
df_missing = orig_df.isnull()
col_list_missing_data = []
for col in df_missing.columns.values.tolist() :
try:
x=df_missing[col].value_counts() [1]
if print_flag:
print("Column {}: {} missing value counts".format(col,x))
col_list_missing_data.append(col)
 except:
continue
return col_list_missing_data
def findObjectTypeCol(orig_df,print_flag=False):
obj_col_list = []
for col in orig_df.columns.values.tolist() :
if orig_df[col].dtypes == "object":
if print_flag:
print(col)
 obj_col_list.append(col)
 return obj_col_list
def fillMissingValues(train_data):
train_data['v2a1'] = train_data['v2a1'].fillna(0 ).astype('float64')
 train_data['v18q1'] = train_data['v18q1'].fillna(0 ).astype('float64')
 train_data['rez_esc'] = train_data['rez_esc'].fillna(0 ).astype('float64')
 train_data['meaneduc'] = train_data['meaneduc'].fillna(0 ).astype('float64')
 train_data['SQBmeaned'] = train_data['SQBmeaned'].fillna(0 ).astype('float64')
 return train_data
def getDependencyRate(train_data):
for i,val in enumerate(train_data['dependency']):
if val == 'no':
train_data.loc[i,'dependency'] = 0
elif val == 'yes':
total_num = float(train_data.loc[i,'hogar_total'])
 num_abv_65_and_below_19 = float(train_data.loc[i,'hogar_nin'])+ float(train_data.loc[i,'hogar_mayor'])
 train_data.loc[i,'dependency'] = num_abv_65_and_below_19/(total_num - num_abv_65_and_below_19)
 train_data['dependency'] = train_data['dependency'].astype('float64')
 return train_data
def encoder(train_data):
dic = {'yes' : '1', 'no' : '0'}
train_data.drop(['Id'],axis=1,inplace=True)
 train_data['idhogar'] = LabelEncoder().fit_transform(train_data['idhogar'])
 train_data = getDependencyRate(train_data)
 train_data['edjefe']= train_data['edjefe'].replace(dic ).astype('float64')
 train_data['edjefa'] = train_data['edjefa'].replace(dic ).astype('float64')
 return train_data
def convertToFloat(train_data,obj_col_list,col_list_missing_data):
obj_col_list.extend(col_list_missing_data)
 for col in train_data.columns.values.tolist() :
if col not in obj_col_list:
train_data[col] = train_data[col].astype('float64')
 return train_data
def addAndModifyFeatures(train_data):
train_data['v2a1_per_room'] = train_data['v2a1']/train_data['rooms']
train_data['v18q1_per_household_member'] = train_data['v18q1']/train_data['hogar_total']
train_data['qmobileph_per_household_member'] = train_data['qmobilephone']/train_data['hogar_total']
train_data['rooms_per_household_member'] = train_data['rooms']/train_data['hogar_total']
train_data['bedroom_per_household_member'] = train_data['bedrooms']/train_data['hogar_total']
train_data['Num_of_non_household_members'] = train_data['r4t3']-train_data['hogar_total']
train_data['hogar_adul_btw_19_and_65'] =(train_data['hogar_adul']-train_data['hogar_mayor'])/train_data['hogar_total']
train_data['r4h1'] = train_data['r4h1']/train_data['r4t3']
train_data['r4h2'] = train_data['r4h2']/train_data['r4t3']
train_data['r4h3'] = train_data['r4h3']/train_data['r4t3']
train_data['r4m1'] = train_data['r4m1']/train_data['r4t3']
train_data['r4m2'] = train_data['r4m2']/train_data['r4t3']
train_data['r4m3'] = train_data['r4m3']/train_data['r4t3']
train_data['r4t1'] = train_data['r4t1']/train_data['r4t3']
train_data['r4t2'] = train_data['r4t2']/train_data['r4t3']
train_data['hogar_nin'] = train_data['hogar_nin']/train_data['hogar_total']
train_data['hogar_adul'] = train_data['hogar_adul']/train_data['hogar_total']
train_data['hogar_mayor'] = train_data['hogar_mayor']/train_data['hogar_total']
train_data.drop(['idhogar','v18q1','rooms','bedrooms','r4t3','hogar_total','agesq','hhsize','tamviv','tamhog'],axis=1,inplace=True)
 return train_data
kept_index = train_features['cp_type']==1
train_features = train_features.loc[kept_index].reset_index(drop=True)
 kept_index = test_features['cp_type']==1
test_features = test_features.loc[kept_index].reset_index(drop=True )<import_modules>
def applyFE(train_data):
col_list_missing_data = findNanCol(train_data)
 obj_col_list = findObjectTypeCol(train_data)
 train_data = convertToFloat(train_data,obj_col_list,col_list_missing_data)
 train_data = fillMissingValues(train_data)
 train_data = encoder(train_data)
 train_data = addAndModifyFeatures(train_data)
 return train_data
def getConstituentsOfPredicted(cf,pred_label,angle=45):
p13 = cf[0][pred_label-1]
p23 = cf[1][pred_label-1]
p33 = cf[2][pred_label-1]
p43 = cf[3][pred_label-1]
pred_labels = ['extreme poverty', 'moderate poverty', 'vulnerable households','non vulnerable households']
sizes = [p13, p23, p33, p43]
colors = ['gold', 'yellowgreen', 'lightcoral', 'lightskyblue']
plt.pie(sizes, labels=pred_labels, colors=colors, autopct='%1.0f%%', shadow=False, startangle=angle, textprops={'fontsize': 14})
 plt.title("Constituents of Costa Rican Household Poverty Levels predicted as "+pred_labels[pred_label-1],fontsize=20)
 plt.axis('equal')
 plt.show()
return p13,p23,p33,p43
def getPredictionOfActualLabels(p,cf,actual_label,angle=45):
labels = ['extreme poverty', 'moderate poverty', 'vulnerable households','non vulnerable households']
sizes = [p[0], p[1], p[2], p[3]]
colors = ['gold', 'yellowgreen', 'lightcoral', 'lightskyblue']
plt.pie(sizes, labels=labels, colors=colors, autopct='%1.0f%%', shadow=False, startangle=angle, textprops={'fontsize': 14})
 plt.title("Predictions of Costa Rican Household Poverty Levels with actual labels as "+labels[actual_label-1],fontsize=20)
 plt.axis('equal')
 plt.show()
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold, MultilabelStratifiedShuffleSplit<load_from_csv>
train_data = applyFE(train_data )
SEED = 42
NB_SPLITS = 5
drug = pd.read_csv('/kaggle/input/lish-moa/train_drug.csv')
 targets = train_targets_scored.columns[1:]
scored = train_targets_scored.merge(drug, on='sig_id', how='left')
 vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NB_SPLITS, shuffle=True,
random_state=SEED)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NB_SPLITS, shuffle=True,
random_state=SEED)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() ,'fold'] =\
scored.loc[scored.fold.isna() ,'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8' )<import_modules>
x_train = train_data.copy()
y_train = train_target.copy()
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-15)
 return np.mean(-aux)
<define_variables>
param_grid = {
'n_estimators': [270],
'max_depth' : [14]
}
rfc=RandomForestClassifier(class_weight = 'balanced_subsample', n_estimators = 200,max_depth = 16)
 scorer = metrics.make_scorer(metrics.f1_score, average = 'weighted')
 rf_cv = GridSearchCV(estimator=rfc, param_grid=param_grid, scoring=scorer, cv= 3)
 rf_cv.fit(x_train, y_train )
<concatenate>
print('Best score for training data:', rf_cv.best_score_,"
")
 print('Best n_estimators:',rf_cv.best_estimator_.n_estimators,"
")
 print('Best max_depth:',rf_cv.best_estimator_.max_depth,"
" )
data_all = pd.concat([train_features, test_features], ignore_index = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(data_all[cols_numeric].var() >= 0.7 ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]<normalization>
y_pred_validate = rf_cv.predict(x_train)
 print("Classification report for rf model %s:
%s
"
%(rf_cv, metrics.classification_report(y_train, y_pred_validate)) )
scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric] )<import_modules>
path = '/kaggle/input/costa-rican-household-poverty-prediction/test.csv'
test_data = pd.read_csv(path)
 test_data = applyFE(test_data)
 y_pred = rf_cv.predict(test_data)
 path = '/kaggle/input/costa-rican-household-poverty-prediction/sample_submission.csv'
test = pd.read_csv(path)
 test['Target'] = y_pred
test.to_csv("submission.csv", index= False)