Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<count_missing_values>
%matplotlib inline
sns.set(style = 'darkgrid')
 print(os.listdir(".. /input"))
rcParams['figure.figsize'] = 25, 12.5
rcParams['font.size'] = 50
pd.set_option('display.max_columns',None)
 warnings.filterwarnings('ignore')
 print(os.listdir(".. /input"))
print(x_val.isnull().sum() )<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
def imputer_null(df):
for cols in df.columns.values:
if df[cols].values.dtype=='object':
df[[cols]]=alpha_imputer.transform(df[cols].values.reshape(-1,1))
else:
df[[cols]]=num_imputer.transform(df[cols].values.reshape(-1,1))
return df<count_missing_values>
submission_sample = pd.read_csv('.. /input/sample_submission.csv' )
x_val = imputer_null(x_val)
 print(x_val.isnull().sum() )<drop_column>
color_dict = OrderedDict({1: 'red', 2: 'orange', 3: 'blue', 4: 'green'})
 poverty_dict = OrderedDict({1: 'extreme', 2: 'moderate', 3: 'vulnerable', 4: 'non vulnerable'} )
x_train.reset_index(drop=True,inplace=True)
 x_train
x_val.reset_index(drop=True,inplace=True)
 x_val<prepare_x_and_y>
train.loc[:,['dependency','edjefe','edjefa']] =train[['dependency','edjefe','edjefa']].replace(to_replace={'yes':1,'no':0} ).astype(np.float64)
 test.loc[:,['dependency','edjefe','edjefa']] = test[['dependency','edjefe','edjefa']].replace(to_replace={'yes':1,'no':0} ).astype(np.float64 )
y_train.reset_index(drop=True,inplace=True)
 y_train
y_val.reset_index(drop=True,inplace=True )<string_transform>
household_checktarget =train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 idhogar_mult_target = household_checktarget[household_checktarget!=True].index
print('Number of households where the poverty level is more than one:{}'.format(len(idhogar_mult_target)) )
ticket_new = x_train['Ticket'].str.split(" ",n=1,expand=True)
 print(ticket_new)
 print(type(ticket_new))
print(x_train.head() )<concatenate>
for idh in idhogar_mult_target:
actual_povertylevel = train.loc[(( train.parentesco1==1)&(train.idhogar==idh)) ,'Target']
train.loc[(train.idhogar==idh),'Target'] = actual_povertylevel
x_train_new = pd.concat([x_train,ticket_new],axis=1 ).drop(['Ticket'],axis=1)
 x_train_new 
<prepare_output>
household_num_head = train.groupby('idhogar')['parentesco1'].sum()
idhogar_nohead = household_num_head[household_num_head!=1].index
print('Number of households where there are no head:{}'.format(len(idhogar_nohead)) )
x_train_new.rename(columns={0:"Ticket_ind",1:"Ticket_no"},inplace=True)
 x_train=x_train_new.copy()
x_train.head()<drop_column>
train[train.idhogar == 'f2bfa75c4']
x_train['Ticket_no'].values
x_train.drop(columns=['Ticket_no'],axis=1,inplace=True)
 x_train.head()<data_type_conversions>
id_columns = ['Id', 'idhogar', 'Target']
household_boolean = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2']
household_continuos = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']
household_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone']
ind_ordered = ['rez_esc', 'escolari', 'age']
squared_common = ['SQBescolari','SQBage', 'SQBhogar_total','SQBedjefe','SQBhogar_nin','SQBovercrowding',
'SQBdependency', 'SQBmeaned']
x_num = x_train[x_train['Ticket_ind'].str.isnumeric() ]
x_num<normalization>
feature_train_ind_cont = train.groupby('idhogar')[ind_ordered].agg(['mean', 'max', 'min', 'sum'])
 feature_test_ind_cont = test.groupby('idhogar')[ind_ordered].agg(['mean', 'max', 'min', 'sum'])
 new_cols = []
for col in feature_train_ind_cont.columns.levels[0]:
for stat in feature_train_ind_cont.columns.levels[1]:
new_cols.append(f'{col}-{stat}')
 feature_train_ind_cont.columns = new_cols
feature_test_ind_cont.columns = new_cols
feature_train_ind_cont.reset_index(inplace=True)
 feature_test_ind_cont.reset_index(inplace=True )
x_alpha = x_train[x_train['Ticket_ind'].str.isalpha() ]
x_alpha.head(30 )<drop_column>
feature_train_ind_cont['rez_esc-mean'].isnull().sum() /len(feature_train_ind_cont )
x_train.drop(columns=['Ticket_ind'],axis=1,inplace=True )<drop_column>
feature_train_ind = pd.merge(feature_train_ind_bool,feature_train_ind_cont,on='idhogar',how='inner')
 feature_test_ind = pd.merge(feature_test_ind_bool,feature_test_ind_cont,on='idhogar',how='inner')
 print('Individual training features data shape:{}
Individual test features data shape:{}'.format(feature_train_ind.shape,feature_test_ind.shape))
x_val.drop(columns=['Ticket'],axis=1,inplace=True )<categorify>
feature_train = pd.merge(feature_train_household,feature_train_ind,on='idhogar',how='left')
 feature_test = pd.merge(feature_test_household,feature_test_ind,on='idhogar',how='left')
 print('Combined features train data shape:{}
Combined features test data shape:{}'.format(feature_train.shape,feature_test.shape))
one_enc = OneHotEncoder(handle_unknown='ignore')
 df_new=pd.DataFrame()
col_names={}
to_be_enc_cols = ['Pclass','Sex','Embarked']
for column_ind in range(len(to_be_enc_cols)) :
one_hot_enc = one_enc.fit_transform(np.array(x_train[to_be_enc_cols[column_ind]] ).reshape(-1,1)).toarray()
col_names[to_be_enc_cols[column_ind]] = one_enc.get_feature_names([to_be_enc_cols[column_ind]])
 df = pd.DataFrame(one_hot_enc,columns=col_names[to_be_enc_cols[column_ind]])
 df_new = pd.concat([df,df_new], axis = 1)
 x_train = pd.concat([x_train,df_new], axis = 1 ).drop(['Pclass','Sex','Embarked'],axis=1)
 print(x_train)
<categorify>
target_train = feature_train['Target']-1
test_idhogar = feature_test['idhogar'].values
try:
feature_train.drop('Target',axis=1,inplace=True)
 feature_train.drop(['Id','idhogar'],axis=1,inplace=True)
 feature_test.drop(['Id','idhogar'],axis=1,inplace=True)
 except:
print("Couldnot drop all columns")
 feature_train, feature_test = feature_train.align(feature_test, axis = 1, join = 'inner')
 feature_X_train,feature_X_valid,target_y_train,target_y_valid = train_test_split(feature_train,target_train,test_size=0.2,stratify=target_train,random_state=123)
def one_hot(df):
df2=pd.DataFrame()
col_names={}
to_be_enc_cols = ['Pclass','Sex','Embarked']
for column_ind in range(len(to_be_enc_cols)) :
one_hot_enc = one_enc.fit_transform(np.array(df[to_be_enc_cols[column_ind]] ).reshape(-1,1)).toarray()
col_names[to_be_enc_cols[column_ind]] = one_enc.get_feature_names([to_be_enc_cols[column_ind]])
 df1 = pd.DataFrame(one_hot_enc,columns=col_names[to_be_enc_cols[column_ind]])
 df2 = pd.concat([df1,df2], axis = 1)
 one_hot_df = pd.concat([df,df2], axis = 1 ).drop(['Pclass','Sex','Embarked'],axis=1)
 return one_hot_df<categorify>
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True
<categorify>
nfolds=5
sfold = StratifiedKFold(n_splits= nfolds,shuffle=True )
<categorify>
<categorify>
dict_optimum_params = {'colsample_bytree': 0.605756972263847,
'learning_rate': 0.01,
'min_child_samples': 19,
'num_leaves': 29,
'subsample': 0.5522796688057564}
x_val = one_hot(x_val)
 x_val<feature_engineering>
test_predictions_df=train_eval(feature_train,feature_test,target_train )
def add_family_feature(df):
df['Family'] = df['SibSp']+df['Parch']
df.drop(['SibSp','Parch'],axis=1,inplace=True)
 return df<feature_engineering>
x_train = add_family_feature(x_train)
 x_val = add_family_feature(x_val)
 x_val.head()<normalization>
submission_df = test.loc[:,['Id','idhogar']]
normalize = MinMaxScaler()
normalize_col = ['Age','Fare']
df_new = pd.DataFrame()
for cols in normalize_col:
x_train_scaled = normalize.fit_transform(np.array(x_train[cols] ).reshape(-1,1))
df_scaled = pd.DataFrame(x_train_scaled,columns=[cols+'_new'])
 df_new = pd.concat([df_new,df_scaled],axis=1)
 print(df_new)
 x_train = pd.concat([x_train,df_new],axis=1)
 x_train.drop(columns=['Age','Fare'],axis=1,inplace=True)
 x_train.head()<create_dataframe>
submission = pd.merge(submission_df,test_predictions_df[['idhogar','Target']],on='idhogar',how='left' ).drop('idhogar',axis=1 )
def normalize_feature(df):
normalize_col = ['Age','Fare']
df_new = pd.DataFrame()
for cols in normalize_col:
x_scaled = normalize.fit_transform(np.array(df[cols] ).reshape(-1,1))
df_scaled = pd.DataFrame(x_scaled,columns=[cols+'_new'])
 df_new = pd.concat([df_new,df_scaled],axis=1)
 df = pd.concat([df,df_new],axis=1)
 df.drop(columns=['Age','Fare'],axis=1,inplace=True)
 return df<feature_engineering>
submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8 )
<prepare_output><EOS> 
submission.to_csv('sample_submission.csv',index=False )