Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<data_type_conversions>
import numpy as np
import pandas as pd
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold
data['MSZoning'] = data['MSZoning'].fillna(data['MSZoning'].mode() [0] )<drop_column>
df = pd.read_csv(".. /input/train.csv" )
data = data.drop(['Utilities'], axis=1 )<data_type_conversions>
def preprocess(df):

 def mk_categoricals(df, prefixes=None, subsets=None):

 def mk_category(dummies):
assert(dummies.sum(axis=1)<= 1 ).all()
nans = dummies.sum(axis=1)!= 1
if nans.any() :
dummies = dummies.assign(_na=nans.astype(int))
return dummies.idxmax(axis=1 ).astype('category')
 categoricals = pd.DataFrame()
if prefixes:
for prefix in prefixes:
columns = df.columns[df.columns.str.startswith(prefix)]
categoricals[prefix] = mk_category(df[columns])
 if subsets:
for feature_name, subset in subsets.items() :
categoricals[feature_name] = mk_category(df[subset])
 return categoricals
groupper = df.groupby('idhogar')
 interactions =(pd.DataFrame(dict(
 head_escolari=df.parentesco1 * df.escolari,
head_female=df.parentesco1 * df.female,
head_partner_escolari=df.parentesco2 * df.escolari)) 
.groupby(df.idhogar)
.max())
 my_features =(groupper.mean() [['escolari', 'age', 'hogar_nin',
'hogar_total', 'epared3', 'epared1',
'etecho3', 'etecho1', 'eviv3', 'eviv1',
'male',
'r4h1', 'r4h2', 'r4h3', 'r4m1', 'r4m2',
'r4m3',
'r4t1', 'r4t2', 'r4t3', 'v2a1', 'rooms',
'bedrooms',
'meaneduc',
'SQBdependency', 'rez_esc', 'refrig',
'tamviv', 'overcrowding']] 
.join(groupper.std() [['escolari', 'age']],
rsuffix='_std')
.join(groupper[['escolari', 'age']].min() , rsuffix="_min")
.join(groupper[['escolari', 'age']].max() , rsuffix="_max")
.join(groupper[['dis']].sum() , rsuffix="_sum")
.assign(child_rate=lambda x: x.hogar_nin / x.hogar_total,
wrf=lambda x: x.epared3 - x.epared1 +
x.etecho3 - x.etecho1 +
x.eviv3 - x.eviv1,
escolari_range=lambda x: x.escolari_max - x.escolari_min,
age_range=lambda x: x.age_max - x.age_min,
rent_per_individual=lambda x: x.v2a1 / x.r4t3,
rent_per_child=lambda x: x.v2a1 / x.r4t1,
rent_per_over65=lambda x: x.v2a1 / x.r4t3,
rent_per_room=lambda x: x.v2a1 / x.rooms,
rent_per_bedroom=lambda x: x.v2a1 / x.bedrooms,
rooms_per_individual=lambda x: x.rooms / x.r4t3,
rooms_per_child=lambda x: x.rooms / x.r4t1,
bedrooms_per_individual=lambda x: x.bedrooms / x.r4t3,
bedrooms_per_child=lambda x: x.bedrooms / x.r4t1,
years_schooling_per_individual=lambda x: x.escolari / x.r4t3,
years_schooling_per_adult=lambda x: x.escolari /(x.r4t3 - x.r4t1),
years_schooling_per_child=lambda x: x.escolari / x.r4t3 
)
.drop(['hogar_nin', 'hogar_total', 'epared3', 'epared1',
'etecho3', 'etecho1', 'eviv3', 'eviv1'],
axis=1)
.join(interactions)
.join(groupper[['computer', 'television',
'qmobilephone', 'v18q1']] 
.mean().sum(axis=1 ).rename('technics')) 
.assign(technics_per_individual=lambda x: x.technics / x.r4t3,
technics_per_child=lambda x: x.technics / x.r4t1)
.join(mk_categoricals(groupper.mean() ,
prefixes=['lugar', 'area', 'tipovivi',
'energcocinar',
'sanitario', 'pared', 'piso',
'abastagua'],
subsets={'electricity': ['public',
'planpri',
'noelec',
'coopele']})) 
)
 return my_features
data["Functional"] = data["Functional"].fillna("Typ" )<feature_engineering>
X = preprocess(df)
 y = df.groupby('idhogar' ).Target.mean().astype(int)
data['Electrical'] = data['Electrical'].fillna(data['Electrical'].mode() [0] )<feature_engineering>
df_test = pd.read_csv(".. /input/test.csv" ).set_index('Id')
 X_test = preprocess(df_test )
data['KitchenQual'] = data['KitchenQual'].fillna(data['KitchenQual'].mode() [0] )<categorify>
kf = StratifiedKFold(n_splits=5, shuffle=True)
 predicts = []
for train_index, test_index in kf.split(X, y):
print("
X_train, X_val = X.iloc[train_index], X.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train, eval_set=[(X_val, y_val)],
early_stopping_rounds=20)
 predicts.append(clf.predict(X_test))
data['Exterior1st'] = data['Exterior1st'].fillna(data['Exterior1st'].mode() [0])
 data['Exterior2nd'] = data['Exterior2nd'].fillna(data['Exterior2nd'].mode() [0] )<feature_engineering>
predict_by_hh = pd.DataFrame(np.array(predicts ).mean(axis=0 ).round().astype(int),
columns=['Target'],
index=X_test.index )
data['SaleType'] = data['SaleType'].fillna(data['SaleType'].mode() [0] )<data_type_conversions>
predict = df_test.join(predict_by_hh, on='idhogar')[['Target']].astype(int )
<data_type_conversions><EOS> 
predict.to_csv("output.csv" )