Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<save_to_csv>
warnings.filterwarnings('ignore')
 %matplotlib inline
if enable_pandas_profilling:
profile_train = ProfileReport(df_train_feats, title='Training Dataset Report', explorative = True)
 profile_train.to_file(output_file='profile_train.html')
 profile_train<drop_column>
pd.set_option('display.max_columns', None )
df_train_feats = df_train_feats.drop(columns=['MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5'])
 df_test_feats = df_test_feats.drop(columns=['MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5'] )<sort_values>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 train_id = df_train['Id']
test_id = df_test['Id']
train_idhogar = df_train['idhogar']
test_idhogar = df_train['idhogar']
df_train.drop(columns=['Id'], inplace=True)
 df_test.drop(columns=['Id'], inplace=True)
 print("Shape of train data: ", df_train.shape)
 print("Shape of test data: ", df_test.shape)
 ntrain = df_train.shape[0]
ntest = df_test.shape[0]
all_data = pd.concat(( df_train, df_test)).reset_index(drop=True )
corr['Weekly_Sales'].sort_values()<drop_column>
print("The feature that we need to predict: ", set(df_train.columns)- set(df_test.columns))
df_train_feats = df_train_feats.drop(columns=['Fuel_Price', 'Temperature','Date', 'CPI', 'Unemployment'])
 df_test_feats = df_test_feats.drop(columns=['Fuel_Price', 'Temperature','Date', 'CPI', 'Unemployment'] )<sort_values>
outside_wall_material_df = df_train_heads.groupby('Target' ).sum() [['paredblolad', 'paredzocalo', 'paredpreb', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother']]
outside_wall_material_df
corr['Weekly_Sales'].sort_values()<groupby>
floor_material_df = df_train_heads.groupby('Target' ).sum() [['pisomoscer', 'pisocemento', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera']]
floor_material_df
df_sales_2010_weekly = df_train_feats[df_train_feats.Year==2010]['Weekly_Sales'].groupby(df_train_feats['Week'] ).sum()
df_sales_2011_weekly = df_train_feats[df_train_feats.Year==2011]['Weekly_Sales'].groupby(df_train_feats['Week'] ).sum()
df_sales_2012_weekly = df_train_feats[df_train_feats.Year==2012]['Weekly_Sales'].groupby(df_train_feats['Week'] ).sum()<feature_engineering>
toilet_df = df_train_heads.groupby('Target' ).sum() [['sanitario1', 'sanitario2', 'sanitario3', 'sanitario5',
'sanitario6']]
toilet_df
df_train_feats.loc[(df_train_feats.Year==2010)&(df_train_feats.Week==13), 'IsHoliday'] = True
df_train_feats.loc[(df_train_feats.Year==2011)&(df_train_feats.Week==16), 'IsHoliday'] = True
df_train_feats.loc[(df_train_feats.Year==2012)&(df_train_feats.Week==14), 'IsHoliday'] = True
df_test_feats.loc[(df_test_feats.Year==2013)&(df_test_feats.Week==13), 'IsHoliday'] = True<categorify>
rubbish_disposal_df = df_train_heads.groupby('Target' ).sum() [['elimbasu1', 'elimbasu2', 'elimbasu3',
'elimbasu4', 'elimbasu5', 'elimbasu6']]
rubbish_disposal_df
def nominal_to_ordinal(x):
if x == 'A':
return 3
elif x == 'B':
return 2
else:
return 1
df_train_feats.Type = df_train_feats.Type.apply(nominal_to_ordinal)
 df_test_feats.Type = df_test_feats.Type.apply(nominal_to_ordinal )<categorify>
roof_material_df = df_train_heads.groupby('Target' ).sum() [['techozinc', 'techoentrepiso', 'techocane', 'techootro']]
roof_material_df
def bool_to_num(x):
if x:
return 1
else:
return 0
df_train_feats.IsHoliday = df_train_feats.IsHoliday.apply(bool_to_num)
 df_test_feats.IsHoliday = df_test_feats.IsHoliday.apply(bool_to_num )<prepare_x_and_y>
water_provision_df = df_train_heads.groupby('Target' ).sum() [['abastaguadentro', 'abastaguafuera', 'abastaguano']]
water_provision_df
X_train = df_train_feats.drop('Weekly_Sales', axis= 1 ).values
y_train = df_train_feats.Weekly_Sales.values<train_model>
electricity_df = df_train_heads.groupby('Target' ).sum() [['public', 'planpri', 'noelec', 'coopele']]
electricity_df
if enable_tpot:
X_tpot_train, X_tpot_test, y_tpot_train, y_tpot_test = train_test_split(X_train, y_train, train_size=0.75, test_size=0.25, random_state=42)
 tpot = TPOTRegressor(generations=5, population_size=50, verbosity=2, random_state=42)
 tpot.fit(X_tpot_train, y_tpot_train)
 print(tpot.score(X_tpot_test, y_tpot_test))
tpot.export('tpot_pipeline.py' )<train_model>
cooking_energy_df = df_train_heads.groupby('Target' ).sum() [['energcocinar1', 'energcocinar2', 'energcocinar3',
'energcocinar4']]
cooking_energy_df
model = RandomForestRegressor(n_estimators=100, max_depth=27, max_features=6, min_samples_split=8, min_samples_leaf=1)
 model.fit(X_train, y_train)
 preds = model.predict(X_train)
 rmse_all = np.sqrt(mean_squared_error(y_train, preds))
print("RMSE: %f" %(rmse_all))
del model<train_model>
avg_household_size_df = df_train_heads.groupby('Target' ).mean() ['hhsize']
avg_household_size_df
model = RandomForestRegressor(n_estimators=50, max_depth=27, max_features=6, min_samples_split=8, min_samples_leaf=1)
 model.fit(X_train, y_train)
 preds = model.predict(X_train)
 rmse_all = np.sqrt(mean_squared_error(y_train, preds))
print("RMSE: %f" %(rmse_all))
del model<train_model>
urban_rural_df = df_train_heads.groupby('Target' ).sum() [['area1', 'area2']]
urban_rural_df['UrbanPercentage'] = urban_rural_df['area1'] * round(( 100/sum(urban_rural_df['area1'])) , 6)
 urban_rural_df['RuralPercentage'] = urban_rural_df['area2'] * round(( 100/sum(urban_rural_df['area2'])) , 6)
 urban_rural_df
model = RandomForestRegressor(n_estimators=50, max_depth=27, max_features=6, min_samples_split=4, min_samples_leaf=1)
 model.fit(X_train, y_train)
 preds = model.predict(X_train)
 rmse_all = np.sqrt(mean_squared_error(y_train, preds))
print("RMSE: %f" %(rmse_all))
del model<train_model>
region_df = df_train_heads.groupby('Target' ).sum() [['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']]
region_df
model = RandomForestRegressor(n_estimators=50, max_depth=27, max_features=6, min_samples_split=3, min_samples_leaf=1)
 model.fit(X_train, y_train)
 preds = model.predict(X_train)
 rmse_all = np.sqrt(mean_squared_error(y_train, preds))
print("RMSE: %f" %(rmse_all))
del model<train_model>
round(((all_data.shape[0] - sum(all_data['v2a1'].value_counts())) / all_data.shape[0])* 100, 2 )
model = RandomForestRegressor(n_estimators=50, max_depth=27, max_features=6, min_samples_split=2, min_samples_leaf=1)
 model.fit(X_train, y_train)
 preds = model.predict(X_train)
 rmse_all = np.sqrt(mean_squared_error(y_train, preds))
print("RMSE: %f" %(rmse_all))<predict_on_test>
all_data.drop(columns=['sanitario1', 'sanitario6',
'elimbasu4', 'elimbasu5', 'elimbasu6',
'techozinc', 'techoentrepiso', 'techocane', 'techootro',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele'], inplace=True )
predicted = model.predict(df_test_feats )<save_to_csv>
yes_no_map = {'no': 0, 'yes': 1}
all_data['dependency'] = all_data['dependency'].replace(yes_no_map ).astype(np.float32)
 all_data['edjefe'] = all_data['edjefe'].replace(yes_no_map ).astype(np.float32)
 all_data['edjefa'] = all_data['edjefa'].replace(yes_no_map ).astype(np.float32 )
df_sample_submission['Weekly_Sales'] = predicted
df_sample_submission.to_csv('submission.csv',index=False )<set_options>
num_binary_features = []
for feature in all_data.columns:
if sorted(df_train[feature].unique())in [[0, 1], [0], [1]]:
num_binary_features.append(feature)
 print("Total number of binary-numerical features: ", len(num_binary_features))
print("Binary-numerical features: ")
 num_binary_features
warnings.filterwarnings("ignore")
 pysqldf = lambda q: sqldf(q, globals() )<load_from_csv>
num_non_binary_features = [feature for feature in all_data.columns if feature not in num_binary_features]
print("Total number of non-binary-numerical features: ", len(num_non_binary_features))
print("Non-binary numerical features: ")
 num_non_binary_features_dict = {feature: len(all_data[feature].unique())for feature in num_non_binary_features}
num_non_binary_features_sorted = sorted(num_non_binary_features_dict,
key=lambda feature: num_non_binary_features_dict[feature],
reverse=True)
 num_non_binary_features_len_sorted = [num_non_binary_features_dict[feature] for feature in num_non_binary_features_sorted]
plt.figure(figsize=(16, 16))
barplot_with_anotate(num_non_binary_features_sorted, num_non_binary_features_len_sorted);
plt.ylabel("No.of unique values");
plt.xlabel("Non-binary numerical features");
features = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/features.csv.zip')
 train = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/train.csv.zip')
 stores = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/stores.csv')
 test = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/test.csv.zip')
 sample_submission = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/sampleSubmission.csv.zip' )<merge>
missing_features(all_data, all_data.columns )
feat_sto = features.merge(stores, how='inner', on='Store' )<create_dataframe>
all_data.loc[:, 'v2a1'].fillna(0, inplace=True )
pd.DataFrame(feat_sto.dtypes, columns=['Type'] )<create_dataframe>
all_data.loc[:, 'v18q1'].fillna(0, inplace=True )
pd.DataFrame({'Type_Train': train.dtypes, 'Type_Test': test.dtypes} )<data_type_conversions>
all_data.loc[:, 'meaneduc'].fillna(all_data['meaneduc'].mean() , inplace=True)
 all_data.loc[:, 'SQBmeaned'].fillna(all_data['SQBmeaned'].mean() , inplace=True )
feat_sto.Date = pd.to_datetime(feat_sto.Date)
 train.Date = pd.to_datetime(train.Date)
 test.Date = pd.to_datetime(test.Date )<feature_engineering>
all_data.drop(columns=['rez_esc'], inplace=True )
feat_sto['Week'] = feat_sto.Date.dt.week
feat_sto['Year'] = feat_sto.Date.dt.year<merge>
all_data['WallQual'] = all_data['epared1'] + 2*all_data['epared2'] + 3*all_data['epared3']
all_data['RoofQual'] = all_data['etecho1'] + 2*all_data['etecho2'] + 3*all_data['etecho3']
all_data['FloorQual'] = all_data['eviv1'] + 2*all_data['eviv2'] + 3*all_data['eviv3']
all_data['EducationLevel'] = all_data['instlevel1'] + 2*all_data['instlevel2'] + 3*all_data['instlevel3'] + \
4*all_data['instlevel4'] + 5*all_data['instlevel5'] + 6*all_data['instlevel6'] + 7*all_data['instlevel7'] + \
8*all_data['instlevel8'] + 9*all_data['instlevel9']
train_detail = train.merge(feat_sto,
how='inner',
on=['Store','Date','IsHoliday'] ).sort_values(by=['Store',
'Dept',
'Date'] ).reset_index(drop=True )<merge>
all_data.drop(columns=['epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3',
'eviv1', 'eviv2', 'eviv3',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5',
'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9'], inplace=True )
test_detail = test.merge(feat_sto,
how='inner',
on=['Store','Date','IsHoliday'] ).sort_values(by=['Store',
'Dept',
'Date'] ).reset_index(drop=True )<drop_column>
redundant_features = ['r4t1', 'r4t2', 'r4t3', 'tamhog', 'tamviv', 'hhsize', 'r4t3', 'v18q', 'mobilephone']
all_data.drop(columns=redundant_features, inplace=True )
del features, train, stores, test<sort_values>
all_data['RentPerRoom'] = all_data['v2a1'] / all_data['rooms']
all_data['AdultsPerRoom'] = all_data['hogar_adul'] / all_data['rooms']
all_data['AdultsPerBedroom'] = all_data['hogar_adul'] / all_data['bedrooms']
null_columns =(train_detail.isnull().sum(axis = 0)/len(train_detail)).sort_values(ascending=False ).index
null_data = pd.concat([
train_detail.isnull().sum(axis = 0), 
(train_detail.isnull().sum(axis = 0)/len(train_detail)).sort_values(ascending=False),
train_detail.loc[:, train_detail.columns.isin(list(null_columns)) ].dtypes], axis=1)
 null_data = null_data.rename(columns={0: '
1: '% null',
2: 'type'} ).sort_values(ascending=False, by = '% null')
 null_data = null_data[null_data["
null_data<feature_engineering>
ind_bool = ['dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'EducationLevel']
ind_ordered = ['escolari', 'age']
train_detail.loc[(train_detail.Year==2010)&(train_detail.Week==13), 'IsHoliday'] = True
train_detail.loc[(train_detail.Year==2011)&(train_detail.Week==16), 'IsHoliday'] = True
train_detail.loc[(train_detail.Year==2012)&(train_detail.Week==14), 'IsHoliday'] = True
test_detail.loc[(test_detail.Year==2013)&(test_detail.Week==13), 'IsHoliday'] = True<drop_column>
f = lambda x: x.std(ddof=0)
 f.__name__ = 'std_0'
ind_agg = all_data.groupby('idhogar')[ind_ordered + ind_bool].agg(['mean', 'max', 'min', 'sum', f])
 new_cols = []
for col in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_cols.append(f'{col}-{stat}')
 ind_agg.columns = new_cols
ind_agg.head()
train_detail = train_detail.drop(columns=['Fuel_Price','MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5'])
 test_detail = test_detail.drop(columns=['Fuel_Price','MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5'] )<categorify>
print("Original number of features:", all_data.shape[1])
 all_data = all_data.merge(ind_agg, on = 'idhogar', how = 'left')
 print("Number of features after merging transformed individual level features", all_data.shape[1])
 all_data.drop(columns=ind_bool+ind_ordered, inplace=True)
 print("Number of features after dropping the individual level features", all_data.shape[1] )
train_detail.Type = train_detail.Type.apply(lambda x: 3 if x == 'A' else(2 if x == 'B' else 1))
test_detail.Type = test_detail.Type.apply(lambda x: 3 if x == 'A' else(2 if x == 'B' else 1))<drop_column>
from sklearn.metrics import f1_score, make_scorer
import lightgbm as lgb
import xgboost as xgb
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import StratifiedKFold
train_detail = train_detail.drop(columns=['Temperature'])
 test_detail = test_detail.drop(columns=['Temperature'] )<drop_column>
all_data.drop(columns=['idhogar'], inplace=True )
train_detail = train_detail.drop(columns=['CPI'])
 test_detail = test_detail.drop(columns=['CPI'] )<drop_column>
df_train = all_data[:ntrain][:]
df_test = all_data[ntrain:][:]
df_test = df_test.drop('Target', axis=1 )
train_detail = train_detail.drop(columns=['Unemployment'])
 test_detail = test_detail.drop(columns=['Unemployment'] )<compute_test_metric>
X_train= df_train.drop('Target', axis= 1)
 Y_train= df_train['Target']
X_test= df_test
def WMAE(dataset, real, predicted):
weights = dataset.IsHoliday.apply(lambda x: 5 if x else 1)
 return np.round(np.sum(weights*abs(real-predicted)) /(np.sum(weights)) , 2 )<train_model>
validation_scores = {}
def random_forest(n_estimators, max_depth):
result = []
for estimator in n_estimators:
for depth in max_depth:
wmaes_cv = []
for i in range(1,5):
print('k:', i, ', n_estimators:', estimator, ', max_depth:', depth)
 x_train, x_test, y_train, y_test = train_test_split(X_train, Y_train, test_size=0.3)
 RF = RandomForestRegressor(n_estimators=estimator, max_depth=depth)
 RF.fit(x_train, y_train)
 predicted = RF.predict(x_test)
 wmaes_cv.append(WMAE(x_test, y_test, predicted))
print('WMAE:', np.mean(wmaes_cv))
result.append({'Max_Depth': depth, 'Estimators': estimator, 'WMAE': np.mean(wmaes_cv)})
 return pd.DataFrame(result )<find_best_model_class>
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )
def random_forest_II(n_estimators, max_depth, max_features):
result = []
for feature in max_features:
wmaes_cv = []
for i in range(1,5):
print('k:', i, ', max_features:', feature)
 x_train, x_test, y_train, y_test = train_test_split(X_train, Y_train, test_size=0.3)
 RF = RandomForestRegressor(n_estimators=n_estimators, max_depth=max_depth, max_features=feature)
 RF.fit(x_train, y_train)
 predicted = RF.predict(x_test)
 wmaes_cv.append(WMAE(x_test, y_test, predicted))
print('WMAE:', np.mean(wmaes_cv))
result.append({'Max_Feature': feature, 'WMAE': np.mean(wmaes_cv)})
 return pd.DataFrame(result )<split>
skf = StratifiedKFold(n_splits=5 )
def random_forest_III(n_estimators, max_depth, max_features, min_samples_split, min_samples_leaf):
result = []
for split in min_samples_split:
for leaf in min_samples_leaf:
wmaes_cv = []
for i in range(1,5):
print('k:', i, ', min_samples_split:', split, ', min_samples_leaf:', leaf)
 x_train, x_test, y_train, y_test = train_test_split(X_train, Y_train, test_size=0.3)
 RF = RandomForestRegressor(n_estimators=n_estimators, max_depth=max_depth, max_features=max_features,
min_samples_leaf=leaf, min_samples_split=split)
 RF.fit(x_train, y_train)
 predicted = RF.predict(x_test)
 wmaes_cv.append(WMAE(x_test, y_test, predicted))
print('WMAE:', np.mean(wmaes_cv))
result.append({'Min_Samples_Leaf': leaf, 'Min_Samples_Split': split, 'WMAE': np.mean(wmaes_cv)})
 return pd.DataFrame(result )<prepare_x_and_y>
lightgbm = lgb.LGBMClassifier(class_weight='balanced', boosting_type='dart',
drop_rate=0.9, min_data_in_leaf=100,
max_bin=255,
n_estimators=500,
bagging_fraction=0.01,
min_sum_hessian_in_leaf=1,
importance_type='gain',
learning_rate=0.1,
max_depth=-1,
num_leaves=31)
X_train = train_detail[['Store','Dept','IsHoliday','Size','Week','Type','Year']]
Y_train = train_detail['Weekly_Sales']<define_search_space>
predicts_lgb = []
for train_index, test_index in skf.split(X_train, Y_train):
X_t, X_v = X_train.iloc[train_index], X_train.iloc[test_index]
y_t, y_v = Y_train.iloc[train_index], Y_train.iloc[test_index]
lightgbm.fit(X_t, y_t, eval_set=[(X_v, y_v)], early_stopping_rounds=50)
 predicts_lgb.append(lightgbm.predict(X_test))
n_estimators = [56, 58, 60]
max_depth = [25, 27, 30]
random_forest(n_estimators, max_depth )<define_search_space>
lightgbm_pred = np.array(predicts_lgb ).mean(axis=0 ).round().astype(int)
 submission_lgb = pd.DataFrame({'Id': test_id,
'Target': lightgbm_pred})
 submission_lgb.to_csv('submissionLGB.csv', index=False )
min_samples_split = [2, 3, 4]
min_samples_leaf = [1, 2, 3]
random_forest_III(n_estimators=58, max_depth=27, max_features=6,
min_samples_split=min_samples_split, min_samples_leaf=min_samples_leaf )<train_model>
predicts_xgb = []
for train_index, test_index in skf.split(X_train, Y_train):
X_t, X_v = X_train.iloc[train_index], X_train.iloc[test_index]
y_t, y_v = Y_train.iloc[train_index], Y_train.iloc[test_index]
xgboost.fit(X_t, y_t, eval_set=[(X_v, y_v)], early_stopping_rounds=50)
 predicts_xgb.append(xgboost.predict(X_test))
RF = RandomForestRegressor(n_estimators=58, max_depth=27, max_features=6, min_samples_split=3, min_samples_leaf=1)
 RF.fit(X_train, Y_train )<predict_on_test>
xgboost_pred = np.array(predicts_xgb ).mean(axis=0 ).round().astype(int)
 submission_xgb = pd.DataFrame({'Id': test_id,
'Target': xgboost_pred})
 submission_xgb.to_csv('submissionXGB.csv', index=False )
X_test = test_detail[['Store', 'Dept', 'IsHoliday', 'Size', 'Week', 'Type', 'Year']]
predict = RF.predict(X_test )<prepare_output>
submission_model_lgb_old = lgb.LGBMClassifier(class_weight='balanced', boosting_type='dart',
drop_rate=0.9, min_data_in_leaf=100,
max_bin=255,
n_estimators=500,
bagging_fraction=0.01,
min_sum_hessian_in_leaf=1,
importance_type='gain',
learning_rate=0.1,
max_depth=-1,
num_leaves=31 )
Final = X_test[['Store', 'Dept', 'Week']]
Final['Weekly_Sales'] = predict<save_to_csv>
submission_model_lgb_old.fit(X_train, Y_train);
sample_submission['Weekly_Sales'] = Final_adj['Weekly_Sales_Adjusted']
sample_submission.to_csv('submission.csv',index=False )<load_from_csv>
final_pred_lgb_old = submission_model_lgb_old.predict(X_test)
 final_pred_lgb_old = final_pred_lgb_old.astype(int )
train=pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/train.csv.zip")
 train.head()<load_from_csv>
submission_lgb_old = pd.DataFrame({'Id': test_id,
'Target': final_pred_lgb_old})
 submission_lgb_old.to_csv('submissionLGBold.csv', index=False )
test=pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/test.csv.zip")
 test.head()<prepare_x_and_y>
<data_type_conversions><EOS>