Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<set_options>
%matplotlib inline
warnings.filterwarnings("ignore" )
warnings.filterwarnings('ignore')
 tf.random.set_seed(42)
<load_from_csv>
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('bed_density', 'bedrooms', 'rooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'), 
('male_over_female', 'r4h3', 'r4m3'), 
('man12plus_over_women12plus', 'r4h2', 'r4m2'), 
('pesioner_over_working', 'hogar_mayor', 'hogar_adul'), 
('children_over_working', 'hogar_nin', 'hogar_adul'), 
('education_fraction', 'escolari', 'age')
 ]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('non_bedrooms', 'rooms', 'bedrooms'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean', 'count'],
'escolari': ['min', 'max', 'mean', 'std'],
'fe_education_fraction': ['min', 'max', 'mean', 'std']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop([f_ for f_ in df.columns if f_.startswith('SQB')or f_ == 'agesq'], axis=1, inplace=True)
 df.drop(['Id'], axis=1, inplace=True)
 df.drop(['hhsize', 'female', 'area2'], axis=1, inplace=True)
 return df
train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv', nrows = 3)
 features = [c for c in train.columns if 'feature' in c]<choose_model_class>
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model<define_search_space>
train = pd.read_csv('.. /input/train.csv', nrows=None)
 test = pd.read_csv('.. /input/test.csv' )
batch_size = 8192
hidden_units = [384, 896, 896, 394]
dropout_rates = [0.10143786981358652, 0.19720339053599725, 0.2703017847244654, 0.23148340929571917, 0.2357768967777311]
label_smoothing = 1e-2
learning_rate = 1e-4<load_pretrained>
def process_df(df_):
encode_data(df_)
 for f_ in ['v2a1', 'v18q1', 'meaneduc', 'SQBmeaned']:
df_[f_] = df_[f_].fillna(0)
 df_['rez_esc'] = df_['rez_esc'].fillna(-1)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )
num_models = 2
models = []
for i in range(num_models):
clf = create_mlp(len(features), 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 clf.load_weights(f'.. /input/js-nn-models/JSModel_{i}.hdf5')
 models.append(clf )<load_pretrained>
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_
f_mean = np.load('.. /input/js-nn-models/f_mean.npy' )<split>
train, test = train_test_apply_func(train, test, convert_OHE2LE )
env = janestreet.make_env()
env_iter = env.iter_test()<categorify>
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e + '_MEAN' for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )
opt_th = 0.5
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = 0.
 for clf in models:
pred += clf(x_tt, training = False ).numpy().item() / num_models
pred_df.action = np.where(pred >= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<import_modules>
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1 )
import numpy as np
import pandas as pd
import os<load_from_csv>
cols_2_drop = ['abastagua_LE', 'agg18_estadocivil1_MEAN', 'agg18_instlevel6_MEAN', 'agg18_parentesco10_MEAN', 'agg18_parentesco11_MEAN', 'agg18_parentesco12_MEAN', 'agg18_parentesco4_MEAN', 'agg18_parentesco5_MEAN', 'agg18_parentesco6_MEAN', 'agg18_parentesco7_MEAN', 'agg18_parentesco8_MEAN', 'agg18_parentesco9_MEAN', 'fe_people_not_living', 'fe_people_weird_stat', 'geo_elimbasu_LE_3_MEAN', 'geo_elimbasu_LE_4_MEAN', 'geo_energcocinar_LE_0_MEAN', 'geo_energcocinar_LE_1_MEAN', 'geo_energcocinar_LE_2_MEAN', 'geo_epared_LE_0_MEAN', 'geo_epared_LE_2_MEAN', 'geo_etecho_LE_2_MEAN', 'geo_eviv_LE_0_MEAN', 'geo_hogar_mayor_MEAN', 'geo_hogar_nin_MEAN', 'geo_manual_elec_LE_1_MEAN', 'geo_manual_elec_LE_2_MEAN', 'geo_manual_elec_LE_3_MEAN', 'geo_pared_LE_0_MEAN', 'geo_pared_LE_1_MEAN', 'geo_pared_LE_3_MEAN', 'geo_pared_LE_4_MEAN', 'geo_pared_LE_5_MEAN', 'geo_pared_LE_6_MEAN', 'geo_pared_LE_7_MEAN', 'hacapo', 'hacdor', 'mobilephone', 'parentesco1', 'parentesco_LE', 'rez_esc', 'techo_LE', 'v14a', 'v18q']
X.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True)
 test.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True )
app_train = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_train.csv')
 app_test = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_test.csv' )<load_from_csv>
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.99, current_iter)
 return max(lr, min_learning_rate)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : 'multiclass',
"eval_metric" : evaluate_macroF1_lgb,
'eval_names': ['train', 'early_stop'],
'callbacks': [lgb.reset_parameter(learning_rate=learning_rate_power_0997)],
'verbose': False,
'categorical_feature': 'auto'}
sk_id = pd.read_csv('.. /input/home-credit-default-risk/application_test.csv' )<drop_column>
opt_parameters = {'colsample_bytree': 0.89, 'min_child_samples': 90, 'num_leaves': 14, 'subsample': 0.96}
train = app_train.drop(columns = ['TARGET'])
 train_labels = app_train['TARGET']<split>
class VotingPrefitClassifier(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
self.estimators_ = [x[1] for x in self.estimators]
self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
return self
X_train, X_test, y_train, y_test = train_test_split(
 train, train_labels, test_size=0.2 )<choose_model_class>
def train_single_model(clf_, X_, y_, random_state_=314, opt_parameters_={}, fit_params_={}):
c = clone(clf_)
 c.set_params(**opt_parameters_)
 c.set_params(random_state=random_state_)
 return c.fit(X_, y_, **fit_params_)
 def train_model_in_nestedCV(model, X, y, metric, metric_args={},
model_name='xmodel',
inner_seed=31416, inner_n=10, outer_seed=314, outer_n=10,
opt_parameters_={}, fit_params_={},
verbose=True):
clfs = []
perf_eval = {'score_i_oof': [],
'score_i_ave': [],
'score_i_std': [],
'score_i_early_stop_ave': [],
'score_o_early_stop': [],
'score_o_early_stop_vc_w0_soft': [],
'score_o_early_stop_vc_w0_hard': []
}
y_full_oof = pd.Series(np.zeros(shape=(y.shape[0],)) ,
index=y.index)
 if 'sample_weight' in metric_args:
sample_weight=metric_args['sample_weight']
outer_cv = StratifiedKFold(outer_n, shuffle=True, random_state=outer_seed)
 for n_outer_fold,(outer_trn_idx, outer_val_idx)in enumerate(outer_cv.split(X,y)) :
print('--- Outer loop iteration: {} ---'.format(n_outer_fold))
X_out, y_out = X.iloc[outer_trn_idx], y.iloc[outer_trn_idx]
X_stp, y_stp = X.iloc[outer_val_idx], y.iloc[outer_val_idx]
inner_cv = StratifiedKFold(inner_n, shuffle=True, random_state=inner_seed+n_outer_fold)
 y_outer_oof = pd.Series(np.zeros(shape=(X_out.shape[0],)) ,
index=X_out.index)
 scores_inner = []
clfs_inner = []
for n_inner_fold,(inner_trn_idx, inner_val_idx)in enumerate(inner_cv.split(X_out,y_out)) :
X_trn, y_trn = X_out.iloc[inner_trn_idx], y_out.iloc[inner_trn_idx]
X_val, y_val = X_out.iloc[inner_val_idx], y_out.iloc[inner_val_idx]
if fit_params_:
fit_params_["eval_set"] = [(X_trn,y_trn),(X_stp,y_stp)]
fit_params_['verbose'] = False
clf = train_single_model(model, X_trn, y_trn, 314+n_inner_fold, opt_parameters_, fit_params_)
 clfs_inner.append(( '{}{}_inner'.format(model_name,n_inner_fold), clf))
y_outer_oof.iloc[inner_val_idx] = clf.predict(X_val)
 if 'sample_weight' in metric_args:
metric_args['sample_weight'] = y_val.map(sample_weight)
 scores_inner.append(metric(y_val, y_outer_oof.iloc[inner_val_idx], **metric_args))
del X_trn, y_trn, X_val, y_val
if 'sample_weight' in metric_args:
metric_args['sample_weight'] = y_outer_oof.map(sample_weight)
 perf_eval['score_i_oof'].append(metric(y_out, y_outer_oof, **metric_args))
perf_eval['score_i_ave'].append(np.mean(scores_inner))
perf_eval['score_i_std'].append(np.std(scores_inner))
if 'sample_weight' in metric_args:
metric_args['sample_weight'] = y_stp.map(sample_weight)
 score_inner_early_stop = [metric(y_stp, clf_.predict(X_stp), **metric_args)
 for _,clf_ in clfs_inner]
perf_eval['score_i_early_stop_ave'].append(np.mean(score_inner_early_stop))
w = np.array(scores_inner)
 for w_, w_name_ in [(None, '_w0')
 ]:
vc = VotingPrefitClassifier(clfs_inner, weights=w_ ).fit(X_stp, y_stp)
 for vote_type in ['soft', 'hard']:
vc.voting = vote_type
if 'sample_weight' in metric_args:
metric_args['sample_weight'] = y_stp.map(sample_weight)
 perf_eval['score_o_early_stop_vc{}_{}'.format(w_name_, vote_type)].append(metric(y_stp, vc.predict(X_stp), **metric_args))
if fit_params_:
fit_params_["eval_set"] = [(X_out,y_out),(X_stp,y_stp)]
if verbose:
fit_params_['verbose'] = 200
clf = train_single_model(model, X_out, y_out, 314+n_outer_fold, opt_parameters_, fit_params_)
 if 'sample_weight' in metric_args:
metric_args['sample_weight'] = y_stp.map(sample_weight)
 perf_eval['score_o_early_stop'].append(metric(y_stp, clf.predict(X_stp), **metric_args))
clfs.append(( '{}{}'.format(model_name,n_outer_fold), clf))
y_full_oof.iloc[outer_val_idx] = clf.predict(X_stp)
 del inner_cv, X_out, y_out, X_stp, y_stp, clfs_inner
return clfs, perf_eval, y_full_oof
def print_nested_perf_clf(name, perf_eval):
print('Performance of the inner-loop model(the two should agree):')
 print(' Mean(mean(Val)) score inner {} Classifier: {:.4f}+-{:.4f}'.format(name,
np.mean(perf_eval['score_i_ave']),
np.std(perf_eval['score_i_ave'])
))
print(' Mean(mean(EarlyStop)) score inner {} Classifier: {:.4f}+-{:.4f}'.format(name,
np.mean(perf_eval['score_i_early_stop_ave']),
np.std(perf_eval['score_i_early_stop_ave'])
))
print('Mean(inner OOF)score inner {} Classifier: {:.4f}+-{:.4f}'.format(name,
np.mean(perf_eval['score_i_oof']),
np.std(perf_eval['score_i_oof'])
))
print('Mean(EarlyStop)score outer {} Classifier: {:.4f}+-{:.4f}'.format(name,
np.mean(perf_eval['score_o_early_stop']),
np.std(perf_eval['score_o_early_stop'])
))
print('Mean(EarlyStop)outer VotingPrefit SOFT: {:.4f}+-{:.4f}'.format(np.mean(perf_eval['score_o_early_stop_vc_w0_soft']),
np.std(perf_eval['score_o_early_stop_vc_w0_soft'])
))
print('Mean(EarlyStop)outer VotingPrefit HARD: {:.4f}+-{:.4f}'.format(np.mean(perf_eval['score_o_early_stop_vc_w0_hard']),
np.std(perf_eval['score_o_early_stop_vc_w0_hard'])
))
clf = LGBMClassifier(nthread=4,n_estimators=10000,learning_rate=0.02,num_leaves=34,colsample_bytree=0.9497036,subsample=0.8715623,max_depth=8,reg_alpha=0.041545473,reg_lambda=0.0735294,min_split_gain=0.0222415,min_child_weight=39.3259775,silent=-1,verbose=-1)
 clf.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_test, y_test)],
eval_metric= 'auc', verbose= 100, early_stopping_rounds= 200)
<predict_on_test>
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=1, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clfs_, perf_eval, y_full_oof = train_model_in_nestedCV(clf, X, y, f1_score,
metric_args={'average':'macro'},
model_name='lgbm',
opt_parameters_=opt_parameters,
fit_params_=fit_params,
inner_n=10, outer_n=10,
verbose=False )
<save_to_csv>
w = np.array(perf_eval['score_o_early_stop'])
 ws = [(None, '_w0'), 
(w/w.sum() , '_w1'), 
(( w**2)/np.sum(w**2), '_w2')
 ]
vc = {}
for w_, w_name_ in ws:
vc['vc{}'.format(w_name_)] = VotingPrefitClassifier(clfs_, weights=w_ ).fit(X, y)
 clf_final = clfs_[0][1]
y_pred = clf.predict_proba(app_test, num_iteration=clf.best_iteration_)[:, 1]
submit = sk_id[['SK_ID_CURR']]
submit['TARGET'] = y_pred
submit.to_csv('sub1.csv', index = False)
<set_options>
global_score = np.mean(perf_eval['score_i_oof'])
 global_score_std = np.std(perf_eval['score_i_oof'])
 print_nested_perf_clf('lgbm', perf_eval)
 print('Outer OOF score {} Classifier: {:.4f}'.format('lgbm', f1_score(y, y_full_oof, average='macro')) )
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.filterwarnings("ignore")
 np.random.seed(5)
<load_from_csv>
perf_eval_df = pd.DataFrame(perf_eval)
 perf_eval_df
train_df = pd.read_csv('.. /input/preprocessed-home-credit/local_data_final.csv')
 test_df = pd.read_csv('.. /input/preprocessed-home-credit/kaggle_test_final.csv')
<normalization>
from sklearn.metrics import precision_score, recall_score, classification_report
def normalize(t, test_df):
train_dff = t.drop(columns = ['SK_ID_CURR',"TARGET"],axis = 1)
 X_train = train_dff.values
print(X_train.shape)
 Y_train = t['TARGET']
test_dff = test_df.drop(columns = ['SK_ID_CURR',"TARGET"],axis = 1)
 X_test = test_dff.values
print(X_test.shape)
 scaler = preprocessing.StandardScaler().fit(X_train)
 print(scaler)
 xtr = pd.DataFrame(( scaler.transform(X_train)).tolist())
 xte = pd.DataFrame(( scaler.transform(X_test)).tolist())
 return xtr, xte, Y_train<prepare_x_and_y>
shap_values = shap.TreeExplainer(clf_final.booster_ ).shap_values(X)
X_train, X_test, Y_train = normalize(t, test_df )<prepare_x_and_y>
y_subm = pd.read_csv('.. /input/sample_submission.csv' )
dff = train_df.drop('SK_ID_CURR', axis = 1)
 dfff = dff.drop('TARGET', axis = 1)
 target = dff['TARGET']
X_train.columns = dfff.columns
X_test.columns = dfff.columns
train_dff = X_train
train_dff['TARGET'] = Y_train.tolist()<normalization>
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm['Target'] = clf_final.predict(test)+ 1
y_subm.to_csv(sub_file, index=False)
 for vc_name_,vc_ in vc.items() :
for vc_type_ in ['soft', 'hard']:
vc_.voting = vc_type_
name = '{}_{}'.format(vc_name_, vc_type_)
 y_subm_vc = y_subm.copy(deep=True)
 y_subm_vc.loc[:,'Target'] = vc_.predict(test)+ 1
sub_file = 'submission_{}_LGB_{:.4f}_{}.csv'.format(name,
global_score,
str(now.strftime('%Y-%m-%d-%H-%M')) 
)
 y_subm_vc.to_csv(sub_file, index=False )