Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<merge>
%matplotlib inline
warnings.filterwarnings("ignore" )
prev_apps_count = prev[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
prev['SK_ID_PREV'] = prev['SK_ID_CURR'].map(prev_apps_count['SK_ID_PREV'])
 prev_apps_avg = prev.groupby('SK_ID_CURR' ).mean()
prev_apps_avg.columns = ['p_' + col for col in prev_apps_avg.columns]
df = df.merge(right=prev_apps_avg.reset_index() , how='left', on='SK_ID_CURR')
 del prev, prev_apps_avg
gc.collect()<load_from_csv>
def encode_data(df):
df['idhogar'] = LabelEncoder().fit_transform(df['idhogar'])
 def feature_importance(forest, X_train, display_results=True):
ranked_list = []
zero_features = []
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]
if display_results:
print("Feature ranking:")
 for f in range(X_train.shape[1]):
if display_results:
print("%d.feature %d(%f)" %(f + 1, indices[f], importances[indices[f]])+ " - " + X_train.columns[indices[f]])
 ranked_list.append(X_train.columns[indices[f]])
 if importances[indices[f]] == 0.0:
zero_features.append(X_train.columns[indices[f]])
 return ranked_list, zero_features
pos = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows = None )<categorify>
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop(['Id'], axis=1, inplace=True)
 return df
pos_cats = _get_categorical_features(pos)
 pos = _get_dummies(pos, pos_cats )<merge>
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df
pos_count = pos[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
pos['SK_ID_PREV'] = pos['SK_ID_CURR'].map(pos_count['SK_ID_PREV'])
 pos_avg = pos.groupby('SK_ID_CURR' ).mean()
df = df.merge(right=pos_avg.reset_index() , how='left', on='SK_ID_CURR')
 del pos, pos_avg
gc.collect()<load_from_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_ids = test.Id
ins = pd.read_csv('.. /input/installments_payments.csv', nrows = None )<categorify>
def process_df(df_):
encode_data(df_)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )
ins_cats = _get_categorical_features(ins)
 ins = _get_dummies(ins, ins_cats )<feature_engineering>
train['dependency'] = np.sqrt(train['SQBdependency'])
 test['dependency'] = np.sqrt(test['SQBdependency'])
 train.loc[train['edjefa'] == "no", "edjefa"] = 0
train.loc[train['edjefe'] == "no", "edjefe"] = 0
test.loc[test['edjefa'] == "no", "edjefa"] = 0
test.loc[test['edjefe'] == "no", "edjefe"] = 0
train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "edjefa"] = train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "escolari"]
train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "edjefe"] = train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "edjefa"] = test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "edjefe"] = test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "escolari"]
train.loc[train['edjefa'] == "yes", "edjefa"] = 4
train.loc[train['edjefe'] == "yes", "edjefe"] = 4
test.loc[test['edjefa'] == "yes", "edjefa"] = 4
test.loc[test['edjefe'] == "yes", "edjefe"] = 4
train['edjefe'] = train['edjefe'].astype("int")
 train['edjefa'] = train['edjefa'].astype("int")
 test['edjefe'] = test['edjefe'].astype("int")
 test['edjefa'] = test['edjefa'].astype("int")
 train['edjef'] = np.max(train[['edjefa','edjefe']], axis=1)
 test['edjef'] = np.max(test[['edjefa','edjefe']], axis=1)
 train['v2a1']=train['v2a1'].fillna(0)
 test['v2a1']=test['v2a1'].fillna(0)
 test['v18q1']=test['v18q1'].fillna(0)
 train['v18q1']=train['v18q1'].fillna(0)
 train['rez_esc']=train['rez_esc'].fillna(0)
 test['rez_esc']=test['rez_esc'].fillna(0)
 train.loc[train.meaneduc.isnull() , "meaneduc"] = 0
train.loc[train.SQBmeaned.isnull() , "SQBmeaned"] = 0
test.loc[test.meaneduc.isnull() , "meaneduc"] = 0
test.loc[test.SQBmeaned.isnull() , "SQBmeaned"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "v14a"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "sanitario1"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "v14a"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "sanitario1"] = 0
ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0 )<merge>
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_
cnt_inst = ins[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
ins['SK_ID_PREV'] = ins['SK_ID_CURR'].map(cnt_inst['SK_ID_PREV'])
 avg_inst = ins.groupby('SK_ID_CURR' ).mean()
avg_inst.columns = ['i_' + f_ for f_ in avg_inst.columns]
df = df.merge(right=avg_inst.reset_index() , how='left', on='SK_ID_CURR')
 del ins, avg_inst
gc.collect()<load_from_csv>
train, test = train_test_apply_func(train, test, convert_OHE2LE )
cc = pd.read_csv('.. /input/credit_card_balance.csv', nrows = None )<categorify>
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )
ccbal_cats = _get_categorical_features(cc)
 credit_card_balance = _get_dummies(cc, ccbal_cats )<merge>
train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0)
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
extract_features(train)
 extract_features(test )
nb_prevs = credit_card_balance[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
credit_card_balance['SK_ID_PREV'] = credit_card_balance['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 avg_cc_bal = credit_card_balance.groupby('SK_ID_CURR' ).mean()
avg_cc_bal.columns = ['cc_bal_' + f_ for f_ in avg_cc_bal.columns]
df = df.merge(right=avg_cc_bal.reset_index() , how='left', on='SK_ID_CURR')
 del cc, avg_cc_bal
gc.collect()<import_modules>
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'female', ]
instlevel_cols = [s for s in train.columns.tolist() if 'instlevel' in s]
needless_cols.extend(instlevel_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )
from lightgbm import LGBMClassifier
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.model_selection import KFold, StratifiedKFold<train_model>
def split_data(train, y, sample_weight=None, households=None, test_percentage=0.20, seed=None):
train2 = train.copy()
cv_hhs = np.random.choice(households, size=int(len(households)* test_percentage), replace=False)
 cv_idx = np.isin(households, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
if sample_weight is not None:
y_train_weights = sample_weight[~cv_idx]
return X_train, y_train, X_test, y_test, y_train_weights
return X_train, y_train, X_test, y_test
train_df = df[df['TARGET'].notnull() ]
test_df = df[df['TARGET'].isnull() ]
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))<choose_model_class>
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1)
 np.random.seed(seed=None)
 train2 = X.copy()
train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.15), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
X_train = train2
y_train = y
train_households = X_train.idhogar
folds = KFold(n_splits=2, shuffle=True, random_state=1001 )<define_variables>
y_train_weights = class_weight.compute_sample_weight('balanced', y_train, indices=None )
oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]<split>
extra_drop_features = [
'agg18_estadocivil1_MEAN',
'agg18_estadocivil6_COUNT',
'agg18_estadocivil7_COUNT',
'agg18_parentesco10_COUNT',
'agg18_parentesco11_COUNT',
'agg18_parentesco12_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco2_COUNT',
'agg18_parentesco3_COUNT',
'agg18_parentesco4_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco6_COUNT',
'agg18_parentesco7_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco9_COUNT',
'geo_elimbasu_LE_4',
'geo_energcocinar_LE_1',
'geo_energcocinar_LE_2',
'geo_epared_LE_0',
'geo_hogar_mayor',
'geo_manual_elec_LE_2',
'geo_pared_LE_3',
'geo_pared_LE_4',
'geo_pared_LE_5',
'geo_pared_LE_6',
'num_over_18',
'parentesco_LE',
'rez_esc']
for train_indices, valid_indices in folds.split(train_df[feats],train_df['TARGET']):
train_features, train_labels = train_df[feats].iloc[train_indices], train_df['TARGET'].iloc[train_indices]
valid_features, valid_labels = train_df[feats].iloc[valid_indices], train_df['TARGET'].iloc[valid_indices]
model = LGBMClassifier(n_estimators=5000, objective = 'binary',
class_weight = 'balanced', learning_rate = 0.05,
reg_alpha = 0.1, reg_lambda = 0.1,
subsample = 0.8, n_jobs = -1, random_state = 50)
 model.fit(train_features, train_labels, eval_metric = 'auc',
eval_set = [(valid_features, valid_labels),(train_features, train_labels)],
eval_names = ['valid', 'train'],
early_stopping_rounds = 100, verbose = 200)
 oof_preds[valid_indices] = model.predict_proba(valid_features, num_iteration=model.best_iteration_)[:, 1]
sub_preds += model.predict_proba(test_df[feats], num_iteration=model.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = model.feature_importances_
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 del model, train_features, train_labels, valid_features, valid_labels
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))<save_to_csv>
xgb_drop_cols = extra_drop_features + ["idhogar", 'parentesco1']
test_df['TARGET'] = sub_preds
test_df[['SK_ID_CURR', 'TARGET']].to_csv('submission.csv', index= False )<set_options>
opt_parameters = {'max_depth':35, 'eta':0.1, 'silent':0, 'objective':'multi:softmax', 'min_child_weight': 1, 'num_class': 4, 'gamma': 2.0, 'colsample_bylevel': 0.9, 'subsample': 0.84, 'colsample_bytree': 0.88, 'reg_lambda': 0.40 }
opt_parameters = {'max_depth':35, 'eta':0.15, 'silent':1, 'objective':'multi:softmax', 'min_child_weight': 2, 'num_class': 4, 'gamma': 2.5, 'colsample_bylevel': 1, 'subsample': 0.95, 'colsample_bytree': 0.85, 'reg_lambda': 0.35 }
def evaluate_macroF1_lgb(predictions, truth):
pred_labels = predictions.argmax(axis=1)
 truth = truth.get_label()
f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', 1-f1)
 fit_params={"early_stopping_rounds":500,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'verbose': False,
}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 50
%matplotlib inline<feature_engineering>
np.random.seed(100)
 def _parallel_fit_estimator(estimator1, X, y, sample_weight=None, threshold=True, **fit_params):
estimator = clone(estimator1)
 if sample_weight is not None:
X_train, y_train, X_test, y_test, y_train_weight = split_data(X, y, sample_weight, households=train_households)
 else:
X_train, y_train, X_test, y_test = split_data(X, y, None, households=train_households)
 fit_params["eval_set"] = [(X_test,y_test)]
if sample_weight is not None:
if isinstance(estimator1, ExtraTreesClassifier)or isinstance(estimator1, RandomForestClassifier):
estimator.fit(X_train, y_train)
 else:
_ = estimator.fit(X_train, y_train, sample_weight=y_train_weight, **fit_params)
 else:
if isinstance(estimator1, ExtraTreesClassifier)or isinstance(estimator1, RandomForestClassifier):
estimator.fit(X_train, y_train)
 else:
_ = estimator.fit(X_train, y_train, **fit_params)
 if not isinstance(estimator1, ExtraTreesClassifier)and not isinstance(estimator1, RandomForestClassifier)and not isinstance(estimator1, xgb.XGBClassifier):
best_cv_round = np.argmax(estimator.evals_result_['validation_0']['mlogloss'])
 best_cv = np.max(estimator.evals_result_['validation_0']['mlogloss'])
 best_train = estimator.evals_result_['train']['macroF1'][best_cv_round]
else:
best_train = f1_score(y_train, estimator.predict(X_train), average="macro")
 best_cv = f1_score(y_test, estimator.predict(X_test), average="macro")
 print("Train F1:", best_train)
 print("Test F1:", best_cv)
 if threshold:
if(( best_cv > 0.37)and(best_train > 0.75)) or(( best_cv > 0.44)and(best_train > 0.65)) :
return estimator
else:
print("Unacceptable!!! Trying again...")
 return _parallel_fit_estimator(estimator1, X, y, sample_weight=sample_weight, **fit_params)
 else:
return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, threshold=True, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, threshold=threshold, **fit_params)
 for clf in clfs if clf is not None)
 return self
path1=".. /input/"
data_files=list(os.listdir(path1))
df_files=pd.DataFrame(data_files,columns=['File_Name'])
 df_files['Size_in_MB']=df_files.File_Name.apply(lambda x:round(os.stat(path1+x ).st_size/(1024*1024),2))
df_files<feature_engineering>
clfs = []
for i in range(15):
clf = xgb.XGBClassifier(random_state=217+i, n_estimators=300, learning_rate=0.15, n_jobs=4, **opt_parameters)
 clfs.append(( 'xgb{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del(clfs)
 _ = vc.fit(X_train.drop(xgb_drop_cols, axis=1), y_train, sample_weight=y_train_weights, threshold=False, **fit_params)
 clf_final = vc.estimators_[0]
def feature_summary(df_fa):
print('DataFrame shape')
 print('rows:',df_fa.shape[0])
 print('cols:',df_fa.shape[1])
 col_list=['Null','Unique_Count','Data_type','Max/Min','Mean','Median','Std','Skewness','Sample_values']
df=pd.DataFrame(index=df_fa.columns,columns=col_list)
 df['Null']=list([len(df_fa[col][df_fa[col].isnull() ])for i,col in enumerate(df_fa.columns)])
 df['Unique_Count']=list([len(df_fa[col].unique())for i,col in enumerate(df_fa.columns)])
 df['Data_type']=list([df_fa[col].dtype for i,col in enumerate(df_fa.columns)])
 for i,col in enumerate(df_fa.columns):
if 'float' in str(df_fa[col].dtype)or 'int' in str(df_fa[col].dtype):
df.at[col,'Max/Min']=str(round(df_fa[col].max() ,2)) +'/'+str(round(df_fa[col].min() ,2))
df.at[col,'Mean']=df_fa[col].mean()
df.at[col,'Median']=df_fa[col].median()
df.at[col,'Std']=df_fa[col].std()
df.at[col,'Skewness']=df_fa[col].skew()
df.at[col,'Sample_values']=list(df_fa[col].unique())
 return(df.fillna('-'))
def drop_corr_col(df_corr):
upper = df_corr.where(np.triu(np.ones(df_corr.shape),
k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.999)]
return(to_drop)
 def cnt_unique(df):
return(len(df.unique()))<load_from_csv>
global_score = f1_score(y_test, clf_final.predict(X_test.drop(xgb_drop_cols, axis=1)) , average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test.drop(xgb_drop_cols, axis=1)) , average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test.drop(xgb_drop_cols, axis=1)) , average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))
%%time
train=pd.read_csv(path1+'application_train.csv')
 test=pd.read_csv(path1+'application_test.csv' )<concatenate>
ranked_features = feature_importance(clf_final, X_train.drop(xgb_drop_cols, axis=1))
print('Train shape:',train.shape,'Test shape:',test.shape)
 df_comb=pd.concat([train.drop(['TARGET'],axis=1),test],ignore_index=True)
 df_comb_fs=feature_summary(df_comb)
 cat_features=df_comb_fs[df_comb_fs.Data_type=='object'].index
for col in cat_features:
df_comb[col]=df_comb[col].apply(lambda x: str(x ).replace(" ","_"))
print('categorical features',len(cat_features))<categorify>
et_drop_cols = ['agg18_age_MAX', 'agg18_age_MEAN', 'agg18_age_MIN', 'agg18_dis_MEAN',
'agg18_escolari_MAX', 'agg18_escolari_MEAN', 'agg18_escolari_MIN',
'agg18_estadocivil1_COUNT', 'agg18_estadocivil1_MEAN',
'agg18_estadocivil2_COUNT', 'agg18_estadocivil2_MEAN',
'agg18_estadocivil3_COUNT', 'agg18_estadocivil3_MEAN',
'agg18_estadocivil4_COUNT', 'agg18_estadocivil4_MEAN',
'agg18_estadocivil5_COUNT', 'agg18_estadocivil5_MEAN',
'agg18_estadocivil6_COUNT', 'agg18_estadocivil6_MEAN',
'agg18_estadocivil7_COUNT', 'agg18_estadocivil7_MEAN',
'agg18_parentesco10_COUNT', 'agg18_parentesco10_MEAN',
'agg18_parentesco11_COUNT', 'agg18_parentesco11_MEAN',
'agg18_parentesco12_COUNT', 'agg18_parentesco12_MEAN',
'agg18_parentesco1_COUNT', 'agg18_parentesco1_MEAN',
'agg18_parentesco2_COUNT', 'agg18_parentesco2_MEAN',
'agg18_parentesco3_COUNT', 'agg18_parentesco3_MEAN',
'agg18_parentesco4_COUNT', 'agg18_parentesco4_MEAN',
'agg18_parentesco5_COUNT', 'agg18_parentesco5_MEAN',
'agg18_parentesco6_COUNT', 'agg18_parentesco6_MEAN',
'agg18_parentesco7_COUNT', 'agg18_parentesco7_MEAN',
'agg18_parentesco8_COUNT', 'agg18_parentesco8_MEAN',
'agg18_parentesco9_COUNT', 'agg18_parentesco9_MEAN']
et_drop_cols.extend(["idhogar", "parentesco1", 'fe_rent_per_person', 'fe_rent_per_room',
'fe_tablet_adult_density', 'fe_tablet_density'] )
cat_train=pd.DataFrame()
for col in cat_features:
dummy=pd.get_dummies(df_comb[col],prefix='DUM_'+col)
 cat_train=pd.concat([cat_train,dummy],axis=1)
 display(cat_train.head())
 print('Newly created dummy columns:',cat_train.shape )<drop_column>
ets = []
for i in range(10):
rf = RandomForestClassifier(max_depth=None, random_state=217+i, n_jobs=4, n_estimators=700, min_impurity_decrease=1e-3, min_samples_leaf=2, verbose=0, class_weight="balanced")
 ets.append(( 'rf{}'.format(i), rf))
vc2 = VotingClassifierLGBM(ets, voting='soft')
 _ = vc2.fit(X_train.drop(et_drop_cols, axis=1), y_train, threshold=False )
cat_train.drop(['DUM_CODE_GENDER_XNA','DUM_NAME_INCOME_TYPE_Businessman','DUM_NAME_INCOME_TYPE_Maternity_leave','DUM_NAME_INCOME_TYPE_Student',
'DUM_NAME_INCOME_TYPE_Unemployed','DUM_NAME_FAMILY_STATUS_Unknown','DUM_OCCUPATION_TYPE_Security_staff','DUM_FONDKAPREMONT_MODE_nan',
'DUM_OCCUPATION_TYPE_IT_staff','DUM_ORGANIZATION_TYPE_Police','DUM_ORGANIZATION_TYPE_Telecom','DUM_ORGANIZATION_TYPE_Business_Entity_Type_1'],
axis=1,inplace=True)
 print('Categorical feature count after dropping irrelevant features',cat_train.shape )<feature_engineering>
vc2.voting = 'soft'
global_rf_score_soft = f1_score(y_test, vc2.predict(X_test.drop(et_drop_cols, axis=1)) , average='macro')
 vc2.voting = 'hard'
global_rf_score_hard = f1_score(y_test, vc2.predict(X_test.drop(et_drop_cols, axis=1)) , average='macro')
 print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_rf_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_rf_score_hard))
df_final['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)
 df_final['CALC_PERC_DAYS_EMPLOYED'] = df_final['DAYS_EMPLOYED'] / df_final['DAYS_BIRTH']
df_final['CALC_PERC_INCOME_CREDIT'] = df_final['AMT_INCOME_TOTAL'] /df_final['AMT_CREDIT']
df_final['CALC_INCOME_PER_PERSON'] = df_final['AMT_INCOME_TOTAL'] / df_final['CNT_FAM_MEMBERS']
df_final['CALC_ANNUITY_INCOME_PERC'] = df_final['AMT_ANNUITY'] / df_final['AMT_INCOME_TOTAL']
df_final['CALC_PAYMENT_RATE'] = df_final['AMT_ANNUITY'] / df_final['AMT_CREDIT']
df_final['CALC_GOODS_PRICE_PER']=df_final['AMT_GOODS_PRICE']/df_final['AMT_CREDIT']
df_final['CALC_PERC_CHILDREN']=df_final['CNT_CHILDREN']/df_final['CNT_FAM_MEMBERS']
df_final['CALC_RATIO_CAR_TO_BRITH']=df_final['OWN_CAR_AGE'] / df_final['DAYS_BIRTH']
df_final['CALC_RATIO_CAR_TO_EMPLOY'] = df_final['OWN_CAR_AGE'] / df_final['DAYS_EMPLOYED']
df_final['CALC_INCOME_PER_CHILD'] = df_final['AMT_INCOME_TOTAL'] /(1 + df_final['CNT_CHILDREN'])
 df_final['CALC_INCOME_PER_PERSON'] = df_final['AMT_INCOME_TOTAL'] / df_final['CNT_FAM_MEMBERS']
df_final['CALC_RATIO_PHONE_TO_BIRTH'] = df_final['DAYS_LAST_PHONE_CHANGE'] / df_final['DAYS_BIRTH']
df_final['CALC_RATIO_PHONE_TO_EMPLOY'] = df_final['DAYS_LAST_PHONE_CHANGE'] / df_final['DAYS_EMPLOYED']<drop_column>
vc2.voting = 'soft'
global_rf_score_soft = f1_score(y_test, vc2.predict(X_test.drop(et_drop_cols, axis=1)) , average='macro')
 vc2.voting = 'hard'
global_rf_score_hard = f1_score(y_test, vc2.predict(X_test.drop(et_drop_cols, axis=1)) , average='macro')
 print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_rf_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_rf_score_hard))
df_final.drop(['FLAG_MOBIL','FLAG_DOCUMENT_2','FLAG_DOCUMENT_4','FLAG_DOCUMENT_10','FLAG_DOCUMENT_12'],axis=1,inplace=True )<drop_column>
def combine_voters(data, weights=[0.5, 0.5]):
vc.voting="soft"
vc1_probs = vc.predict_proba(data.drop(xgb_drop_cols, axis=1))
vc2.voting="soft"
vc2_probs = vc2.predict_proba(data.drop(et_drop_cols, axis=1))
final_vote =(vc1_probs * weights[0])+(vc2_probs * weights[1])
 predictions = np.argmax(final_vote, axis=1)
 return predictions
del dummy
gc.collect()<load_from_csv>
combo_preds = combine_voters(X_test, weights=[0.5, 0.5])
 global_combo_score_soft = f1_score(y_test, combo_preds, average='macro')
 global_combo_score_soft
%%time
bur=pd.read_csv(path1+'bureau.csv')
 print('bureau set reading complete...')
 bur_bal=pd.read_csv(path1+'bureau_balance.csv')
 print('bureau balance set reading complete...' )<categorify>
combo_preds = combine_voters(X_test, weights=[0.4, 0.6])
 global_combo_score_soft= f1_score(y_test, combo_preds, average='macro')
 global_combo_score_soft
%%time
for col in ['CREDIT_CURRENCY','CREDIT_TYPE','CREDIT_ACTIVE']:
bur[col]=bur[col].apply(lambda x: str(x ).replace(" ","_"))
dummy=pd.DataFrame()
for col in ['CREDIT_CURRENCY','CREDIT_TYPE','CREDIT_ACTIVE']:
dummy=pd.concat([dummy,pd.get_dummies(bur[col],prefix='DUM_'+col)],axis=1 )<concatenate>
combo_preds = combine_voters(X_test, weights=[0.6, 0.4])
 global_combo_score_soft = f1_score(y_test, combo_preds, average='macro')
 global_combo_score_soft
bur_f=pd.concat([bur.drop(['CREDIT_CURRENCY','CREDIT_TYPE','CREDIT_ACTIVE'],axis=1),dummy],axis=1 )<feature_engineering>
y_subm = pd.DataFrame()
y_subm['Id'] = test_ids
bur_f['CALC_PER_CREDIT_MAX_OVERDUE']=bur_f['AMT_CREDIT_MAX_OVERDUE']/bur_f['AMT_CREDIT_SUM']
bur_f['CALC_PER_CREDIT_SUM_DEBT']=bur_f['AMT_CREDIT_SUM_DEBT']/bur_f['AMT_CREDIT_SUM']
bur_f['CALC_PER_CREDIT_SUM_LIMIT']=bur_f['AMT_CREDIT_SUM_LIMIT']/bur_f['AMT_CREDIT_SUM']
bur_f['CALC_PER_CREDIT_SUM_OVERDUE']=bur_f['AMT_CREDIT_SUM_OVERDUE']/bur_f['AMT_CREDIT_SUM']
bur_f['CALC_PER_ANNUITY']=bur_f['AMT_ANNUITY']/bur_f['AMT_CREDIT_SUM']
bur_f['CALC_CREDIT_LIMIT_CROSSED']=bur_f['AMT_CREDIT_SUM_LIMIT']-bur_f['AMT_CREDIT_SUM']
bur_f['CALC_CREDIT_PER_DAY']=bur_f['AMT_CREDIT_SUM']/bur_f['DAYS_CREDIT_ENDDATE'].abs()
bur_f['CALC_CREDIT_CLOSED']=(bur_f['DAYS_ENDDATE_FACT'] < 0 ).astype(int )<drop_column>
vc.voting = 'soft'
y_subm_lgb = y_subm.copy(deep=True)
 y_subm_lgb['Target'] = vc.predict(test.drop(xgb_drop_cols, axis=1)) + 1
vc2.voting = 'soft'
y_subm_rf = y_subm.copy(deep=True)
 y_subm_rf['Target'] = vc2.predict(test.drop(et_drop_cols, axis=1)) + 1
y_subm_ens = y_subm.copy(deep=True)
 y_subm_ens['Target'] = combine_voters(test)+ 1
<feature_engineering><EOS> 
now = datetime.now()
sub_file_lgb = 'submission_soft_XGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_rf = 'submission_soft_RF_{:.4f}_{}.csv'.format(global_rf_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_ens = 'submission_ens_{:.4f}_{}.csv'.format(global_combo_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm_lgb.to_csv(sub_file_lgb, index=False)
 y_subm_rf.to_csv(sub_file_rf, index=False)
 y_subm_ens.to_csv(sub_file_ens, index=False )