Kaggle competition: Costa Rican Household Poverty Level Prediction
def create_eval_nq_dataset(file_path, seq_length):

 name_to_features = {
'unique_ids': tf.io.FixedLenFeature([], tf.int64),
'input_ids': tf.io.FixedLenFeature([seq_length], tf.int64),
'input_mask': tf.io.FixedLenFeature([seq_length], tf.int64),
'segment_ids': tf.io.FixedLenFeature([seq_length], tf.int64),
'token_map': tf.io.FixedLenFeature([seq_length], tf.int64),
}
input_fn = input_pipeline.file_based_input_fn_builder(file_path, name_to_features)
 return input_fn()
def post_process() :
raw_prediction_file = f'sub_{CKPT}.p'
predict_file = f'{INPUT_DIR}/simplified-nq-test.jsonl'
predict_tfrecord = 'simplified-nq-test.tfrecords'
with open(raw_prediction_file,'rb')as f:
all_results = pickle.load(f)
 candidates_dict = read_candidates(predict_file)
 eval_features = [
r for r in create_eval_nq_dataset(predict_tfrecord,FLAGS.max_seq_length)
 ]
print(len(eval_features),len(all_results))
results = [r._asdict() for c,r in enumerate(all_results)if c<len(eval_features)]
nq_pred_dict,nbest_summary_dict = compute_pred_dict(candidates_dict, eval_features,results)
 output = {"predictions": list(nq_pred_dict.values())}
return output<define_variables>
%matplotlib inline
plt.style.use('fivethirtyeight')
 plt.rcParams['font.size'] = 18
plt.rcParams['patch.edgecolor'] = 'k'
%%time
output = post_process()<string_transform>
pd.options.display.max_columns = 150
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
def create_short_answer(entry):
if entry["short_answers_score"] < short_thr:
return ""
answer = []
for short_answer in entry["short_answers"]:
if short_answer["start_token"] > -1:
answer.append(str(short_answer["start_token"])+ ":" + str(short_answer["end_token"]))
if entry["yes_no_answer"] != "NONE":
answer.append(entry["yes_no_answer"])
 return " ".join(answer)
 def create_long_answer(entry):
if entry["long_answer_score"] < long_thr:
return ""
answer = []
if entry["long_answer"]["start_token"] > -1:
answer.append(str(entry["long_answer"]["start_token"])+ ":" + str(entry["long_answer"]["end_token"]))
return " ".join(answer)
 def write_sub(test_answers_df):
path = f'{INPUT_DIR}/sample_submission.csv'
test_answers_df["long_answer_score"] = test_answers_df["predictions"].apply(lambda q: q["long_answer_score"])
 test_answers_df["short_answer_score"] = test_answers_df["predictions"].apply(lambda q: q["short_answers_score"])
 test_answers_df["long_answer"] = test_answers_df["predictions"].apply(create_long_answer)
 test_answers_df["short_answer"] = test_answers_df["predictions"].apply(create_short_answer)
 test_answers_df["example_id"] = test_answers_df["predictions"].apply(lambda q: str(q["example_id"]))
long_answers = dict(zip(test_answers_df["example_id"], test_answers_df["long_answer"]))
short_answers = dict(zip(test_answers_df["example_id"], test_answers_df["short_answer"]))
sample_submission = pd.read_csv(path)
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings
sample_submission.to_csv('submission.csv',index=False)
<prepare_output>
maps={'yes':1,'no':0}
for df in [train,test]:
df['dependency'] = df['dependency'].replace(maps ).astype(np.float64)
 df['edjefa'] = df['edjefa'].replace(maps ).astype(np.float64)
 df['edjefe'] = df['edjefe'].replace(maps ).astype(np.float64 )
%%time
df = pd.DataFrame(output)
 long_thr = 2.7368
short_thr = 6.8347
write_sub(df )<count_values>
test['Target'] = np.nan
data = train.append(test, ignore_index = True )
pd.read_csv('submission.csv' ).count()<import_modules>
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )
FLAGS.__dict__<set_options>
households_leader = train.groupby('idhogar')['parentesco1'].sum()
households_no_head = train.loc[train['idhogar'].isin(households_leader[households_leader == 0].index), :]
print('There are {} households without a head.'.format(households_no_head['idhogar'].nunique()))
plt.rcParams['font.size'] = 18
%matplotlib inline
N_FOLDS = 5
MAX_EVALS = 5
print(os.listdir(".. /input")) 
<load_from_csv>
households_no_head_equal = households_no_head.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 print('{} Households with no head have different labels.'.format(sum(households_no_head_equal == False)) )
features = pd.read_csv('.. /input/home-credit-default-risk/application_train.csv')
 features = features.sample(n = 16000, random_state = 42)
 features = features.select_dtypes('number')
 labels = np.array(features['TARGET'].astype(np.int32)).reshape(( -1,))
features = features.drop(columns = ['TARGET', 'SK_ID_CURR'])
 train_features, test_features, train_labels, test_labels = train_test_split(features, labels, test_size = 6000, random_state = 42)
 print('Train shape: ', train_features.shape)
 print('Test shape: ', test_features.shape)
 train_features.head()<create_dataframe>
for household in not_equal.index:
true_target=int(train[(train['idhogar']==household)&(train['parentesco1']==1.0)]["Target"])
 train.loc[train["idhogar"]==household,"Target"]=true_target
all_equal=train.groupby("idhogar")["Target"].apply(lambda x:x.nunique() ==1)
 not_equal=all_equal[all_equal !=True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )
model = lgb.LGBMClassifier(random_state = 50)
 train_set = lgb.Dataset(train_features, label = train_labels)
 test_set = lgb.Dataset(test_features, label = test_labels )<find_best_params>
missing=pd.DataFrame(data.isnull().sum())
 missing=missing.rename(columns={0:'total'})
 missing["percentage"]=missing['total']/len(data)
 missing.sort_values('percentage',ascending=False ).head(10 ).drop("Target" )
hyperparameters = model.get_params()
del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round=10000, nfold = N_FOLDS, metrics = 'auc', early_stopping_rounds=100, verbose_eval=False, seed = 42)
 best = cv_results['auc-mean'][-1]
best_std = cv_results['auc-stdv'][-1]
print('The maximium ROC AUC in cross validation was {:.5f} with std of {:.5f}.'.format(best, best_std))
print('The ideal number of iterations was {}.'.format(len(cv_results['auc-mean'])) )<compute_train_metric>
heads.groupby('v18q')['v18q1'].apply(lambda x:x.isnull().sum() )
model.n_estimators = len(cv_results['auc-mean'])
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
baseline_auc = roc_auc_score(test_labels, preds)
 print('The baseline model scores {:.5f} ROC-AUC on the test set.'.format(baseline_auc))<choose_model_class>
data['v18q1'] = data['v18q1'].fillna(0)
def objective(hyperparameters):

 global ITERATION
ITERATION += 1
if 'n_estimators' in hyperparameters:
del hyperparameters['n_estimators']
subsample = hyperparameters['boosting_type'].get('subsample', 1.0)
 hyperparameters['boosting_type'] = hyperparameters['boosting_type']['boosting_type']
hyperparameters['subsample'] = subsample
for parameter_name in ['num_leaves', 'subsample_for_bin', 'min_child_samples']:
hyperparameters[parameter_name] = int(hyperparameters[parameter_name])
 start = timer()
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round = 1000, nfold = N_FOLDS, early_stopping_rounds = 100, metrics = 'auc', seed = 50)
 run_time = timer() - start
best_score = cv_results['auc-mean'][-1]
loss = 1-best_score
n_estimators = len(cv_results['auc-mean'])
 hyperparameters['n_estimators'] = n_estimators
of_connection = open(OUT_FILE, 'a')
 writer = csv.writer(of_connection)
 writer.writerow([loss, hyperparameters, ITERATION, run_time, best_score])
 of_connection.close()
return{'loss': loss, 'hyperparameters': hyperparameters, 'iteration': ITERATION, 'train_time': run_time, 'status': STATUS_OK}<import_modules>
data.loc[data['tipovivi1']==1,'v2a1']=0
data["v2a1-missing"]=data["v2a1"].isnull()
data["v2a1-missing"].value_counts()
from hyperopt import hp
from hyperopt.pyll.stochastic import sample<init_hyperparams>
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()
learning_rate = {'learning_rate': hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)) }<load_pretrained>
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5
boosting_type = {'boosting_type': hp.choice('boosting_type', [{'boosting_type': 'gbdt', 'subsample': hp.uniform('subsample', 0.5, 1)},
{'boosting_type': 'dart', 'subsample': hp.uniform('subsample', 0.5, 1)},
{'boosting_type': 'goss', 'subsample':1.0}])}
hyperparams = sample(boosting_type)
 hyperparams<init_hyperparams>
es = ft.EntitySet(id = 'Target')
 es.entity_from_dataframe(entity_id = 'costa rica', dataframe = data, index = 'Id' )
subsample = hyperparams['boosting_type'].get('subsample', 1.0)
 hyperparams['boosting_type'] = hyperparams['boosting_type']['boosting_type']
hyperparams['subsample'] = subsample
hyperparams<categorify>
es.normalize_entity(base_entity_id='costa rica', new_entity_id='households', index = 'idhogar',
additional_variables = ['v2a1', 'hhsize'])
space = {
'boosting_type': hp.choice('boosting_type',
[{'boosting_type': 'gbdt', 'subsample': hp.uniform('gdbt_subsample', 0.5, 1)},
{'boosting_type': 'dart', 'subsample': hp.uniform('dart_subsample', 0.5, 1)},
{'boosting_type': 'goss', 'subsample': 1.0}]),
'num_leaves': hp.quniform('num_leaves', 20, 150, 1),
'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(0.5)) ,
'subsample_for_bin': hp.quniform('subsample_for_bin', 20000, 300000, 20000),
'min_child_samples': hp.quniform('min_child_samples', 20, 500, 5),
'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),
'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),
'colsample_bytree': hp.uniform('colsample_by_tree', 0.6, 1.0),
'is_unbalance': hp.choice('is_unbalance', [True, False]),
}<feature_engineering>
id_ = ['Id', 'idhogar', 'Target']
x = sample(space)
 subsample = x['boosting_type'].get('subsample', 1.0)
 x['boosting_type'] = x['boosting_type']['boosting_type']
x['subsample'] = subsample
x<save_to_csv>
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']
OUT_FILE = 'bayes_test.csv'
of_connection = open(OUT_FILE, 'w')
 writer = csv.writer(of_connection)
 ITERATION = 0
headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score']
writer.writerow(headers)
 of_connection.close()
results = objective(sample(space))
print('The cross validation loss = {:.5f}'.format(results['loss']))
print('The optimal number of estimators was {}.'.format(results['hyperparameters']['n_estimators'])) 
<find_best_params>
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']
tpe_algorithm = tpe.suggest<choose_model_class>
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']
trials = Trials()<save_to_csv>
x = ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats: ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable: ', len(x)== data.shape[1] )
OUT_FILE = 'bayes_test.csv'
of_connection = open(OUT_FILE, 'w')
 writer = csv.writer(of_connection)
 ITERATION = 0
headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score']
writer.writerow(headers)
 of_connection.close()<import_modules>
upper=corr_matrix.where(np.triu(np.ones(corr_matrix.shape),k=1 ).astype(np.bool))
to_drop=[column for column in upper.columns if any(upper[column]>0.95)]
to_drop
from hyperopt import fmin<find_best_params>
corr_matrix.loc[corr_matrix['tamhog'].abs() > 0.9, corr_matrix['tamhog'].abs() > 0.9]
global ITERATION
ITERATION = 0
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials, max_evals = MAX_EVALS)
 best<find_best_params>
heads = heads.drop(columns = ['tamhog', 'hogar_total', 'r4t3'] )
trials_dict = sorted(trials.results, key = lambda x: x['loss'])
 trials_dict[:1]<load_from_csv>
heads['hhsize-diff']=heads['tamviv']-heads['hhsize']
results = pd.read_csv(OUT_FILE )<prepare_output>
elec = []
for i, row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull()
def evaluate(results, name):

 new_results = results.copy()
new_results['hyperparameters'] = new_results['hyperparameters'].map(ast.literal_eval)
 new_results = new_results.sort_values('score', ascending = False ).reset_index(drop = True)
 print('The highest cross validation score from {} was {:.5f} found on iteration {}.'.format(name, new_results.loc[0, 'score'], new_results.loc[0, 'iteration']))
hyperparameters = new_results.loc[0, 'hyperparameters']
model = lgb.LGBMClassifier(**hyperparameters)
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
print('ROC AUC from {} on test data = {:.5f}.'.format(name, roc_auc_score(test_labels, preds)))
 hyp_df = pd.DataFrame(columns = list(new_results.loc[0, 'hyperparameters'].keys()))
for i, hyp in enumerate(new_results['hyperparameters']):
hyp_df = hyp_df.append(pd.DataFrame(hyp, index = [0]), ignore_index = True)
 hyp_df['iteration'] = new_results['iteration']
hyp_df['score'] = new_results['score']
return hyp_df<prepare_output>
heads = heads.drop(columns = ['noelec', 'coopele', 'public', 'planpri'] )
bayes_results = evaluate(results, name = 'Bayesian')
 bayes_results<find_best_params>
heads=heads.drop(columns=['area2'])
 heads.groupby('area1')['Target'].value_counts(normalize=True )
MAX_EVALS = 10
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials, max_evals = MAX_EVALS)
<save_to_csv>
heads['walls']=np.argmax(np.array(heads[['epared1', 'epared2', 'epared3']]),axis=1 )
MAX_EVALS = 300
OUT_FILE = 'bayesian_trials_300.csv'
of_connection = open(OUT_FILE, 'w')
 writer = csv.writer(of_connection)
 headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score']
writer.writerow(headers)
 of_connection.close()
trials = Trials()
global ITERATION
ITERATION = 0
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials, max_evals = MAX_EVALS)
 trials_dict = sorted(trials.results, key = lambda x: x['loss'])
 print('Finished, best results')
 print(trials_dict[:1])
 with open('trials.json', 'w')as f:
f.write(json.dumps(trials_dict))<load_from_csv>
heads = heads.drop(columns = ['epared1', 'epared2', 'epared3'] )
bayes_results = pd.read_csv('.. /input/bayes-300/bayesian_trials_300.csv' ).sort_values('score', ascending = False ).reset_index()
bayes_params = evaluate(bayes_results, name = 'Bayesian' )<prepare_output>
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 heads = heads.drop(columns = ['etecho1', 'etecho2', 'etecho3'])
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
 heads = heads.drop(columns = ['eviv1', 'eviv2', 'eviv3'] )
scores = pd.DataFrame({'ROC AUC': bayes_params['score'], 'iteration': bayes_params['iteration'], 'search': 'Bayesian'})
 scores['ROC AUC'] = scores['ROC AUC'].astype(np.float32)
 scores['iteration'] = scores['iteration'].astype(np.int32)
 scores.head()<find_best_params>
heads['walls+roof+floor'] = heads['walls'] + heads['roof'] + heads['floor']
best_bayes_params = bayes_params.iloc[bayes_params['score'].idxmax() , :].copy()<load_from_csv>
heads['warning'] = -1 *(heads['sanitario1'] + 
(heads['elec'] == 0)+
heads['pisonotiene'] +
heads['abastaguano'] + 
(heads['cielorazo'] == 0))
train = pd.read_csv('.. /input/data-will/simple_features_train.csv')
 test = pd.read_csv('.. /input/homecred-final-test/simple_features_test.csv')
 test_ids = test['SK_ID_CURR']
train_labels = np.array(train['TARGET'].astype(np.int32)).reshape(( -1,))
train = train.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test = test.drop(columns = ['SK_ID_CURR'])
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )<categorify>
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']
bayes_results['hyperparameters'] = bayes_results['hyperparameters'].map(ast.literal_eval)
<compute_train_metric>
ind = ind.drop(columns = 'male')
hyperparameters = dict(**bayes_results.loc[0, 'hyperparameters'])
 del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round=10000, early_stopping_rounds=100, metrics = 'auc', nfold = N_FOLDS)
 print('The cross validation score on the full dataset for Bayes Opt.= {:.5f} with std: {:.5f}.'.format(cv_results['auc-mean'][-1], cv_results['auc-stdv'][-1]))
print('Number of estimators = {}.'.format(len(cv_results['auc-mean'])) )<save_to_csv>
ind_agg = ind_agg.drop(columns = to_drop)
 ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg, on = 'idhogar', how = 'left')
 print('Final features shape: ', final.shape )
model = lgb.LGBMClassifier(n_estimators = len(cv_results['auc-mean']), **hyperparameters)
 model.fit(train, train_labels)
 preds = model.predict_proba(test)[:, 1]
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': preds})
 submission.to_csv('submission_bayesian_optimization2.csv', index = False )<import_modules>
final=final.drop(["v2a1","rooms",'v18q1','rez_esc','male',],axis=1 )
import pandas as pd
import numpy as np
from sklearn.metrics import roc_auc_score, precision_recall_curve, roc_curve, average_precision_score
from sklearn.model_selection import StratifiedKFold
from lightgbm import LGBMClassifier
from datetime import datetime
import matplotlib.pyplot as plt
import seaborn as sns
import gc<load_from_csv>
head_gender = ind.loc[ind['parentesco1'] == 1, ['idhogar', 'female']]
final = final.merge(head_gender, on = 'idhogar', how = 'left' ).rename(columns = {'female': 'female-head'} )
def build_model_input() :
buro_bal = pd.read_csv('.. /input/bureau_balance.csv')
 print('Buro bal shape : ', buro_bal.shape)
 print('transform to dummies')
 buro_bal = pd.concat(
 [buro_bal, pd.get_dummies(buro_bal.STATUS, prefix='buro_bal_status')],
axis=1 ).drop(
 'STATUS', axis=1)
 print('Counting buros')
 buro_counts = buro_bal[['SK_ID_BUREAU', 'MONTHS_BALANCE']].groupby('SK_ID_BUREAU' ).count()
buro_bal['buro_count'] = buro_bal['SK_ID_BUREAU'].map(buro_counts['MONTHS_BALANCE'])
 print('averaging buro bal')
 avg_buro_bal = buro_bal.groupby('SK_ID_BUREAU' ).mean()
avg_buro_bal.columns = ['avg_buro_' + f_ for f_ in avg_buro_bal.columns]
del buro_bal
gc.collect()
print('Read Bureau')
 buro = pd.read_csv('.. /input/bureau.csv')
 print('Go to dummies')
 buro_credit_active_dum = pd.get_dummies(buro.CREDIT_ACTIVE, prefix='ca_')
 buro_credit_currency_dum = pd.get_dummies(buro.CREDIT_CURRENCY, prefix='cu_')
 buro_credit_type_dum = pd.get_dummies(buro.CREDIT_TYPE, prefix='ty_')
 buro_full = pd.concat(
 [
buro, buro_credit_active_dum, buro_credit_currency_dum,
buro_credit_type_dum
],
axis=1)
 del buro_credit_active_dum, buro_credit_currency_dum, buro_credit_type_dum
gc.collect()
print('Merge with buro avg')
 buro_full = buro_full.merge(
 right=avg_buro_bal.reset_index() ,
how='left',
on='SK_ID_BUREAU',
suffixes=('', '_bur_bal'))
print('Counting buro per SK_ID_CURR')
 nb_bureau_per_curr = buro_full[['SK_ID_CURR', 'SK_ID_BUREAU']].groupby('SK_ID_CURR' ).count()
buro_full['SK_ID_BUREAU'] = buro_full['SK_ID_CURR'].map(nb_bureau_per_curr['SK_ID_BUREAU'])
 print('Averaging bureau')
 avg_buro = buro_full.groupby('SK_ID_CURR' ).mean()
print(avg_buro.head())
 del buro, buro_full
gc.collect()
print('Read prev')
 prev = pd.read_csv('.. /input/previous_application.csv')
 prev_cat_features = [
f_ for f_ in prev.columns if prev[f_].dtype == 'object'
]
print('Go to dummies')
 prev_dum = pd.DataFrame()
for f_ in prev_cat_features:
prev_dum = pd.concat(
 [prev_dum, pd.get_dummies(prev[f_], prefix=f_ ).astype(np.uint8)],
axis=1)
 prev = pd.concat([prev, prev_dum], axis=1)
 del prev_dum
gc.collect()
print('Counting number of Prevs')
 nb_prev_per_curr = prev[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
prev['SK_ID_PREV'] = prev['SK_ID_CURR'].map(nb_prev_per_curr['SK_ID_PREV'])
 print('Averaging prev')
 avg_prev = prev.groupby('SK_ID_CURR' ).mean()
print(avg_prev.head())
 del prev
gc.collect()
print('Reading POS_CASH')
 pos = pd.read_csv('.. /input/POS_CASH_balance.csv')
 print('Go to dummies')
 pos = pd.concat([pos, pd.get_dummies(pos['NAME_CONTRACT_STATUS'])], axis=1)
 print('Compute nb of prevs per curr')
 nb_prevs = pos[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
pos['SK_ID_PREV'] = pos['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 print('Go to averages')
 avg_pos = pos.groupby('SK_ID_CURR' ).mean()
del pos, nb_prevs
gc.collect()
print('Reading CC balance')
 cc_bal = pd.read_csv('.. /input/credit_card_balance.csv')
 print('Go to dummies')
 cc_bal = pd.concat(
 [
cc_bal, pd.get_dummies(
 cc_bal['NAME_CONTRACT_STATUS'], prefix='cc_bal_status_')
 ],
axis=1)
 nb_prevs = cc_bal[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
cc_bal['SK_ID_PREV'] = cc_bal['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 print('Compute average')
 avg_cc_bal = cc_bal.groupby('SK_ID_CURR' ).mean()
avg_cc_bal.columns = ['cc_bal_' + f_ for f_ in avg_cc_bal.columns]
del cc_bal, nb_prevs
gc.collect()
print('Reading Installments')
 inst = pd.read_csv('.. /input/installments_payments.csv')
 nb_prevs = inst[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
inst['SK_ID_PREV'] = inst['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 avg_inst = inst.groupby('SK_ID_CURR' ).mean()
avg_inst.columns = ['inst_' + f_ for f_ in avg_inst.columns]
print('Read data and test')
 data = pd.read_csv('.. /input/application_train.csv')
 test = pd.read_csv('.. /input/application_test.csv')
 print('Shapes : ', data.shape, test.shape)
 y = data['TARGET']
ids = data['SK_ID_CURR']
del data['TARGET']
categorical_feats = [f for f in data.columns if data[f].dtype == 'object']
categorical_feats
for f_ in categorical_feats:
data[f_], indexer = pd.factorize(data[f_])
 test[f_] = indexer.get_indexer(test[f_])
 data = data.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_prev.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_prev.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_pos.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_pos.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_cc_bal.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_cc_bal.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_inst.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_inst.reset_index() , how='left', on='SK_ID_CURR')
 del avg_buro, avg_prev
gc.collect()
return data, test, y, ids<prepare_x_and_y>
head_gender = ind.loc[ind['parentesco1'] == 1, ['idhogar', 'female']]
final = final.merge(head_gender, on = 'idhogar', how = 'left' ).rename(columns = {'female': 'female-head'} )
def train_model(data_, test_, y_, folds_):
oof_preds = np.zeros(data_.shape[0])
 sub_preds = np.zeros(test_.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in data_.columns if f not in ['SK_ID_CURR']]
for n_fold,(trn_idx, val_idx)in enumerate(folds_.split(data_, y_)) :
trn_x, trn_y = data_[feats].iloc[trn_idx], y_.iloc[trn_idx]
val_x, val_y = data_[feats].iloc[val_idx], y_.iloc[val_idx]
clf = LGBMClassifier(
 nthread=4,
n_estimators=10000,
learning_rate=0.03,
num_leaves=34,
colsample_bytree=0.9497036,
subsample=0.8715623,
max_depth=8,
reg_alpha=0.041545473,
reg_lambda=0.0735294,
min_split_gain=0.0222415,
min_child_weight=39.3259775,
silent=-1,
verbose=-1,)
 clf.fit(
 trn_x,
trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric='auc',
verbose=100,
early_stopping_rounds=100 
)
 oof_preds[val_idx] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_[feats],
num_iteration=clf.best_iteration_)[:, 1] / folds_.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' % 
(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 del clf, trn_x, trn_y, val_x, val_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(y, oof_preds))
test_['TARGET'] = sub_preds
df_oof_preds = pd.DataFrame({'SK_ID_CURR':ids, 'TARGET':y, 'PREDICTION':oof_preds})
 df_oof_preds = df_oof_preds[['SK_ID_CURR', 'TARGET', 'PREDICTION']]
return oof_preds, df_oof_preds, test_[['SK_ID_CURR', 'TARGET'
]], feature_importance_df, roc_auc_score(y, oof_preds )<compute_train_metric>
scorer=make_scorer(f1_score,greater_is_better=True,average='macro' )
def display_precision_recall(y_, oof_preds_, folds_idx_):
plt.figure(figsize=(6, 6))
scores = []
for n_fold,(_, val_idx)in enumerate(folds_idx_):
fpr, tpr, thresholds = roc_curve(y_.iloc[val_idx], oof_preds_[val_idx])
 score = average_precision_score(y_.iloc[val_idx], oof_preds_[val_idx])
 scores.append(score)
 plt.plot(
 fpr,
tpr,
lw=1,
alpha=0.3,
label='AP fold %d(AUC = %0.4f)' %(n_fold + 1, score))
precision, recall, thresholds = precision_recall_curve(y_, oof_preds_)
 score = average_precision_score(y_, oof_preds_)
 plt.plot(
 precision,
recall,
color='b',
label='Avg ROC(AUC = %0.4f $\pm$ %0.4f)' %(score, np.std(scores)) ,
lw=2,
alpha=.8)
 plt.xlim([-0.05, 1.05])
 plt.ylim([-0.05, 1.05])
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.title('LightGBM Recall / Precision')
 plt.legend(loc="best")
 plt.tight_layout()
plt.savefig('recall_precision_curve-01.png' )<save_to_csv>
train_labels = np.array(list(final[final['Target'].notnull() ]['Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 test_set = final[final['Target'].isnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 print(test_set.shape)
 submission_base = test[['Id', 'idhogar']].copy()
if __name__ == '__main__':
gc.enable()
data, test, y, ids = build_model_input()
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1001)
 oof_preds, df_oof_preds, test_preds, importances, score = train_model(data, test, y, folds)
 now = datetime.now()
score = str(round(score, 6)).replace('.', '')
 sub_file = 'submission_5x-average-LGB-run-01-v1_' + score + '_' + str(now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
test_preds.to_csv(sub_file, index=False)
 oof_file = 'train_5x-LGB-run-01-v1-oof_' + score + '_' + str(now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
df_oof_preds.to_csv(oof_file, index=False)
 folds_idx = [(trn_idx, val_idx)
 for trn_idx, val_idx in folds.split(data, y)]
display_importances(feature_importance_df_=importances)
 display_roc_curve(y_=y, oof_preds_=oof_preds, folds_idx_=folds_idx)
 display_precision_recall(y_=y, oof_preds_=oof_preds, folds_idx_=folds_idx )<save_to_csv>
pipeline=Pipeline([('imputer',Imputer(strategy='median')) , 
('scaler',MinMaxScaler())])
 features=list(train_set.columns)
 train_set=pipeline.fit_transform(train_set)
 test_set=pipeline.transform(test_set )
data.to_csv('data_all', index=False)
 test.to_csv('test_all', index=False)
 y.to_csv('target_all', index=False)
 ids.to_csv('skids_all', index=False )<compute_train_metric>
model=RandomForestClassifier(n_estimators=100,random_state=10,n_jobs=-1)
 cv_score=cross_val_score(model,train_set,train_labels,cv=10,scoring=scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}' )
roc_arr = []
for n_fold,(_, val_idx)in enumerate(folds_idx):
fpr, tpr, thresholds = roc_curve(y.iloc[val_idx], oof_preds[val_idx])
 score = roc_auc_score(y.iloc[val_idx], oof_preds[val_idx])
 roc_arr.append(score)
<set_options>
model.fit(train_set,train_labels)
 feature_importances=pd.DataFrame({'feature':features,'importance':model.feature_importances_})
 feature_importances.head()
pd.set_option('display.max_columns', None)
 %matplotlib inline
color = sns.color_palette()
warnings.filterwarnings("ignore" )<load_from_csv>
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegressionCV, RidgeClassifierCV
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier
application_train = pd.read_csv('.. /input/application_train.csv')
 application_test= pd.read_csv('.. /input/application_test.csv')
 bureau = pd.read_csv('.. /input/bureau.csv')
 bureau_balance = pd.read_csv('.. /input/bureau_balance.csv')
 POS_CASH_balance = pd.read_csv('.. /input/POS_CASH_balance.csv')
 credit_card_balance = pd.read_csv('.. /input/credit_card_balance.csv')
 previous_application = pd.read_csv('.. /input/previous_application.csv')
 installments_payments = pd.read_csv('.. /input/installments_payments.csv' )<load_from_csv>
model_results = cv_model(train_set, train_labels, LinearSVC() ,
'LSVC', model_results )
application_train = pd.read_csv('.. /input/application_train.csv')
 application_test= pd.read_csv('.. /input/application_test.csv' )<split>
model_results = cv_model(train_set, train_labels,
GaussianNB() , 'GNB', model_results )
def feature_type_split(data, special_list=[]):
cat_list = []
dis_num_list = []
num_list = []
for i in data.columns.tolist() :
if data[i].dtype == 'object':
cat_list.append(i)
 elif data[i].nunique() < 25:
dis_num_list.append(i)
 elif i in special_list:
dis_num_list.append(i)
 else:
num_list.append(i)
 return cat_list, dis_num_list, num_list
cat_list, dis_num_list, num_list = feature_type_split(application_train, special_list=['AMT_REQ_CREDIT_BUREAU_YEAR'] )<feature_engineering>
model_results = cv_model(train_set, train_labels,
MLPClassifier(hidden_layer_sizes=(32, 64, 128, 64, 32)) ,
'MLP', model_results )
application_train['TERM'] = application_train.AMT_CREDIT / application_train.AMT_ANNUITY
application_test['TERM'] = application_test.AMT_CREDIT / application_test.AMT_ANNUITY<categorify>
model_results = cv_model(train_set, train_labels,
LinearDiscriminantAnalysis() ,
'LDA', model_results )
application_train['OVER_EXPECT_CREDIT'] =(application_train.AMT_CREDIT > application_train.AMT_GOODS_PRICE ).map({False:0, True:1})
 application_test['OVER_EXPECT_CREDIT'] =(application_test.AMT_CREDIT > application_test.AMT_GOODS_PRICE ).map({False:0, True:1} )<feature_engineering>
model_results = cv_model(train_set, train_labels,
RidgeClassifierCV() , 'RIDGE', model_results )
application_train['MEAN_BUILDING_SCORE_AVG'] = application_train.iloc[:, 44:58].mean(skipna=True, axis=1)
 application_train['TOTAL_BUILDING_SCORE_AVG'] = application_train.iloc[:, 44:58].sum(skipna=True, axis=1)
 application_test['MEAN_BUILDING_SCORE_AVG'] = application_test.iloc[:, 44:58].mean(skipna=True, axis=1)
 application_test['TOTAL_BUILDING_SCORE_AVG'] = application_test.iloc[:, 44:58].sum(skipna=True, axis=1 )<feature_engineering>
for n in [5, 10, 20]:
print(f'
KNN with {n} neighbors
')
 model_results = cv_model(train_set, train_labels,
KNeighborsClassifier(n_neighbors = n),
f'knn-{n}', model_results )
application_train['FLAG_DOCUMENT_TOTAL'] = application_train.iloc[:, 96:116].sum(axis=1)
 application_test['FLAG_DOCUMENT_TOTAL'] = application_test.iloc[:, 96:116].sum(axis=1 )<feature_engineering>
model_results = cv_model(train_set, train_labels,
ExtraTreesClassifier(n_estimators = 100, random_state = 10),
'EXT', model_results )
application_train['AMT_REQ_CREDIT_BUREAU_TOTAL'] = application_train.iloc[:, 116:122].sum(axis=1)
 application_test['AMT_REQ_CREDIT_BUREAU_TOTAL'] = application_test.iloc[:, 116:122].sum(axis=1 )<feature_engineering>
model_results = cv_model(train_set, train_labels,
RandomForestClassifier(100, random_state=10),
'RF', model_results )
application_train['BIRTH_EMPLOTED_INTERVEL'] = application_train.DAYS_EMPLOYED - application_train.DAYS_BIRTH
application_test['BIRTH_EMPLOTED_INTERVEL'] = application_test.DAYS_EMPLOYED - application_test.DAYS_BIRTH<feature_engineering>
test_ids = list(final.loc[final['Target'].isnull() , 'idhogar'])
application_train['BIRTH_REGISTRATION_INTERVEL'] = application_train.DAYS_REGISTRATION - application_train.DAYS_BIRTH
application_test['BIRTH_REGISTRATION_INTERVEL'] = application_test.DAYS_REGISTRATION - application_test.DAYS_BIRTH<feature_engineering>
def submit(model, train, train_labels, test, test_ids):

 model.fit(train, train_labels)
 predictions = model.predict(test)
 predictions = pd.DataFrame({'idhogar': test_ids,
'Target': predictions})
 submission = submission_base.merge(predictions,
on = 'idhogar',
how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission
application_train['MEAN_BUILDING_SCORE_AVG'] = application_train.iloc[:, 44:58].mean(skipna=True, axis=1)
 application_train['TOTAL_BUILDING_SCORE_AVG'] = application_train.iloc[:, 44:58].sum(skipna=True, axis=1)
 application_test['MEAN_BUILDING_SCORE_AVG'] = application_test.iloc[:, 44:58].mean(skipna=True, axis=1)
 application_test['TOTAL_BUILDING_SCORE_AVG'] = application_test.iloc[:, 44:58].sum(skipna=True, axis=1)
 application_train['INCOME_PER_FAMILY_MEMBER'] = application_train.AMT_INCOME_TOTAL / application_train.CNT_FAM_MEMBERS
application_test['INCOME_PER_FAMILY_MEMBER'] = application_test.AMT_INCOME_TOTAL / application_test.CNT_FAM_MEMBERS
application_train['SEASON_REMAINING'] = application_train.AMT_INCOME_TOTAL/4 - application_train.AMT_ANNUITY
application_test['SEASON_REMAINING'] = application_test.AMT_INCOME_TOTAL/4 - application_test.AMT_ANNUITY
application_train['RATIO_INCOME_GOODS'] = application_train.AMT_INCOME_TOTAL - application_train.AMT_GOODS_PRICE
application_test['RATIO_INCOME_GOODS'] = application_test.AMT_INCOME_TOTAL - application_test.AMT_GOODS_PRICE
application_train['CHILDREN_RATIO'] = application_train['CNT_CHILDREN'] / application_train['CNT_FAM_MEMBERS']
application_test['CHILDREN_RATIO'] = application_test['CNT_CHILDREN'] / application_test['CNT_FAM_MEMBERS']
def label_encoder(input_df, encoder_dict=None):

 categorical_feats = input_df.columns[input_df.dtypes == 'object']
for feat in categorical_feats:
encoder = LabelEncoder()
input_df[feat] = encoder.fit_transform(input_df[feat].fillna('NULL'))
return input_df, categorical_feats.tolist() , encoder_dict
application_train, categorical_feats, encoder_dict = label_encoder(application_train)
 application_test, categorical_feats, encoder_dict = label_encoder(application_test )<import_modules>
rf_submission = submit(RandomForestClassifier(n_estimators = 100,
random_state=10, n_jobs = -1),
train_set, train_labels, test_set, test_ids)
 print(rf_submission)
 rf_submission.to_csv('rf_submission.csv', index = False )
import lightgbm as lgb
from lightgbm import LGBMClassifier
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import StratifiedKFold<prepare_x_and_y>
test_set = pd.DataFrame(test_set, columns = features)
 print(test_set.shape)
 train_set, test_set = train_set.align(test_set, axis = 1, join = 'inner')
 features = list(train_set.columns )
X = application_train.drop(['SK_ID_CURR', 'TARGET'], axis=1)
 y = application_train.TARGET
X_pred = application_test.drop(['SK_ID_CURR'], axis=1 )<define_variables>
estimator = RandomForestClassifier(random_state = 10, n_estimators = 100, n_jobs = -1)
 selector = RFECV(estimator, step = 1, cv = 3, scoring= scorer, n_jobs = -1)
folds = StratifiedKFold(n_splits=5,random_state=6)
 oof_preds = np.zeros(X.shape[0])
 sub_preds = np.zeros(X_pred.shape[0] )<split>
new_drops=[cols for cols in train_set.columns if train_set[cols].isnull().sum() >2000]
train_set=train_set.drop(columns=new_drops)
 test_set=test_set.drop(columns=new_drops)
 train_set.isnull().sum().sum()
start = time.time()
valid_score = 0
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
trn_x, trn_y = X.iloc[trn_idx], y[trn_idx]
val_x, val_y = X.iloc[val_idx], y[val_idx]
train_data = lgb.Dataset(data=trn_x, label=trn_y,categorical_feature=categorical_feats)
 valid_data = lgb.Dataset(data=val_x, label=val_y)
 param = {'application':'binary','num_iterations':4000, 'learning_rate':0.05, 'num_leaves':24, 'feature_fraction':0.8, 'bagging_fraction':0.9,
'lambda_l1':0.1, 'lambda_l2':0.1, 'min_split_gain':0.01, 'early_stopping_round':100, 'max_depth':7, 'min_child_weight':40, 'metric':'auc'}
lgb_es_model = lgb.train(param, train_data, valid_sets=[train_data, valid_data], verbose_eval=100, categorical_feature=categorical_feats)
 oof_preds[val_idx] = lgb_es_model.predict(val_x, num_iteration=lgb_es_model.best_iteration)
 sub_preds += lgb_es_model.predict(X_pred, num_iteration=lgb_es_model.best_iteration)/ folds.n_splits
print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 valid_score += roc_auc_score(val_y, oof_preds[val_idx])
 print('valid score:', str(round(valid_score/folds.n_splits,4)))
 end = time.time()
print('training time:', str(round(( end - start)/60)) , 'mins' )<save_to_csv>
selector.fit(train_set, train_labels )
application_test= pd.read_csv('.. /input/application_test.csv')
 output = pd.DataFrame({'SK_ID_CURR':application_test.SK_ID_CURR, 'TARGET': sub_preds})
 output.to_csv('only_application_pred.csv', index=False )<set_options>
rankings = pd.DataFrame({'feature': list(train_set.columns), 'rank': list(selector.ranking_)} ).sort_values('rank')
 rankings.head(10 )
warnings.filterwarnings('ignore')
<load_from_csv>
train_selected = selector.transform(train_set)
 test_selected = selector.transform(test_set )
app_train = pd.read_csv('.. /input/application_train.csv')
 app_test = pd.read_csv('.. /input/application_test.csv')
 print('Training data shape: {}, Testing data shape: {}'.format(app_train.shape,app_test.shape))<categorify>
selected_features = train_set.columns[np.where(selector.ranking_==1)]
train_selected = pd.DataFrame(train_selected, columns = selected_features)
 test_selected = pd.DataFrame(test_selected, columns = selected_features )
df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True )<count_unique_values>
model_results = cv_model(train_selected, train_labels, model, 'RF-SEL', model_results )
app_train.select_dtypes('object' ).apply(pd.Series.nunique, axis = 0 )<count_values>
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True
print(df['CODE_GENDER'].value_counts())
 print('
Size Before {}'.format(df.shape))
df = df[df['CODE_GENDER'] != 'XNA']
print('Size After {}'.format(df.shape))<categorify>
def model_gbm(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None):

 feature_names = list(features.columns)
 print(feature_names)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {'boosting_type': 'dart',
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36, 'reg_alpha': 0.76,
'reg_lambda': 0.43,
'subsample_for_bin': 40000,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = -1, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 print(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
modeld=lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = -1, n_estimators = 10000,
random_state = 10)
 for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
modeld=model
model.fit(X_train, y_train, early_stopping_rounds = 100,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 200)
 display(model)
 display(model.best_score_)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 display(fold_probabilitites)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
display(fold_predictions)
 fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(model.feature_importances_)
 display(importances)
 display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 display("feature_importances")
 display(feature_importances)
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 display(valid_scores)
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 display(predictions)
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 display(submission)
 display("Model Name")
 display(model)
 return submission, feature_importances, valid_scores
le = LabelEncoder()
le_count = 0
for col in df:
if df[col].dtype == 'object':
if len(list(df[col].unique())) <= 2:
le.fit(df[col])
 df[col] = le.transform(df[col])
 le_count += 1
print('{} variable are label encoded'.format(le_count))
categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= True)
 print('The shape of dataset after One hot encoding: {}'.format(df.shape))<feature_engineering>
%%capture --no-display
%%capture --no-display
predictions, gbm_fi = model_gbm(train_set, train_labels, test_set, test_ids, return_preds=True )
df['DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']
df['PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']<count_values>
predictions.to_csv('lgb1_submission.csv', index = False )
app_train['TARGET'].value_counts()<set_options>
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels,
test_set, test_ids, return_preds=False)
 submission.to_csv('gbm_baseline.csv' )
del app_test, app_train
gc.collect()<categorify>
%%capture --no-display
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels,
test_selected, test_ids)
def _get_categorical_features(df):
feats = [col for col in list(df.columns)if df[col].dtype == 'object']
return feats
def _factorize_categoricals(df, cats):
for col in cats:
df[col], _ = pd.factorize(df[col])
 return df
def _get_dummies(df, cats):
for col in cats:
df = pd.concat([df, pd.get_dummies(df[col], prefix=col)], axis=1)
 return df<load_from_csv>
model_results = model_results.append(pd.DataFrame({'model': ["GBM_10Fold", "GBM_10Fold_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True )
bureau = pd.read_csv('.. /input/bureau.csv', nrows = None)
 bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = None )<categorify>
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels, test_set, test_ids,
nfolds=10, return_preds=False )
bereau_cats = _get_categorical_features(bureau)
 bb_cats = _get_categorical_features(bb)
 bureau = _get_dummies(bureau,bereau_cats)
 bb = _get_dummies(bb,bb_cats )<merge>
submission.to_csv('gbm_10fold.csv', index = False)
bureau_avg = bureau.groupby('SK_ID_CURR' ).mean()
bureau_avg['buro_count'] = bureau[['SK_ID_BUREAU','SK_ID_CURR']].groupby('SK_ID_CURR' ).count() ['SK_ID_BUREAU']
bureau_avg.columns = ['b_' + f_ for f_ in bureau_avg.columns]
df = df.merge(right=bureau_avg.reset_index() , how='left', on='SK_ID_CURR')
 del bb, bureau_avg
gc.collect()<load_from_csv>
%%capture
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels, test_selected, test_ids,
nfolds=10 )
prev = pd.read_csv('.. /input/previous_application.csv', nrows = None )<categorify>
submission.to_csv('gmb_10fold_selected.csv', index = False)
<categorify><EOS> 
model_results = model_results.append(pd.DataFrame({'model': ["GBM_10Fold", "GBM_10Fold_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True )