Kaggle competition: Home Credit Default Risk
epochs, validation_split =(600, 0.0)
 history = model.fit(Xs, y, workers=4, epochs=epochs, verbose=0, validation_split=validation_split,
batch_size=512, shuffle=True, callbacks=[ModelCallback() ], initial_epoch=0)
 plot_history(( history), title="", since=0 )<save_to_csv>
import matplotlib.pyplot as plt
import lightgbm as lgb
import gc
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import confusion_matrix,precision_recall_curve,auc,roc_auc_score,roc_curve,recall_score,classification_report
from sklearn.preprocessing import LabelEncoder
test = load_csv("test")
 X, Xs = to_model(merge_data(super_battle(load_csv("test")) , pokemon))
prediction = model.predict_classes(Xs)
 df_submission = pd.DataFrame({"Winner": prediction}, index = test.index.rename("battle_number"))
for i, p1, p2 in load_csv("test" ).values:
if(p1, p2)in acc100: df_submission.iloc[i] = acc100[(p1, p2)][0]
df_submission.to_csv("solution.csv" )<load_pretrained>
app_train = pd.read_csv('.. /input/application_train.csv')
 print('Training data shape: ', app_train.shape)
 app_train.head()
model.save_weights("cp-{epoch:04d}.ckpt".format(epoch=epochs))
! mkdir "saved_models"
saved_model_path = tf.contrib.saved_model.save_keras_model(model, "./saved_models" )<load_from_csv>
app_test = pd.read_csv('.. /input/application_test.csv')
 print('Testing data shape: ', app_test.shape)
 app_test.head()
battles = pd.read_csv(".. /input/battles.csv")
 pokemon = pd.read_csv(".. /input/pokemon.csv")
 pokemon.describe()<feature_engineering>
app_train['TARGET'].value_counts()
print('The proportion of label 1 is %.2f' %(sum(app_train['TARGET']==1)/app_train.shape[0]*100), '%' )
def normalizeColumns(columns, dataset):

 for column in columns:
dataset[column] =(dataset[column] - dataset[column].mean())/ dataset[column].std()<categorify>
def missing_values_table(df):
mis_val = df.isnull().sum()
mis_val_percent = df.isnull().sum() * 100 / df.shape[0]
mis_val_table = pd.concat([mis_val, mis_val_percent], axis = 1)
 mis_val_table_rename_columns = mis_val_table.rename(columns = {0: 'Missing Values', 1: 'Percentage'})
 mis_val_table_rename_columns = mis_val_table_rename_columns[
mis_val_table_rename_columns.iloc[:, 1]!=0].sort_values('Percentage', ascending=False ).round(1)
 print('The total dataframe has ' + str(df.shape[1])+ ' columns')
 print('There are ' + str(mis_val_table_rename_columns.shape[0])+ ' columns')
 return mis_val_table_rename_columns
pokemon.fillna("noType", inplace=True)
 pokemon = pd.concat([
pokemon,
pd.get_dummies(pokemon["Type 1"], prefix="t1"),
pd.get_dummies(pokemon["Type 2"], prefix="t2"),
], axis=1)
 pokemon.drop(["Type 1", "Type 2", "Name", "Generation", "Legendary"], axis=1, inplace=True)
 normalizeColumns(["HP", "Sp.Atk", "Sp.Def", "Attack", "Defense", "Speed"], pokemon )<merge>
missing_values = missing_values_table(app_train)
 missing_values.head(20 )
def mergePokemonStats(battles, pokemon):
data = battles \ 
.merge(pokemon, left_on="First_pokemon", right_on=" 
.merge(pokemon, left_on="Second_pokemon", right_on=" 
.sort_values(['battle_number'])\ 
.drop(["
data = data.reindex(sorted(data.columns), axis=1)
 return data<merge>
app_train.dtypes.value_counts()
data = mergePokemonStats(battles, pokemon)
 data.head()<prepare_x_and_y>
app_train.select_dtypes('object' ).apply(pd.Series.nunique, axis=0 )
y = data["Winner"].values
x = data.drop(["Winner"],axis=1 ).values<import_modules>
app_test.dtypes.value_counts()
import tensorflow as tf
import sklearn.model_selection
import sklearn.ensemble
import sklearn.metrics<choose_model_class>
app_train = pd.get_dummies(app_train)
 app_test = pd.get_dummies(app_test)
 print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )
mlp = tf.keras.Sequential([
tf.keras.layers.Dense(310, activation=tf.nn.relu),
tf.keras.layers.GaussianNoise(0.17),
tf.keras.layers.Dropout(0.50),
tf.keras.layers.Dense(310, activation=tf.nn.relu),
tf.keras.layers.Dropout(0.50),
tf.keras.layers.Dense(1, activation=tf.nn.sigmoid)
 ] )<choose_model_class>
train_labels = app_train['TARGET']
app_train, app_test = app_train.align(app_test, join = 'inner', axis = 1)
 app_train['TARGET'] = train_labels
print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )
mlp.compile(optimizer="adam",
loss="binary_crossentropy",
metrics=["accuracy"] )<train_model>
app_train['DAYS_EMPLOYED_ANOM'] = app_train['DAYS_EMPLOYED'] == 365243
app_train['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)
 app_test['DAYS_EMPLOYED_ANOM'] = app_test['DAYS_EMPLOYED'] == 365243
app_test['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)
 print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )
history = mlp.fit(
 x,
y,
epochs=350,
workers=4,
batch_size=512,
shuffle=True,
validation_split=0.1
)<split>
app_train_domain = app_train.copy()
app_test_domain = app_test.copy()
app_train_domain['CREDIT_INCOME_PERCENT'] = app_train_domain['AMT_CREDIT'] / app_train_domain['AMT_INCOME_TOTAL']
app_train_domain['ANNUITY_INCOME_PERCENT'] = app_train_domain['AMT_ANNUITY'] / app_train_domain['AMT_INCOME_TOTAL']
app_train_domain['CREDIT_TERM'] = app_train_domain['AMT_ANNUITY'] / app_train_domain['AMT_CREDIT']
app_train_domain['DAYS_EMPLOYED_PERCENT'] = app_train_domain['DAYS_EMPLOYED'] / app_train_domain['DAYS_BIRTH']
app_test_domain['CREDIT_INCOME_PERCENT'] = app_test_domain['AMT_CREDIT'] / app_test_domain['AMT_INCOME_TOTAL']
app_test_domain['ANNUITY_INCOME_PERCENT'] = app_test_domain['AMT_ANNUITY'] / app_test_domain['AMT_INCOME_TOTAL']
app_test_domain['CREDIT_TERM'] = app_test_domain['AMT_ANNUITY'] / app_test_domain['AMT_CREDIT']
app_test_domain['DAYS_EMPLOYED_PERCENT'] = app_test_domain['DAYS_EMPLOYED'] / app_test_domain['DAYS_BIRTH']
print('Domain Training Features shape: ', app_train_domain.shape)
 print('Domain Testing Features shape: ', app_test_domain.shape )
x_train, x_test, y_train, y_test = sklearn.model_selection.train_test_split(x, y, test_size=0.1)
 gradientBoosting = sklearn.ensemble.GradientBoostingClassifier(
 n_estimators=1000,
max_depth=10,
verbose = 1 
)
 gradientBoosting.fit(x_train, y_train )<compute_test_metric>
bureau = pd.read_csv('.. /input/bureau.csv')
 bureau.head()
sklearn.metrics.accuracy_score(y_test, gradientBoosting.predict(x_test), normalize=True, sample_weight=None )<load_from_csv>
previous_loan_counts = bureau.groupby('SK_ID_CURR', as_index=False)['SK_ID_BUREAU'].count().rename(columns = {'SK_ID_BUREAU': 'previous_loan_counts'})
 previous_loan_counts.head()
submission = pd.read_csv(".. /input/test.csv")
 test = mergePokemonStats(submission, pokemon )<predict_on_test>
def agg_numeric(df, group_var, df_name):

 for col in df:
if col != group_var and 'SK_ID' in col:
df = df.drop(columns = col)
 group_ids = df[group_var]
numeric_df = df.select_dtypes('number')
 numeric_df[group_var] = group_ids
agg = numeric_df.groupby(group_var ).agg(['count', 'mean', 'max', 'min', 'sum'] ).reset_index()
columns = [group_var]
for var in agg.columns.levels[0]:
if var != group_var:
for stat in agg.columns.levels[1][:-1]:
columns.append('%s_%s_%s' %(df_name, var, stat))
agg.columns = columns
return agg
bureau_agg = agg_numeric(bureau.drop(columns = ['SK_ID_BUREAU']), group_var = 'SK_ID_CURR', df_name = 'bureau')
 bureau_agg.head()
predictionMLP = np.transpose(mlp.predict(test.values))
predictionGB = gradientBoosting.predict_proba(test.values)
 predictionGB = np.array([predict[1] for predict in predictionGB])
 prediction =(( predictionMLP)+(0.10*predictionGB)) /1.10
prediction[prediction <= 0.5] = 0
prediction[prediction > 0.5] = 1<data_type_conversions>
def count_categorical(df, group_var, df_name):

 categorical = pd.get_dummies(df.select_dtypes('object'))
categorical[group_var] = df[group_var]
categorical = categorical.groupby(group_var ).agg(['sum', 'mean'])
 column_names = []
for var in categorical.columns.levels[0]:
for stat in ['count', 'count_norm']:
column_names.append('%s_%s_%s' %(df_name, var, stat))
categorical.columns = column_names
return categorical
bureau_counts = count_categorical(bureau, group_var = 'SK_ID_CURR', df_name = 'bureau')
 bureau_counts.head()
submission["Winner"] = prediction[0].astype("int")
 submission.drop(["First_pokemon", "Second_pokemon"], axis=1, inplace=True)
 submission.head()<save_to_csv>
train = app_train_domain.merge(previous_loan_counts, on = 'SK_ID_CURR', how = 'left')
 train['previous_loan_counts'] = train['previous_loan_counts'].fillna(0)
 test = app_test_domain.merge(previous_loan_counts, on = 'SK_ID_CURR', how = 'left')
 test['previous_loan_counts'] = test['previous_loan_counts'].fillna(0 )
submission.to_csv("submission.csv", index=False )<save_to_csv>
train = train.merge(bureau_agg, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_agg, on = 'SK_ID_CURR', how = 'left' )
submission.to_csv("submission.csv", index=False )<load_from_csv>
train = train.merge(bureau_counts, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_counts, on = 'SK_ID_CURR', how = 'left' )
battles = pd.read_csv('.. /input/battles.csv')
 pokemon = pd.read_csv('.. /input/pokemon.csv')
 test = pd.read_csv('.. /input/test.csv')
 pokemon = pokemon.drop(columns='Name', errors='ignore')
 pokemon.loc[:, 'Type 1'] = pokemon.loc[:, 'Type 1'].fillna('None')
 pokemon.loc[:, 'Type 2'] = pokemon.loc[:, 'Type 2'].fillna('None')
 pokemon.Legendary = pokemon.Legendary.astype(int)
 data = pd.merge(battles, pokemon, left_on='First_pokemon' ,right_on='
data = pd.merge(data, pokemon, left_on='Second_pokemon',right_on='
data = data.sort_values(['battle_number'])
 data = data.drop(columns=['
data = data.iloc[:, 3:]
data.head()<merge>
print('Before align train.shape: ', train.shape)
 print('Before align test.shape: ', test.shape)
 train_labels = train['TARGET']
train, test = train.align(test, join = 'inner', axis = 1)
 train['TARGET'] = train_labels
print('After align train.shape: ', train.shape)
 print('After align test.shape: ', test.shape )
tout = pd.merge(test, pokemon, left_on='First_pokemon' ,right_on='
tout = pd.merge(tout, pokemon, left_on='Second_pokemon',right_on='
tout = tout.sort_values(['battle_number'])
 tout = tout.drop(columns=['
tout = tout.iloc[:, 3:]
tout.head()<split>
bureau_balance = pd.read_csv('.. /input/bureau_balance.csv')
 bureau_balance.head()
TEST_SIZE = 0.01
tr_idx, ts_idx = train_test_split(range(len(battles)) , test_size=TEST_SIZE)
 cat_names = ['Generation_A', 'Generation_B', 'Type 1_A', 'Type 1_B', 'Type 2_A', 'Type 2_B', 'Legendary_A', 'Legendary_B']
procs = [Categorify, Normalize]
db = TabularDataBunch.from_df(path='.', df=data, dep_var='Winner', valid_idx=ts_idx, procs=procs, test_df=tout, cat_names=cat_names, bs=1024)
 learn = tabular_learner(db, layers=[64, 32, 16, 8], metrics=accuracy)
 learn.lr_find()
learn.recorder.plot()<train_model>
bureau_balance_agg = agg_numeric(bureau_balance, group_var = 'SK_ID_BUREAU', df_name = 'bureau_balance')
 bureau_balance_agg.head()
learn.fit_one_cycle(50, 1e-1 )<prepare_output>
bureau_balance_counts = count_categorical(bureau_balance, group_var = 'SK_ID_BUREAU', df_name = 'bureau_balance')
 bureau_balance_counts.head()
preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 preds = np.argmax(preds, axis=1 ).numpy()
submission = pd.DataFrame(test.iloc[:, 0])
 submission['Winner'] = preds<save_to_csv>
bureau_by_loan = bureau_balance_agg.merge(bureau_balance_counts, right_index = True, left_on = 'SK_ID_BUREAU', how = 'outer' )
print(submission.head())
 submission.to_csv('./submission.csv', index=False )<save_to_csv>
bureau_by_loan = bureau_by_loan.merge(bureau[['SK_ID_BUREAU', 'SK_ID_CURR']], on = 'SK_ID_BUREAU', how = 'left' )
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv(index=False)
 b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 create_download_link(submission )<set_options>
bureau_balance_by_client = agg_numeric(bureau_by_loan.drop(columns = ['SK_ID_BUREAU']), group_var = 'SK_ID_CURR', df_name = 'client')
 bureau_balance_by_client.head()
%matplotlib inline<load_from_csv>
train = train.merge(bureau_balance_by_client, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_balance_by_client, on = 'SK_ID_CURR', how = 'left' )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
print('Before align train.shape: ', train.shape)
 print('Before align test.shape: ', test.shape)
 train_labels = train['TARGET']
train, test = train.align(test, join = 'inner', axis = 1)
 train['TARGET'] = train_labels
print('After align train.shape: ', train.shape)
 print('After align test.shape: ', test.shape )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<count_missing_values>
gc.enable()
del app_train, app_test, app_train_domain, app_test_domain, bureau, bureau_balance, bureau_agg, bureau_balance_agg, bureau_balance_counts, bureau_by_loan, bureau_balance_by_client
gc.collect()
train.isnull().sum()<count_missing_values>
def train_with_cv(train_data, test_data, n_folds, seed_varying):
train_ids = train_data['SK_ID_CURR']
test_ids = test_data['SK_ID_CURR']
train_labels = train_data['TARGET']
train_features = train_data.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test_features = test_data.drop(columns = ['SK_ID_CURR'])
 feature_names = list(train_features.columns)
 feature_importance_values = np.zeros(len(feature_names))
train_features = np.array(train_features)
 test_features = np.array(test_features)
 k_fold = KFold(n_splits = n_folds, shuffle = True, random_state = 50+seed_varying)
 test_pred = np.zeros(test_features.shape[0])
 out_of_fold = np.zeros(train_features.shape[0])
 valid_scores = []
train_scores = []
for train_indices, valid_indices in k_fold.split(train_features):
x_train, y_train = train_features[train_indices], train_labels[train_indices]
x_valid, y_valid = train_features[valid_indices], train_labels[valid_indices]
model = lgb.LGBMClassifier(n_estimators=10000, objective = 'binary',
class_weight = 'balanced', learning_rate = 0.05,
reg_alpha = 0.1, reg_lambda = 0.1,
subsample = 0.8, n_jobs = -1, random_state = 50+seed_varying)
 model.fit(x_train, y_train, eval_metric = 'auc',
eval_set = [(x_valid, y_valid),(x_train, y_train)],
eval_names = ['valid', 'train'], categorical_feature = 'auto',
early_stopping_rounds = 100, verbose = -1)
 best_iteration = model.best_iteration_
feature_importance_values += model.feature_importances_ / k_fold.n_splits
test_pred += model.predict_proba(test_features, num_iteration = best_iteration)[:, 1] / k_fold.n_splits
out_of_fold[valid_indices] = model.predict_proba(x_valid, num_iteration = best_iteration)[:, 1]
valid_score = model.best_score_['valid']['auc']
train_score = model.best_score_['train']['auc']
valid_scores.append(valid_score)
 train_scores.append(train_score)
 gc.enable()
del model, x_train, y_train, x_valid, y_valid
gc.collect()
pred_score = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': test_pred})
 feature_importances = pd.DataFrame({'feature': feature_names, 'importance': feature_importance_values})
 valid_auc = roc_auc_score(train_labels, out_of_fold)
 valid_scores.append(valid_auc)
 train_scores.append(np.mean(train_scores))
fold_names = list(range(n_folds))
fold_names.append('overall')
 metrics = pd.DataFrame({'fold': fold_names,
'train': train_scores,
'valid': valid_scores})
 return pred_score, feature_importances, metrics
test.isnull().sum()<count_missing_values>
train_times = 3
n_folds = 5
i = 0
metrics_all = np.zeros(( train_times, 2))
for seed_varying in range(train_times):
print('
=======================================================')
 print('The ', seed_varying, ' time of train')
 print('
=======================================================')
 sub, fi, metrics = train_with_cv(train, test, n_folds, seed_varying)
 if i==0:
submission = sub
feat_
else:
submission['TARGET'] += sub['TARGET']
feat_import['importance'] += fi['importance']
metrics_all[i, :] = metrics.iloc[-1, 1:3]
i += 1
metrics_all_average = metrics_all.mean(axis = 0)
 metrics_all = np.row_stack([metrics_all, metrics_all_average])
 train_time_names = list(range(train_times))
train_time_names.append('Average')
 metrics_final = pd.DataFrame({'train_time': train_time_names,
'train': metrics_all[:,0],
'valid': metrics_all[:,1]})
 submission['TARGET'] = submission['TARGET'] / train_times
feat_import['importance'] = feat_import['importance'] / train_times
submission.to_csv('lightgbm_version_10.csv', index = False )