Kaggle competition: Home Credit Default Risk
label = label.to_numpy()
label.shape<predict_on_test>
import matplotlib.pyplot as plt
import lightgbm as lgb
import gc
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import confusion_matrix,precision_recall_curve,auc,roc_auc_score,roc_curve,recall_score,classification_report
from sklearn.preprocessing import LabelEncoder
preds1 = clf1.predict(label)
 preds2 = clf2.predict(label)
 preds3 = clf3.predict(label)
 preds4 = clf4.predict(label)
 preds5 = clf5.predict(label)
 preds6 = clf6.predict(label)
 predss = sclf.predict(label )<save_to_csv>
app_train = pd.read_csv('.. /input/application_train.csv')
 print('Training data shape: ', app_train.shape)
 app_train.head()
submission1 = pd.DataFrame({'id': testId, 'class': preds1})
 submission1.to_csv('KNN.csv', index=False)
 submission2 = pd.DataFrame({'id': testId, 'class': preds2})
 submission2.to_csv('Random.csv', index=False)
 submission3 = pd.DataFrame({'id': testId, 'class': preds3})
 submission3.to_csv('Gauss.csv', index=False)
 submission4 = pd.DataFrame({'id': testId, 'class': preds4})
 submission4.to_csv('Extra.csv', index=False)
 submission5 = pd.DataFrame({'id': testId, 'class': preds5})
 submission5.to_csv('XGB.csv', index=False)
 submission6 = pd.DataFrame({'id': testId, 'class': preds6})
 submission6.to_csv('Logistic.csv', index=False)
 submissions = pd.DataFrame({'id': testId, 'class': predss})
 submissions.to_csv('Classifier.csv', index=False )<load_from_csv>
app_test = pd.read_csv('.. /input/application_test.csv')
 print('Testing data shape: ', app_test.shape)
 app_test.head()
df = pd.read_csv('/kaggle/input/eval-lab-2-f464/train.csv')
 t = pd.read_csv('/kaggle/input/eval-lab-2-f464/test.csv' )<prepare_x_and_y>
app_train['TARGET'].value_counts()
print('The proportion of label 1 is %.2f' %(sum(app_train['TARGET']==1)/app_train.shape[0]*100), '%' )
y = df['class']
params = [ 'chem_1','chem_2', 'chem_4', 'chem_6', 'attribute']
x = df[params]
df 
<import_modules>
def missing_values_table(df):
mis_val = df.isnull().sum()
mis_val_percent = df.isnull().sum() * 100 / df.shape[0]
mis_val_table = pd.concat([mis_val, mis_val_percent], axis = 1)
 mis_val_table_rename_columns = mis_val_table.rename(columns = {0: 'Missing Values', 1: 'Percentage'})
 mis_val_table_rename_columns = mis_val_table_rename_columns[
mis_val_table_rename_columns.iloc[:, 1]!=0].sort_values('Percentage', ascending=False ).round(1)
 print('The total dataframe has ' + str(df.shape[1])+ ' columns')
 print('There are ' + str(mis_val_table_rename_columns.shape[0])+ ' columns that have missing values.')
 return mis_val_table_rename_columns
import xgboost as xgb
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split<import_modules>
missing_values = missing_values_table(app_train)
 missing_values.head(20 )
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score 
<train_model>
app_train.dtypes.value_counts()
model = RandomForestClassifier(n_estimators = 1000)
 model.fit(x,y )<save_to_csv>
app_train.select_dtypes('object' ).apply(pd.Series.nunique, axis=0 )
y_pred = model.predict(x_test[params])
 y_pred = pd.DataFrame(data = y_pred)
 answer = pd.concat([x_test['id'], y_pred], axis = 1)
 answer.columns = ['id', 'class']
answer.to_csv('lab2_9.csv', index = False )<load_from_csv>
app_test.dtypes.value_counts()
df_train = pd.read_csv('/kaggle/input/1056lab-diabetes-diagnosis/train.csv', index_col=0)
 df_test = pd.read_csv('/kaggle/input/1056lab-diabetes-diagnosis/test.csv', index_col=0 )<set_options>
app_train = pd.get_dummies(app_train)
 app_test = pd.get_dummies(app_test)
 print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )
%matplotlib inline
sns.countplot(x='Diabetes', data=df_train)
 plt.show()<categorify>
train_labels = app_train['TARGET']
app_train, app_test = app_train.align(app_test, join = 'inner', axis = 1)
 app_train['TARGET'] = train_labels
print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )
df_train_dummies = pd.get_dummies(df_train, columns=['Gender'], drop_first=True)
 df_train_dummies<categorify>
app_train['DAYS_EMPLOYED_ANOM'] = app_train['DAYS_EMPLOYED'] == 365243
app_train['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)
 app_test['DAYS_EMPLOYED_ANOM'] = app_test['DAYS_EMPLOYED'] == 365243
app_test['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)
 print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )
df_test_dummies = pd.get_dummies(df_test, columns=['Gender'], drop_first=True)
 df_test_dummies<train_model>
app_train_domain = app_train.copy()
app_test_domain = app_test.copy()
app_train_domain['CREDIT_INCOME_PERCENT'] = app_train_domain['AMT_CREDIT'] / app_train_domain['AMT_INCOME_TOTAL']
app_train_domain['ANNUITY_INCOME_PERCENT'] = app_train_domain['AMT_ANNUITY'] / app_train_domain['AMT_INCOME_TOTAL']
app_train_domain['CREDIT_TERM'] = app_train_domain['AMT_ANNUITY'] / app_train_domain['AMT_CREDIT']
app_train_domain['DAYS_EMPLOYED_PERCENT'] = app_train_domain['DAYS_EMPLOYED'] / app_train_domain['DAYS_BIRTH']
app_test_domain['CREDIT_INCOME_PERCENT'] = app_test_domain['AMT_CREDIT'] / app_test_domain['AMT_INCOME_TOTAL']
app_test_domain['ANNUITY_INCOME_PERCENT'] = app_test_domain['AMT_ANNUITY'] / app_test_domain['AMT_INCOME_TOTAL']
app_test_domain['CREDIT_TERM'] = app_test_domain['AMT_ANNUITY'] / app_test_domain['AMT_CREDIT']
app_test_domain['DAYS_EMPLOYED_PERCENT'] = app_test_domain['DAYS_EMPLOYED'] / app_test_domain['DAYS_BIRTH']
print('Domain Training Features shape: ', app_train_domain.shape)
 print('Domain Testing Features shape: ', app_test_domain.shape )
X_train_dummies = df_train_dummies.drop(columns='Diabetes' ).values
y_train_dummies = df_train_dummies['Diabetes'].values
X_train, X_valid, y_train, y_valid = train_test_split(X_train_dummies, y_train_dummies, test_size=0.2, random_state=0)
 dtc = RandomForestClassifier()
dtc.fit(X_train, y_train )<train_on_grid>
bureau = pd.read_csv('.. /input/bureau.csv')
 bureau.head()
clf = RandomForestClassifier()
params = {'criterion':('gini', 'entropy'), 'max_depth':[1, 2, 3, 4, 5], 'n_estimators':list(range(20,100,10)) }
gscv = GridSearchCV(clf, params, cv=5)
 gscv.fit(X_train_dummies, y_train_dummies )<find_best_score>
previous_loan_counts = bureau.groupby('SK_ID_CURR', as_index=False)['SK_ID_BUREAU'].count().rename(columns = {'SK_ID_BUREAU': 'previous_loan_counts'})
 previous_loan_counts.head()
print('%.3f %r' %(gscv.best_score_, gscv.best_params_))<train_model>
def agg_numeric(df, group_var, df_name):

 for col in df:
if col != group_var and 'SK_ID' in col:
df = df.drop(columns = col)
 group_ids = df[group_var]
numeric_df = df.select_dtypes('number')
 numeric_df[group_var] = group_ids
agg = numeric_df.groupby(group_var ).agg(['count', 'mean', 'max', 'min', 'sum'] ).reset_index()
columns = [group_var]
for var in agg.columns.levels[0]:
if var != group_var:
for stat in agg.columns.levels[1][:-1]:
columns.append('%s_%s_%s' %(df_name, var, stat))
agg.columns = columns
return agg
bureau_agg = agg_numeric(bureau.drop(columns = ['SK_ID_BUREAU']), group_var = 'SK_ID_CURR', df_name = 'bureau')
 bureau_agg.head()
dtc = RandomForestClassifier(criterion='entropy', max_depth=5, n_estimators=60)
 dtc.fit(X_train_dummies, y_train_dummies )<compute_train_metric>
def count_categorical(df, group_var, df_name):

 categorical = pd.get_dummies(df.select_dtypes('object'))
categorical[group_var] = df[group_var]
categorical = categorical.groupby(group_var ).agg(['sum', 'mean'])
 column_names = []
for var in categorical.columns.levels[0]:
for stat in ['count', 'count_norm']:
column_names.append('%s_%s_%s' %(df_name, var, stat))
categorical.columns = column_names
return categorical
bureau_counts = count_categorical(bureau, group_var = 'SK_ID_CURR', df_name = 'bureau')
 bureau_counts.head()
y_pred = dtc.predict_proba(X_valid)[:, 1]
fpr, tpr, thresholds = roc_curve(y_valid, y_pred)
 auc(fpr, tpr )<predict_on_test>
train = app_train_domain.merge(previous_loan_counts, on = 'SK_ID_CURR', how = 'left')
 train['previous_loan_counts'] = train['previous_loan_counts'].fillna(0)
 test = app_test_domain.merge(previous_loan_counts, on = 'SK_ID_CURR', how = 'left')
 test['previous_loan_counts'] = test['previous_loan_counts'].fillna(0 )
X_test = df_test_dummies.values
y_pred = dtc.predict_proba(X_test)[:, 1]<save_to_csv>
train = train.merge(bureau_agg, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_agg, on = 'SK_ID_CURR', how = 'left' )
submit = pd.read_csv('/kaggle/input/1056lab-diabetes-diagnosis/sampleSubmission.csv')
 submit['Diabetes'] = y_pred
submit.to_csv('submission.csv', index=False )<load_from_csv>
train = train.merge(bureau_counts, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_counts, on = 'SK_ID_CURR', how = 'left' )
df_train = pd.read_csv('.. /input/1056lab-diabetes-diagnosis/train.csv',index_col=0)
 df_test = pd.read_csv('.. /input/1056lab-diabetes-diagnosis/test.csv',index_col=0)
 df_train<count_values>
print('Before align train.shape: ', train.shape)
 print('Before align test.shape: ', test.shape)
 train_labels = train['TARGET']
train, test = train.align(test, join = 'inner', axis = 1)
 train['TARGET'] = train_labels
print('After align train.shape: ', train.shape)
 print('After align test.shape: ', test.shape )
df_train['Diabetes'].value_counts()<categorify>
bureau_balance = pd.read_csv('.. /input/bureau_balance.csv')
 bureau_balance.head()
df_train_dummies = pd.get_dummies(df_train, columns=['Gender'], drop_first=True)
 df_train_dummies<categorify>
bureau_balance_agg = agg_numeric(bureau_balance, group_var = 'SK_ID_BUREAU', df_name = 'bureau_balance')
 bureau_balance_agg.head()
df_test_dummies = pd.get_dummies(df_test, columns=['Gender'], drop_first=True)
 df_test_dummies<set_options>
bureau_balance_counts = count_categorical(bureau_balance, group_var = 'SK_ID_BUREAU', df_name = 'bureau_balance')
 bureau_balance_counts.head()
%matplotlib inline
sns.pairplot(df_train_dummies, hue='Diabetes')
 plt.show()<categorify>
bureau_by_loan = bureau_balance_agg.merge(bureau_balance_counts, right_index = True, left_on = 'SK_ID_BUREAU', how = 'outer' )
df_train_dummies2 = df_train_dummies[['HDL Chol', 'Chol/HDL ratio', 'Weight', 'Systolic BP', 'Diastolic BP','Waist','Hip','Diabetes']]
df_test_dummies2 = df_test_dummies[['HDL Chol', 'Chol/HDL ratio', 'Weight', 'Systolic BP', 'Diastolic BP','Waist','Hip']]<split>
bureau_by_loan = bureau_by_loan.merge(bureau[['SK_ID_BUREAU', 'SK_ID_CURR']], on = 'SK_ID_BUREAU', how = 'left' )
X_train_dummies = df_train_dummies2.drop('Diabetes', axis=1 ).values
y_train_dummies = df_train_dummies2['Diabetes'].values
X_train, X_valid, y_train, y_valid = train_test_split(X_train_dummies, y_train_dummies, test_size=0.2, random_state=0 )<train_on_grid>
bureau_balance_by_client = agg_numeric(bureau_by_loan.drop(columns = ['SK_ID_BUREAU']), group_var = 'SK_ID_CURR', df_name = 'client')
 bureau_balance_by_client.head()
clf = RandomForestClassifier()
params = {'criterion':('gini', 'entropy'),
'n_estimators':[240,260,280,300],
'max_depth':[1, 2, 3, 4, 5],
'random_state':[5]
}
gscv = GridSearchCV(clf, params, cv=5,scoring='roc_auc')
 gscv.fit(X_train, y_train )<find_best_params>
train = train.merge(bureau_balance_by_client, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_balance_by_client, on = 'SK_ID_CURR', how = 'left' )
scores = gscv.cv_results_['mean_test_score']
params = gscv.cv_results_['params']
for score, param in zip(scores, params):
print('%.3f %r' %(score, param))<find_best_score>
print('Before align train.shape: ', train.shape)
 print('Before align test.shape: ', test.shape)
 train_labels = train['TARGET']
train, test = train.align(test, join = 'inner', axis = 1)
 train['TARGET'] = train_labels
print('After align train.shape: ', train.shape)
 print('After align test.shape: ', test.shape )
print('%.3f %r' %(gscv.best_score_, gscv.best_params_))<train_model>
gc.enable()
del app_train, app_test, app_train_domain, app_test_domain, bureau, bureau_balance, bureau_agg, bureau_counts, bureau_balance_agg, bureau_balance_counts, bureau_by_loan, bureau_balance_by_client
gc.collect()
clf = RandomForestClassifier(criterion='entropy', max_depth= 4, n_estimators= 280,random_state= 5)
 clf.fit(X_train_dummies,y_train_dummies )<predict_on_test>
missing_train = missing_values_table(train)
 missing_train.head(10 )
X_test = df_test_dummies2.values
y_pred = clf.predict_proba(X_test)[:, 1]<save_to_csv>
missing_test = missing_values_table(test)
 missing_test.head(10 )
submit = pd.read_csv('/kaggle/input/1056lab-diabetes-diagnosis/sampleSubmission.csv')
 submit['Diabetes'] = y_pred
submit.to_csv('submission2.csv', index=False )<define_variables>
missing_train_vars = list(missing_train.index[missing_train['Percentage'] > 90])
 print('There are', len(missing_train_vars), 'columns having missing percent larger than 90%% in the train')
 missing_test_vars = list(missing_test.index[missing_test['Percentage'] > 90])
 print('There are', len(missing_test_vars), 'columns having missing percent larger than 90%% in the test')
 missing_columns = list(set(missing_test_vars + missing_train_vars))
print('There are %d columns with more than 90%% missing in either the training or testing data.' % len(missing_columns))
sys.path.insert(0,'.. /input/efficientnet/EfficientNet-b5')
<import_modules>
train = train.drop(columns = missing_columns)
 test = test.drop(columns = missing_columns)
from __future__ import print_function, division
import argparse
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
from torchvision import datasets, models, transforms
import time
import os
import pandas as pd
from efficientnet.model import EfficientNet
import torch.nn.functional as F
import sys
from PIL import Image<init_hyperparams>
corrs = corrs.sort_values('TARGET', ascending = False)
 pd.DataFrame(corrs['TARGET'].head(10))
use_gpu = torch.cuda.is_available()
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
data_dir = '.. /input/issm2020-ai-challenge'
batch_size = 4
lr = 0.1
momentum = 0.9
num_epochs = 100
input_size = 480
class_num = 10
num_workers = 4<compute_train_metric>
threshold = 0.8
above_threshold_vars = {}
for col in corrs:
above_threshold_vars[col] = list(corrs.index[corrs[col] > threshold] )
def linear_combination(x, y, epsilon):
return epsilon*x +(1-epsilon)*y
def reduce_loss(loss, reduction='mean'):
return loss.mean() if reduction=='mean'else loss.sum() if reduction=='sum' else loss
class LabelSmoothingCrossEntropy(nn.Module):
def __init__(self, epsilon:float=0.1, reduction='mean'):
super().__init__()
self.epsilon = epsilon
self.reduction = reduction
def forward(self, preds, target):
n = preds.size() [-1]
log_preds = F.log_softmax(preds, dim=-1)
 loss = reduce_loss(-log_preds.sum(dim=-1), self.reduction)
 nll = F.nll_loss(log_preds, target, reduction=self.reduction)
 return linear_combination(loss/n, nll, self.epsilon )<choose_model_class>
cols_to_remove = []
cols_seen = []
cols_to_remove_pair = []
for key, value in above_threshold_vars.items() :
cols_seen.append(key)
 for x in value:
if x == key:
next
else:
if x not in cols_seen:
cols_to_remove.append(x)
 cols_to_remove_pair.append(key)
 cols_to_remove = list(set(cols_to_remove))
print('Number of columns to remove: ', len(cols_to_remove))
def exp_lr_scheduler(optimizer, epoch, init_lr=0.01, lr_decay_epoch=10):
lr = init_lr *(0.8**(epoch // lr_decay_epoch))
print('LR is set to {}'.format(lr))
for param_group in optimizer.param_groups:
param_group['lr'] = lr
return optimizer<choose_model_class>
train_corrs_removed = train.drop(columns = cols_to_remove)
 test_corrs_removed = test.drop(columns = cols_to_remove)
 print('Training Corrs Removed Shape: ', train_corrs_removed.shape)
 print('Testing Corrs Removed Shape: ', test_corrs_removed.shape)
def train_model(model_ft, criterion, optimizer, lr_scheduler, num_epochs=50):
train_loss = []
since = time.time()
best_model_wts = model_ft.state_dict()
best_acc = 0.0
best_train_acc = 0.0
for epoch in range(num_epochs):
model_ft.train(True)
 dset_loaders, dset_sizes = loaddata(data_dir=data_dir, batch_size=batch_size, set_name='semTrain', shuffle=True)
 print('Data Size', dset_sizes)
 print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
 optimizer = lr_scheduler(optimizer, epoch)
 running_loss = 0.0
running_corrects = 0
count = 0
for i,data in enumerate(dset_loaders['semTrain']):
inputs, labels = data
labels = torch.squeeze(labels.type(torch.LongTensor))
if use_gpu:
inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())
 else:
inputs, labels = Variable(inputs), Variable(labels)
 outputs = model_ft(inputs)
 loss = criterion(outputs, labels)
 _, preds = torch.max(outputs.data, 1)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
count += 1
if count % 30 == 0 or outputs.size() [0] < batch_size:
print('Epoch:{}: loss:{:.3f}'.format(epoch, loss.item()))
train_loss.append(loss.item())
 running_loss += loss.item() * inputs.size(0)
 running_corrects += torch.sum(preds == labels.data)
 epoch_loss = running_loss / dset_sizes
epoch_acc = running_corrects.double() / dset_sizes
print('Train Loss: {:.4f} Acc: {:.4f}'.format(
 epoch_loss, epoch_acc))
if epoch_acc >= 0.9999:
best_acc = epoch_acc
best_model_wts = model_ft.state_dict()
print('update best weight at No.',epoch,' epoch' )<load_pretrained>
gc.enable()
del train, test
gc.collect()
model_ft = EfficientNet.from_name('efficientnet-b2')
 net_weight = '.. /input/efficientnet-pytorch-b0-b7/efficientnet-b2-8bb594d6.pth'
state_dict = torch.load(net_weight)
 model_ft.load_state_dict(state_dict)
<feature_engineering>
def train_with_cv(train_data, test_data, n_folds, seed_varying):
train_ids = train_data['SK_ID_CURR']
test_ids = test_data['SK_ID_CURR']
train_labels = train_data['TARGET']
train_features = train_data.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test_features = test_data.drop(columns = ['SK_ID_CURR'])
 feature_names = list(train_features.columns)
 feature_importance_values = np.zeros(len(feature_names))
train_features = np.array(train_features)
 test_features = np.array(test_features)
 k_fold = KFold(n_splits = n_folds, shuffle = True, random_state = 50+seed_varying)
 test_pred = np.zeros(test_features.shape[0])
 out_of_fold = np.zeros(train_features.shape[0])
 valid_scores = []
train_scores = []
for train_indices, valid_indices in k_fold.split(train_features):
x_train, y_train = train_features[train_indices], train_labels[train_indices]
x_valid, y_valid = train_features[valid_indices], train_labels[valid_indices]
model = lgb.LGBMClassifier(n_estimators=10000, objective = 'binary',
class_weight = 'balanced', learning_rate = 0.05,
reg_alpha = 0.1, reg_lambda = 0.1,
subsample = 0.8, n_jobs = -1, random_state = 50+seed_varying)
 model.fit(x_train, y_train, eval_metric = 'auc',
eval_set = [(x_valid, y_valid),(x_train, y_train)],
eval_names = ['valid', 'train'], categorical_feature = 'auto',
early_stopping_rounds = 100, verbose = -1)
 best_iteration = model.best_iteration_
feature_importance_values += model.feature_importances_ / k_fold.n_splits
test_pred += model.predict_proba(test_features, num_iteration = best_iteration)[:, 1] / k_fold.n_splits
out_of_fold[valid_indices] = model.predict_proba(x_valid, num_iteration = best_iteration)[:, 1]
valid_score = model.best_score_['valid']['auc']
train_score = model.best_score_['train']['auc']
valid_scores.append(valid_score)
 train_scores.append(train_score)
 gc.enable()
del model, x_train, y_train, x_valid, y_valid
gc.collect()
pred_score = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': test_pred})
 feature_importances = pd.DataFrame({'feature': feature_names, 'importance': feature_importance_values})
 valid_auc = roc_auc_score(train_labels, out_of_fold)
 valid_scores.append(valid_auc)
 train_scores.append(np.mean(train_scores))
fold_names = list(range(n_folds))
fold_names.append('overall')
 metrics = pd.DataFrame({'fold': fold_names,
'train': train_scores,
'valid': valid_scores})
 return pred_score, feature_importances, metrics
num_ftrs = model_ft._fc.in_features
model_ft._fc = nn.Linear(num_ftrs, class_num)
 criterion = LabelSmoothingCrossEntropy()<choose_model_class>
train_times = 5
n_folds = 5
i = 0
metrics_all = np.zeros(( train_times, 2))
for seed_varying in range(train_times):
print('
=======================================================')
 print('The ', seed_varying, ' time of train')
 print('
=======================================================')
 sub, fi, metrics = train_with_cv(train_corrs_removed, test_corrs_removed, n_folds, seed_varying)
 if i==0:
submission = sub
feat_
else:
submission['TARGET'] += sub['TARGET']
feat_import['importance'] += fi['importance']
metrics_all[i, :] = metrics.iloc[-1, 1:3]
i += 1
metrics_all_average = metrics_all.mean(axis = 0)
 metrics_all = np.row_stack([metrics_all, metrics_all_average])
 train_time_names = list(range(train_times))
train_time_names.append('Average')
 metrics_final = pd.DataFrame({'train_time': train_time_names,
'train': metrics_all[:,0],
'valid': metrics_all[:,1]})
 submission['TARGET'] = submission['TARGET'] / train_times
feat_import['importance'] = feat_import['importance'] / train_times
submission.to_csv('lightgbm_version_11.csv', index = False )