Kaggle competition: Digit Recognizer
null_thresh =.8
null_cols = null_ratios[null_ratios > null_thresh].index
merged_df.drop(null_cols, axis=1, inplace=True)
 if null_cols.shape[0] > 0:
print('Columns dropped for being over %.2f null:'%(null_thresh))
for col in null_cols:
print(col )<correct_missing_values>
import torch
import torch.nn as nn
from torchvision import transforms
from torch.utils.data import Dataset, DataLoader
from torch.utils.data.sampler import SubsetRandomSampler
import torchvision.transforms as transforms
merged_df.fillna(0, inplace=True)
<normalization>
import pandas as pd
import numpy as np
from matplotlib.pyplot import imshow
import matplotlib.pylab as plt
from PIL import Image
for feat in merged_df.columns:
if(merged_df[feat].max() > 100)|(merged_df[feat].min() < -100):
merged_df[feat]=merged_df[feat].astype(np.float64)
 scaler = StandardScaler()
continuous_feats = merged_df.columns[merged_df.dtypes == 'float64']
print('Scaling %d features...'%(continuous_feats.shape))
s1 = merged_df.shape[0],1
for feat in continuous_feats:
merged_df[feat] = scaler.fit_transform(merged_df[feat].values.reshape(s1))<split>
train_on_gpu = torch.cuda.is_available()
if not train_on_gpu:
print('Training on CPU...')
 else:
print('Training on GPU...' )
train_df = merged_df[:len_train]
test_df = merged_df[len_train:]
del merged_df
gc.collect() 
<define_variables>
dataset = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 dataset.head(5 )
L2c = 4e-4
lr0 = 0.02
lr_decay = 0.90
iterations = 11
ROWS = train_df.shape[0]
VARS = train_df.shape[1]
NUMB = 10000
NN = int(ROWS/NUMB )<prepare_x_and_y>
plt.imshow(dataset.iloc[0][1:].values.astype(np.uint8 ).reshape(( 28, 28)) , cmap='binary')
 plt.title(dataset.iloc[0][0])
 plt.show()
tf.disable_v2_behavior()
y_ = tf.placeholder(tf.float32, [None, 1])
 x = tf.placeholder(tf.float32, [None, VARS])
 W = tf.Variable(tf.truncated_normal([VARS,1],mean=0.0,stddev=0.001),dtype=np.float32)
 NUML1 = 10
W1 = tf.Variable(tf.truncated_normal([VARS,NUML1],mean=0.0,stddev=0.0001),dtype=np.float32)
 W1f = tf.Variable(tf.truncated_normal([NUML1,1],mean=0.0,stddev=0.0001),dtype=np.float32)
 logit1 = tf.matmul(x, W)+ tf.matmul(tf.nn.relu(tf.matmul(x, W1)) , W1f)
 y = tf.nn.sigmoid(logit1 )<find_best_params>
class DatasetMNIST(Dataset):
def __init__(self, data, transform=None):
self.data = data
self.transform = transform
def __len__(self):
return len(self.data)
 def __getitem__(self, index):
data = self.data.iloc[index]
image = data[1:].values.astype(np.uint8 ).reshape(( 28, 28))
label = data[0]
if self.transform is not None:
image = self.transform(image)
 return image, label
loss0 = tf.reduce_mean(( y_-y)*(y_-y))
loss1 = L2c *(tf.nn.l2_loss(W)+ tf.nn.l2_loss(W1)+ tf.nn.l2_loss(W1f))
loss = loss0 + loss1
global_step = tf.Variable(0, trainable=False)
 learning_rate = tf.train.exponential_decay(lr0, global_step, NN, lr_decay)
 train_step = tf.train.AdamOptimizer(learning_rate=learning_rate ).minimize(loss,global_step=global_step)
 sess = tf.InteractiveSession()
tf.global_variables_initializer().run()<define_variables>
BATCH_SIZE = 16
VALID_SIZE = 0.2
transform_train = transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomRotation(20),
transforms.ToTensor() ,
transforms.Normalize(mean=(0.5,), std=(0.5,))
])
 transform_valid = transforms.Compose([
transforms.ToPILImage() ,
transforms.ToTensor() ,
transforms.Normalize(mean=(0.5,), std=(0.5,))
] )
y0=target.values.astype(np.float32)
 x0=train_df.values.astype(np.float32)
 del train_df
gc.collect()
y0_1=np.where(y0[0:int(NN*0.8)*NUMB] == 1)[0]
y0_0=np.where(y0[0:int(NN*0.8)*NUMB] == 0)[0]
for i in range(iterations):
for j in range(int(NN*0.8)) :
pos_ratio = 0.5
pos_idx = np.random.choice(y0_1, size=int(np.round(NUMB*pos_ratio)))
 neg_idx = np.random.choice(y0_0, size=int(np.round(NUMB*(1-pos_ratio))))
idx = np.concatenate([pos_idx, neg_idx])
 fd = {y_: y0[idx].reshape(NUMB,1),x: x0[idx,:]}
_= sess.run([train_step], feed_dict=fd)
 if i%10 == 0:
fd = {y_: y0.reshape(y0.shape[0],1),x: x0}
y1 = sess.run(y, feed_dict=fd)
 lim = int(NN*0.8)* NUMB
auc1 = roc_auc_score(y0[0:lim],y1[0:lim,0])
 auc2 = roc_auc_score(y0[lim:y0.shape[0]],y1[lim:y0.shape[0],0])
 print('iteration %d, auc train/validatet %.5f/%.5f'%(i,auc1,auc2)) 
<save_to_csv>
train_data = DatasetMNIST(dataset, transform=transform_train)
 valid_data = DatasetMNIST(dataset, transform=transform_valid )
x0 = test_df.values.astype(np.float32)
 fd = {y_: np.zeros([x0.shape[0],1]),x: x0}
y_pred = sess.run(y, feed_dict=fd)
 out_df = pd.DataFrame({'SK_ID_CURR': meta_df[len_train:], 'TARGET': y_pred[:,0]})
 out_df.to_csv('nn_submission.csv', index=False)
<load_from_csv>
num_train = len(train_data)
 indices = list(range(num_train))
np.random.shuffle(indices)
 split = int(np.floor(VALID_SIZE * num_train))
train_idx, valid_idx = indices[split:], indices[:split]
print("Size of Train Set: ", len(train_idx))
print("Size of Valid Set: ", len(valid_idx))
train_sampler = SubsetRandomSampler(train_idx)
 valid_sampler = SubsetRandomSampler(valid_idx)
 train_loader = DataLoader(train_data, batch_size=BATCH_SIZE, sampler=train_sampler)
 valid_loader = DataLoader(valid_data, batch_size=BATCH_SIZE, sampler=valid_sampler )
nn_result=pd.read_csv('.. /input/pial-data-n/nn_p.csv')
 log_result=pd.read_csv('.. /input/pial-data/lgb_p.csv')
 nn_result.rename(columns={'TARGET':'nn_TARGET'},inplace=True)
 log_result.rename(columns={'TARGET':'log_TARGET'},inplace=True)
 sub=pd.merge(nn_result,log_result,on='SK_ID_CURR')
 sub['TARGET']=0*sub['nn_TARGET']+1*sub['log_TARGET']
sub=sub.drop(columns=['log_TARGET','nn_TARGET'])
 sub.to_csv('subm_nn_log_p.csv', index=False, float_format='%.8f' )<set_options>
class CNN_model(nn.Module):
def __init__(self):
super(CNN_model, self ).__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(1, 16, 3, padding=1),
nn.BatchNorm2d(16),
nn.ReLU() ,
nn.Conv2d(16, 16, 3, padding=1),
nn.BatchNorm2d(16),
nn.ReLU() ,
nn.MaxPool2d(2,2)
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(16, 32, 3, padding=1),
nn.BatchNorm2d(32),
nn.ReLU() ,
nn.Conv2d(32, 32, 3, padding=1),
nn.BatchNorm2d(32),
nn.ReLU() ,
nn.MaxPool2d(2,2)
)
 self.conv3 = nn.Sequential(
 nn.Conv2d(32, 64, 3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU() ,
nn.Conv2d(64, 64, 3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU() 
)
 self.fc = nn.Linear(64*7*7, 10)
 self.bn_fc = nn.BatchNorm1d(10)
 def forward(self, x):
out = self.conv1(x)
 out = self.conv2(out)
 out = self.conv3(out)
 out = out.view(out.size(0), -1)
 out = self.fc(out)
 out = self.bn_fc(out)
 return out
model = CNN_model()
model
warnings.filterwarnings('ignore' )<categorify>
if train_on_gpu:
model = model.cuda()
print("Train on GPU")
 else:
print("Train on CPU" )
def one_hot_encoder(df, nan_as_category=True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns=categorical_columns, dummy_na=nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns
def group(df_to_agg, prefix, aggregations, aggregate_by= 'SK_ID_CURR'):
agg_df = df_to_agg.groupby(aggregate_by ).agg(aggregations)
 agg_df.columns = pd.Index(['{}{}_{}'.format(prefix, e[0], e[1].upper())
 for e in agg_df.columns.tolist() ])
 return agg_df.reset_index()
def group_and_merge(df_to_agg, df_to_merge, prefix, aggregations, aggregate_by= 'SK_ID_CURR'):
agg_df = group(df_to_agg, prefix, aggregations, aggregate_by= aggregate_by)
 return df_to_merge.merge(agg_df, how='left', on= aggregate_by)
 def do_sum(dataframe, group_cols, counted, agg_name):
gp = dataframe[group_cols + [counted]].groupby(group_cols)[counted].sum().reset_index().rename(columns={counted: agg_name})
 dataframe = dataframe.merge(gp, on=group_cols, how='left')
 return dataframe
def reduce_mem_usage(dataframe):
m_start = dataframe.memory_usage().sum() / 1024 ** 2
for col in dataframe.columns:
col_type = dataframe[col].dtype
if col_type != object:
c_min = dataframe[col].min()
c_max = dataframe[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
dataframe[col] = dataframe[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
dataframe[col] = dataframe[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
dataframe[col] = dataframe[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
dataframe[col] = dataframe[col].astype(np.int64)
 elif str(col_type)[:5] == 'float':
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
dataframe[col] = dataframe[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
dataframe[col] = dataframe[col].astype(np.float32)
 else:
dataframe[col] = dataframe[col].astype(np.float64)
 m_end = dataframe.memory_usage().sum() / 1024 ** 2
return dataframe
nan_as_category = True
def risk_groupanizer(dataframe, column_names, target_val=1, upper_limit_ratio=8.2, lower_limit_ratio=8.2):
all_cols = dataframe.columns
for col in column_names:
temp_df = dataframe.groupby([col] + ['TARGET'])[['SK_ID_CURR']].count().reset_index()
temp_df['ratio%'] = round(temp_df['SK_ID_CURR']*100/temp_df.groupby([col])['SK_ID_CURR'].transform('sum'), 1)
 col_groups_high_risk = temp_df[(temp_df['TARGET'] == target_val)& 
(temp_df['ratio%'] >= upper_limit_ratio)][col].tolist()
col_groups_low_risk = temp_df[(temp_df['TARGET'] == target_val)& 
(lower_limit_ratio >= temp_df['ratio%'])][col].tolist()
if upper_limit_ratio != lower_limit_ratio:
col_groups_medium_risk = temp_df[(temp_df['TARGET'] == target_val)& 
(upper_limit_ratio > temp_df['ratio%'])&(temp_df['ratio%'] > lower_limit_ratio)][col].tolist()
for risk, col_groups in zip(['_high_risk', '_medium_risk', '_low_risk'],
[col_groups_high_risk, col_groups_medium_risk, col_groups_low_risk]):
dataframe[col + risk] = [1 if val in col_groups else 0 for val in dataframe[col].values]
else:
for risk, col_groups in zip(['_high_risk', '_low_risk'], [col_groups_high_risk, col_groups_low_risk]):
dataframe[col + risk] = [1 if val in col_groups else 0 for val in dataframe[col].values]
if dataframe[col].dtype == 'O' or dataframe[col].dtype == 'object':
dataframe.drop(col, axis=1, inplace=True)
 return dataframe, list(set(dataframe.columns ).difference(set(all_cols)))
 def ligthgbm_feature_selection(dataframe, index_cols, auc_limit=0.7):
dataframe = dataframe.rename(columns=lambda x: re.sub('[^A-Za-z0-9_]+', '_', x))
clf = LGBMClassifier(random_state=0)
 train_df = dataframe[dataframe['TARGET'].notnull() ]
train_df_X = train_df.drop('TARGET', axis=1)
 train_df_y = train_df['TARGET']
train_columns = [col for col in train_df_X.columns if col not in index_cols]
max_auc_score = 1
best_cols = []
while max_auc_score > auc_limit:
train_columns = [col for col in train_columns if col not in best_cols]
clf.fit(train_df_X[train_columns], train_df_y)
 feats_imp = pd.Series(clf.feature_importances_, index=train_columns)
 max_auc_score = roc_auc_score(train_df_y, clf.predict_proba(train_df_X[train_columns])[:, 1])
 best_cols = feats_imp[feats_imp > 0].index.tolist()
dataframe.drop(train_columns, axis=1, inplace=True)
 return dataframe<load_from_csv>
criterion = nn.CrossEntropyLoss()
learning_rate = 0.001
optimizer = torch.optim.Adam(model.parameters() , lr = learning_rate )
def application() :
df = pd.read_csv(r'.. /input/home-credit-default-risk/application_train.csv')
 test_df = pd.read_csv(r'.. /input/home-credit-default-risk/application_test.csv')
 df = df.append(test_df ).reset_index()
df = df[df['CODE_GENDER'] != 'XNA']
df = df[df['AMT_INCOME_TOTAL'] < 20000000]
df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace=True)
 df['DAYS_LAST_PHONE_CHANGE'].replace(0, np.nan, inplace=True)
 for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:
df[bin_feature], uniques = pd.factorize(df[bin_feature])
 df, cat_cols = one_hot_encoder(df, nan_as_category)
 docs = [f for f in df.columns if 'FLAG_DOC' in f]
df['DOCUMENT_COUNT'] = df[docs].sum(axis=1)
 df['NEW_DOC_KURT'] = df[docs].kurtosis(axis=1)
 def get_age_label(days_birth):

 age_years = -days_birth / 365
if age_years < 27: return 1
elif age_years < 40: return 2
elif age_years < 50: return 3
elif age_years < 65: return 4
elif age_years < 99: return 5
else: return 0
df['AGE_RANGE'] = df['DAYS_BIRTH'].apply(lambda x: get_age_label(x))
df['EXT_SOURCES_PROD'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2'] * df['EXT_SOURCE_3']
df['EXT_SOURCES_WEIGHTED'] = df.EXT_SOURCE_1 * 2 + df.EXT_SOURCE_2 * 1 + df.EXT_SOURCE_3 * 3
np.warnings.filterwarnings('ignore', r'All-NaN(slice|axis)encountered')
 for function_name in ['min', 'max', 'mean', 'nanmedian', 'var']:
feature_name = 'EXT_SOURCES_{}'.format(function_name.upper())
 df[feature_name] = eval('np.{}'.format(function_name))(
 df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']], axis=1)
 df['DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']
df['PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']
df['CREDIT_TO_GOODS_RATIO'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']
df['INCOME_TO_EMPLOYED_RATIO'] = df['AMT_INCOME_TOTAL'] / df['DAYS_EMPLOYED']
df['INCOME_TO_BIRTH_RATIO'] = df['AMT_INCOME_TOTAL'] / df['DAYS_BIRTH']
df['ID_TO_BIRTH_RATIO'] = df['DAYS_ID_PUBLISH'] / df['DAYS_BIRTH']
df['CAR_TO_BIRTH_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']
df['CAR_TO_EMPLOYED_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']
df['PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']
df['APPS_EXT_SOURCE_MEAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 df['APPS_EXT_SOURCE_STD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)
 df['APPS_EXT_SOURCE_STD'] = df['APPS_EXT_SOURCE_STD'].fillna(df['APPS_EXT_SOURCE_STD'].mean())
 df['APP_SCORE1_TO_BIRTH_RATIO'] = df['EXT_SOURCE_1'] /(df['DAYS_BIRTH'] / 365.25)
 df['APP_SCORE2_TO_BIRTH_RATIO'] = df['EXT_SOURCE_2'] /(df['DAYS_BIRTH'] / 365.25)
 df['APP_SCORE3_TO_BIRTH_RATIO'] = df['EXT_SOURCE_3'] /(df['DAYS_BIRTH'] / 365.25)
 df['APP_SCORE1_TO_EMPLOY_RATIO'] = df['EXT_SOURCE_1'] /(df['DAYS_EMPLOYED'] / 365.25)
 df['APP_EXT_SOURCE_2*EXT_SOURCE_3*DAYS_BIRTH'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2'] * df['DAYS_BIRTH']
df['APP_SCORE1_TO_FAM_CNT_RATIO'] = df['EXT_SOURCE_1'] / df['CNT_FAM_MEMBERS']
df['APP_SCORE1_TO_GOODS_RATIO'] = df['EXT_SOURCE_1'] / df['AMT_GOODS_PRICE']
df['APP_SCORE1_TO_CREDIT_RATIO'] = df['EXT_SOURCE_1'] / df['AMT_CREDIT']
df['APP_SCORE1_TO_SCORE2_RATIO'] = df['EXT_SOURCE_1'] / df['EXT_SOURCE_2']
df['APP_SCORE1_TO_SCORE3_RATIO'] = df['EXT_SOURCE_1'] / df['EXT_SOURCE_3']
df['APP_SCORE2_TO_CREDIT_RATIO'] = df['EXT_SOURCE_2'] / df['AMT_CREDIT']
df['APP_SCORE2_TO_REGION_RATING_RATIO'] = df['EXT_SOURCE_2'] / df['REGION_RATING_CLIENT']
df['APP_SCORE2_TO_CITY_RATING_RATIO'] = df['EXT_SOURCE_2'] / df['REGION_RATING_CLIENT_W_CITY']
df['APP_SCORE2_TO_POP_RATIO'] = df['EXT_SOURCE_2'] / df['REGION_POPULATION_RELATIVE']
df['APP_SCORE2_TO_PHONE_CHANGE_RATIO'] = df['EXT_SOURCE_2'] / df['DAYS_LAST_PHONE_CHANGE']
df['APP_EXT_SOURCE_1*EXT_SOURCE_2'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2']
df['APP_EXT_SOURCE_1*EXT_SOURCE_3'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_3']
df['APP_EXT_SOURCE_2*EXT_SOURCE_3'] = df['EXT_SOURCE_2'] * df['EXT_SOURCE_3']
df['APP_EXT_SOURCE_1*DAYS_EMPLOYED'] = df['EXT_SOURCE_1'] * df['DAYS_EMPLOYED']
df['APP_EXT_SOURCE_2*DAYS_EMPLOYED'] = df['EXT_SOURCE_2'] * df['DAYS_EMPLOYED']
df['APP_EXT_SOURCE_3*DAYS_EMPLOYED'] = df['EXT_SOURCE_3'] * df['DAYS_EMPLOYED']
df['APPS_GOODS_INCOME_RATIO'] = df['AMT_GOODS_PRICE'] / df['AMT_INCOME_TOTAL']
df['APPS_CNT_FAM_INCOME_RATIO'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['APPS_INCOME_EMPLOYED_RATIO'] = df['AMT_INCOME_TOTAL'] / df['DAYS_EMPLOYED']
df['CREDIT_TO_GOODS_RATIO_2'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']
df['APP_AMT_INCOME_TOTAL_12_AMT_ANNUITY_ratio'] = df['AMT_INCOME_TOTAL'] / 12.- df['AMT_ANNUITY']
df['APP_INCOME_TO_EMPLOYED_RATIO'] = df['AMT_INCOME_TOTAL'] / df['DAYS_EMPLOYED']
df['APP_DAYS_LAST_PHONE_CHANGE_DAYS_EMPLOYED_ratio'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_EMPLOYED']
df['APP_DAYS_EMPLOYED_DAYS_BIRTH_diff'] = df['DAYS_EMPLOYED'] - df['DAYS_BIRTH']
print('"Application_Train_Test" final shape:', df.shape)
 return df<load_from_csv>
n_epochs=20
valid_loss_min = np.Inf
train_losses, valid_losses = [], []
accuracy_list=[]
N_valid = len(valid_idx )
def bureau_bb() :
bureau = pd.read_csv(r'.. /input/home-credit-default-risk/bureau.csv')
 bb = pd.read_csv(r'.. /input/home-credit-default-risk/bureau_balance.csv')
 bureau['CREDIT_DURATION'] = -bureau['DAYS_CREDIT'] + bureau['DAYS_CREDIT_ENDDATE']
bureau['ENDDATE_DIF'] = bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_ENDDATE_FACT']
bureau['DEBT_PERCENTAGE'] = bureau['AMT_CREDIT_SUM'] / bureau['AMT_CREDIT_SUM_DEBT']
bureau['DEBT_CREDIT_DIFF'] = bureau['AMT_CREDIT_SUM'] - bureau['AMT_CREDIT_SUM_DEBT']
bureau['CREDIT_TO_ANNUITY_RATIO'] = bureau['AMT_CREDIT_SUM'] / bureau['AMT_ANNUITY']
bureau['BUREAU_CREDIT_FACT_DIFF'] = bureau['DAYS_CREDIT'] - bureau['DAYS_ENDDATE_FACT']
bureau['BUREAU_CREDIT_ENDDATE_DIFF'] = bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE']
bureau['BUREAU_CREDIT_DEBT_RATIO'] = bureau['AMT_CREDIT_SUM_DEBT'] / bureau['AMT_CREDIT_SUM']
bureau['BUREAU_IS_DPD'] = bureau['CREDIT_DAY_OVERDUE'].apply(lambda x: 1 if x > 0 else 0)
 bureau['BUREAU_IS_DPD_OVER120'] = bureau['CREDIT_DAY_OVERDUE'].apply(lambda x: 1 if x > 120 else 0)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)
 bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size', 'mean']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 num_aggregations = {
'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],
'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],
'DAYS_CREDIT_UPDATE': ['mean'],
'CREDIT_DAY_OVERDUE': ['max', 'mean', 'min'],
'AMT_CREDIT_MAX_OVERDUE': ['mean', 'max'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean', 'max', 'sum'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'AMT_ANNUITY': ['max', 'mean', 'sum'],
'CNT_CREDIT_PROLONG': ['sum'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum'],
'SK_ID_BUREAU': ['count'],
'DAYS_ENDDATE_FACT': ['min', 'max', 'mean'],
'ENDDATE_DIF': ['min', 'max', 'mean'],
'BUREAU_CREDIT_FACT_DIFF': ['min', 'max', 'mean'],
'BUREAU_CREDIT_ENDDATE_DIFF': ['min', 'max', 'mean'],
'BUREAU_CREDIT_DEBT_RATIO': ['min', 'max', 'mean'],
'DEBT_CREDIT_DIFF': ['min', 'max', 'mean'],
'BUREAU_IS_DPD': ['mean', 'sum'],
'BUREAU_IS_DPD_OVER120': ['mean', 'sum']
}
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ])
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACTIVE_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')
 closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLOSED_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 print('"Bureau/Bureau Balance" final shape:', bureau_agg.shape)
 return bureau_agg<load_from_csv>
torch.cuda.empty_cache()
def previous_application() :
prev = pd.read_csv(r'.. /input/home-credit-default-risk/previous_application.csv')
 prev, cat_cols = one_hot_encoder(prev, nan_as_category=True)
 prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace=True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace=True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace=True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace=True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace=True)
 prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']
prev['APPLICATION_CREDIT_DIFF'] = prev['AMT_APPLICATION'] - prev['AMT_CREDIT']
prev['CREDIT_TO_ANNUITY_RATIO'] = prev['AMT_CREDIT'] / prev['AMT_ANNUITY']
prev['DOWN_PAYMENT_TO_CREDIT'] = prev['AMT_DOWN_PAYMENT'] / prev['AMT_CREDIT']
total_payment = prev['AMT_ANNUITY'] * prev['CNT_PAYMENT']
prev['SIMPLE_INTERESTS'] =(total_payment / prev['AMT_CREDIT'] - 1)/ prev['CNT_PAYMENT']
prev['DAYS_LAST_DUE_DIFF'] = prev['DAYS_LAST_DUE_1ST_VERSION'] - prev['DAYS_LAST_DUE']
prev['PREV_GOODS_DIFF'] = prev['AMT_APPLICATION'] - prev['AMT_GOODS_PRICE']
prev['PREV_ANNUITY_APPL_RATIO'] = prev['AMT_ANNUITY']/prev['AMT_APPLICATION']
prev['PREV_GOODS_APPL_RATIO'] = prev['AMT_GOODS_PRICE'] / prev['AMT_APPLICATION']
num_aggregations = {
'AMT_ANNUITY': ['min', 'max', 'mean', 'sum'],
'AMT_APPLICATION': ['min', 'max', 'mean', 'sum'],
'AMT_CREDIT': ['min', 'max', 'mean', 'sum'],
'APP_CREDIT_PERC': ['min', 'max', 'mean', 'var'],
'AMT_DOWN_PAYMENT': ['min', 'max', 'mean', 'sum'],
'AMT_GOODS_PRICE': ['min', 'max', 'mean', 'sum'],
'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
'SK_ID_PREV': ['nunique'],
'DAYS_TERMINATION': ['max'],
'CREDIT_TO_ANNUITY_RATIO': ['mean', 'max'],
'APPLICATION_CREDIT_DIFF': ['min', 'max', 'mean', 'sum'],
'DOWN_PAYMENT_TO_CREDIT': ['mean'],
'PREV_GOODS_DIFF': ['mean', 'max', 'sum'],
'PREV_GOODS_APPL_RATIO': ['mean', 'max'],
'DAYS_LAST_DUE_DIFF': ['mean', 'max', 'sum'],
'SIMPLE_INTERESTS': ['mean', 'max']
}
cat_aggregations = {}
for cat in cat_cols:
cat_aggregations[cat] = ['mean']
prev_agg = prev.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 prev_agg.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist() ])
 approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
approved_agg = approved.groupby('SK_ID_CURR' ).agg(num_aggregations)
 approved_agg.columns = pd.Index(['APPROVED_' + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist() ])
 prev_agg = prev_agg.join(approved_agg, how='left', on='SK_ID_CURR')
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
refused_agg = refused.groupby('SK_ID_CURR' ).agg(num_aggregations)
 refused_agg.columns = pd.Index(['REFUSED_' + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist() ])
 prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')
 print('"Previous Applications" final shape:', prev_agg.shape)
 return prev_agg<load_from_csv>
for epoch in range(n_epochs):
train_loss = 0
model.train()
for x, y in train_loader:
if train_on_gpu:
x, y = x.cuda() , y.cuda()
optimizer.zero_grad()
z = model(x)
 loss = criterion(z, y)
 loss.backward()
optimizer.step()
train_loss += loss.item()
valid_loss = 0
correct = 0
accuracy = 0
with torch.no_grad() :
model.eval()
for x_test, y_test in valid_loader:
if train_on_gpu:
x_test, y_test = x_test.cuda() , y_test.cuda()
z = model(x_test)
 valid_loss += criterion(z, y_test ).item()
_, yhat = torch.max(z.data, 1)
 correct +=(yhat == y_test ).sum().item()
accuracy = correct / N_valid
train_loss = train_loss / len(train_loader)
 valid_loss = valid_loss / len(valid_loader)
 accuracy_list.append(accuracy)
 train_losses.append(train_loss)
 valid_losses.append(valid_loss)
 print(f"Epoch: {epoch}/{n_epochs} ",
f"Training Loss: {train_loss:.4f} ",
f"Validation Loss: {valid_loss:.4f} ",
f"Test Accuracy: {accuracy:.4f}")
 network_learned = valid_loss < valid_loss_min
if network_learned:
valid_loss_min = valid_loss
checkpoint = {'model': CNN_model() ,
'state_dict': model.state_dict() ,
'optimizer' : optimizer.state_dict() }
torch.save(checkpoint, 'model_mnist.pth')
 print("Saved this model!")
 torch.cuda.empty_cache()
def pos_cash() :
pos = pd.read_csv(r'.. /input/home-credit-default-risk/POS_CASH_balance.csv')
 pos, cat_cols = one_hot_encoder(pos, nan_as_category=True)
 pos['LATE_PAYMENT'] = pos['SK_DPD'].apply(lambda x: 1 if x > 0 else 0)
 pos['POS_IS_DPD'] = pos['SK_DPD'].apply(lambda x: 1 if x > 0 else 0)
 pos['POS_IS_DPD_UNDER_120'] = pos['SK_DPD'].apply(lambda x: 1 if(x > 0)&(x < 120)else 0)
 pos['POS_IS_DPD_OVER_120'] = pos['SK_DPD'].apply(lambda x: 1 if x >= 120 else 0)
 aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size', 'min'],
'SK_DPD': ['max', 'mean', 'sum', 'var', 'min'],
'SK_DPD_DEF': ['max', 'mean', 'sum'],
'SK_ID_PREV': ['nunique'],
'LATE_PAYMENT': ['mean'],
'SK_ID_CURR': ['count'],
'CNT_INSTALMENT': ['min', 'max', 'mean', 'sum'],
'CNT_INSTALMENT_FUTURE': ['min', 'max', 'mean', 'sum'],
'POS_IS_DPD': ['mean', 'sum'],
'POS_IS_DPD_UNDER_120': ['mean', 'sum'],
'POS_IS_DPD_OVER_120': ['mean', 'sum'],
}
for cat in cat_cols:
aggregations[cat] = ['mean']
pos_agg = pos.groupby('SK_ID_CURR' ).agg(aggregations)
 pos_agg.columns = pd.Index(['POS_' + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR' ).size()
sort_pos = pos.sort_values(by=['SK_ID_PREV', 'MONTHS_BALANCE'])
 gp = sort_pos.groupby('SK_ID_PREV')
 df_pos = pd.DataFrame()
df_pos['SK_ID_CURR'] = gp['SK_ID_CURR'].first()
df_pos['MONTHS_BALANCE_MAX'] = gp['MONTHS_BALANCE'].max()
df_pos['POS_LOAN_COMPLETED_MEAN'] = gp['NAME_CONTRACT_STATUS_Completed'].mean()
df_pos['POS_COMPLETED_BEFORE_MEAN'] = gp['CNT_INSTALMENT'].first() - gp['CNT_INSTALMENT'].last()
df_pos['POS_COMPLETED_BEFORE_MEAN'] = df_pos.apply(lambda x: 1 if x['POS_COMPLETED_BEFORE_MEAN'] > 0 \
and x['POS_LOAN_COMPLETED_MEAN'] > 0 else 0, axis=1)
 df_pos['POS_REMAINING_INSTALMENTS'] = gp['CNT_INSTALMENT_FUTURE'].last()
df_pos['POS_REMAINING_INSTALMENTS_RATIO'] = gp['CNT_INSTALMENT_FUTURE'].last() /gp['CNT_INSTALMENT'].last()
df_gp = df_pos.groupby('SK_ID_CURR' ).sum().reset_index()
df_gp.drop(['MONTHS_BALANCE_MAX'], axis=1, inplace= True)
 pos_agg = pd.merge(pos_agg, df_gp, on= 'SK_ID_CURR', how= 'left')
 pos = do_sum(pos, ['SK_ID_PREV'], 'LATE_PAYMENT', 'LATE_PAYMENT_SUM')
 last_month_df = pos.groupby('SK_ID_PREV')['MONTHS_BALANCE'].idxmax()
sort_pos = pos.sort_values(by=['SK_ID_PREV', 'MONTHS_BALANCE'])
 gp = sort_pos.iloc[last_month_df].groupby('SK_ID_CURR' ).tail(3)
 gp_mean = gp.groupby('SK_ID_CURR' ).mean().reset_index()
pos_agg = pd.merge(pos_agg, gp_mean[['SK_ID_CURR', 'LATE_PAYMENT_SUM']], on='SK_ID_CURR', how='left')
 print('"Pos-Cash" balance final shape:', pos_agg.shape)
 return pos_agg<load_from_csv>
checkpoint = torch.load('model_mnist.pth')
 model.load_state_dict(checkpoint['state_dict'])
 optimizer.load_state_dict(checkpoint['optimizer'] )
def installment() :
ins = pd.read_csv(r'.. /input/home-credit-default-risk/installments_payments.csv')
 ins, cat_cols = one_hot_encoder(ins, nan_as_category=True)
 ins = do_sum(ins, ['SK_ID_PREV', 'NUM_INSTALMENT_NUMBER'], 'AMT_PAYMENT', 'AMT_PAYMENT_GROUPED')
 ins['PAYMENT_DIFFERENCE'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT_GROUPED']
ins['PAYMENT_RATIO'] = ins['AMT_INSTALMENT'] / ins['AMT_PAYMENT_GROUPED']
ins['PAID_OVER_AMOUNT'] = ins['AMT_PAYMENT'] - ins['AMT_INSTALMENT']
ins['PAID_OVER'] =(ins['PAID_OVER_AMOUNT'] > 0 ).astype(int)
 ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD_diff'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD_diff'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD_diff'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD_diff'].apply(lambda x: x if x > 0 else 0)
 ins['LATE_PAYMENT'] = ins['DBD'].apply(lambda x: 1 if x > 0 else 0)
 ins['INSTALMENT_PAYMENT_RATIO'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['LATE_PAYMENT_RATIO'] = ins.apply(lambda x: x['INSTALMENT_PAYMENT_RATIO'] if x['LATE_PAYMENT'] == 1 else 0, axis=1)
 ins['SIGNIFICANT_LATE_PAYMENT'] = ins['LATE_PAYMENT_RATIO'].apply(lambda x: 1 if x > 0.05 else 0)
 ins['DPD_7'] = ins['DPD'].apply(lambda x: 1 if x >= 7 else 0)
 ins['DPD_15'] = ins['DPD'].apply(lambda x: 1 if x >= 15 else 0)
 ins['INS_IS_DPD_UNDER_120'] = ins['DPD'].apply(lambda x: 1 if(x > 0)&(x < 120)else 0)
 ins['INS_IS_DPD_OVER_120'] = ins['DPD'].apply(lambda x: 1 if(x >= 120)else 0)
 aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DPD': ['max', 'mean', 'sum', 'var'],
'DBD': ['max', 'mean', 'sum', 'var'],
'PAYMENT_PERC': ['max', 'mean', 'sum', 'var'],
'PAYMENT_DIFF': ['max', 'mean', 'sum', 'var'],
'AMT_INSTALMENT': ['max', 'mean', 'sum', 'min'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum', 'min'],
'SK_ID_PREV': ['size', 'nunique'],
'PAYMENT_DIFFERENCE': ['mean'],
'PAYMENT_RATIO': ['mean', 'max'],
'LATE_PAYMENT': ['mean', 'sum'],
'SIGNIFICANT_LATE_PAYMENT': ['mean', 'sum'],
'LATE_PAYMENT_RATIO': ['mean'],
'DPD_7': ['mean'],
'DPD_15': ['mean'],
'PAID_OVER': ['mean'],
'DPD_diff':['mean', 'min', 'max'],
'DBD_diff':['mean', 'min', 'max'],
'DAYS_INSTALMENT': ['mean', 'max', 'sum'],
'INS_IS_DPD_UNDER_120': ['mean', 'sum'],
'INS_IS_DPD_OVER_120': ['mean', 'sum']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
ins_agg = ins.groupby('SK_ID_CURR' ).agg(aggregations)
 ins_agg.columns = pd.Index(['INSTAL_' + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist() ])
 ins_agg['INSTAL_COUNT'] = ins.groupby('SK_ID_CURR' ).size()
cond_day = ins['DAYS_ENTRY_PAYMENT'] >= -365
ins_d365_grp = ins[cond_day].groupby('SK_ID_CURR')
 ins_d365_agg_dict = {
'SK_ID_CURR': ['count'],
'NUM_INSTALMENT_VERSION': ['nunique'],
'DAYS_ENTRY_PAYMENT': ['mean', 'max', 'sum'],
'DAYS_INSTALMENT': ['mean', 'max', 'sum'],
'AMT_INSTALMENT': ['mean', 'max', 'sum'],
'AMT_PAYMENT': ['mean', 'max', 'sum'],
'PAYMENT_DIFF': ['mean', 'min', 'max', 'sum'],
'PAYMENT_PERC': ['mean', 'max'],
'DPD_diff': ['mean', 'min', 'max'],
'DPD': ['mean', 'sum'],
'INS_IS_DPD_UNDER_120': ['mean', 'sum'],
'INS_IS_DPD_OVER_120': ['mean', 'sum']}
ins_d365_agg = ins_d365_grp.agg(ins_d365_agg_dict)
 ins_d365_agg.columns = ['INS_D365' +('_' ).join(column ).upper() for column in ins_d365_agg.columns.ravel() ]
ins_agg = ins_agg.merge(ins_d365_agg, on='SK_ID_CURR', how='left')
 print('"Installments Payments" final shape:', ins_agg.shape)
 return ins_agg<load_from_csv>
print(model )
def credit_card() :
cc = pd.read_csv(r'.. /input/home-credit-default-risk/credit_card_balance.csv')
 cc, cat_cols = one_hot_encoder(cc, nan_as_category=True)
 cc['LIMIT_USE'] = cc['AMT_BALANCE'] / cc['AMT_CREDIT_LIMIT_ACTUAL']
cc['PAYMENT_DIV_MIN'] = cc['AMT_PAYMENT_CURRENT'] / cc['AMT_INST_MIN_REGULARITY']
cc['LATE_PAYMENT'] = cc['SK_DPD'].apply(lambda x: 1 if x > 0 else 0)
 cc['DRAWING_LIMIT_RATIO'] = cc['AMT_DRAWINGS_ATM_CURRENT'] / cc['AMT_CREDIT_LIMIT_ACTUAL']
cc['CARD_IS_DPD_UNDER_120'] = cc['SK_DPD'].apply(lambda x: 1 if(x > 0)&(x < 120)else 0)
 cc['CARD_IS_DPD_OVER_120'] = cc['SK_DPD'].apply(lambda x: 1 if x >= 120 else 0)
 cc_agg = cc.groupby('SK_ID_CURR' ).agg(['min', 'max', 'mean', 'sum', 'var'])
 cc_agg.columns = pd.Index(['CC_' + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist() ])
 cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR' ).size()
last_ids = cc.groupby('SK_ID_PREV')['MONTHS_BALANCE'].idxmax()
last_months_df = cc[cc.index.isin(last_ids)]
cc_agg = group_and_merge(last_months_df,cc_agg,'CC_LAST_', {'AMT_BALANCE': ['mean', 'max']})
 CREDIT_CARD_TIME_AGG = {
'AMT_BALANCE': ['mean', 'max'],
'LIMIT_USE': ['max', 'mean'],
'AMT_CREDIT_LIMIT_ACTUAL':['max'],
'AMT_DRAWINGS_ATM_CURRENT': ['max', 'sum'],
'AMT_DRAWINGS_CURRENT': ['max', 'sum'],
'AMT_DRAWINGS_POS_CURRENT': ['max', 'sum'],
'AMT_INST_MIN_REGULARITY': ['max', 'mean'],
'AMT_PAYMENT_TOTAL_CURRENT': ['max','sum'],
'AMT_TOTAL_RECEIVABLE': ['max', 'mean'],
'CNT_DRAWINGS_ATM_CURRENT': ['max','sum', 'mean'],
'CNT_DRAWINGS_CURRENT': ['max', 'mean', 'sum'],
'CNT_DRAWINGS_POS_CURRENT': ['mean'],
'SK_DPD': ['mean', 'max', 'sum'],
'LIMIT_USE': ['min', 'max'],
'DRAWING_LIMIT_RATIO': ['min', 'max'],
'LATE_PAYMENT': ['mean', 'sum'],
'CARD_IS_DPD_UNDER_120': ['mean', 'sum'],
'CARD_IS_DPD_OVER_120': ['mean', 'sum']
}
for months in [12, 24, 48]:
cc_prev_id = cc[cc['MONTHS_BALANCE'] >= -months]['SK_ID_PREV'].unique()
cc_recent = cc[cc['SK_ID_PREV'].isin(cc_prev_id)]
prefix = 'INS_{}M_'.format(months)
 cc_agg = group_and_merge(cc_recent, cc_agg, prefix, CREDIT_CARD_TIME_AGG)
 print('"Credit Card Balance" final shape:', cc_agg.shape)
 return cc_agg<create_dataframe>
test_dataset = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 test_dataset.head(5 )
def data_post_processing(dataframe):
print(f'---=> the DATA POST-PROCESSING is beginning, the dataset has {dataframe.shape[1]} features')
 index_cols = ['TARGET', 'SK_ID_CURR', 'SK_ID_BUREAU', 'SK_ID_PREV', 'index']
dataframe = dataframe.rename(columns=lambda x: re.sub('[^A-Za-z0-9_]+', '_', x))
print('names of feature are renamed')
 dataframe = reduce_mem_usage(dataframe)
 print(f'---=> pandas data types of features in the dataset are converted for a reduced memory usage')
 noninformative_cols = []
for col in dataframe.columns:
if len(dataframe[col].value_counts())< 2:
noninformative_cols.append(col)
 dataframe.drop(noninformative_cols, axis=1, inplace=True)
 print(f'---=> {dataframe.shape[1]} features are remained after removing non-informative features')
 feature_num = dataframe.shape[1]
auc_limit = 0.7
all_features = dataframe.columns.tolist()
selected_feature_df = pd.read_csv('.. /input/homecredit-best-subs/removed_cols_lgbm.csv')
 selected_features = selected_feature_df.removed_cols.tolist()
remained_features = set(all_features ).difference(set(selected_features))
dataframe = dataframe[remained_features]
print(f'{feature_num - dataframe.shape[1]} features are eliminated by LightGBM classifier with an {auc_limit} auc score limit in step I')
 print(f'---=> {dataframe.shape[1]} features are remained after removing features not interesting for LightGBM classifier')
 start_feats_num = dataframe.shape[1]
cat_cols = [col for col in dataframe.columns if 3 < len(dataframe[col].value_counts())< 20 and col not in index_cols]
dataframe, _ = risk_groupanizer(dataframe, column_names=cat_cols, upper_limit_ratio=8.1, lower_limit_ratio=8.1)
 print(f'---=> {dataframe.shape[1] - start_feats_num} features are generated with the risk_groupanizer')
 print(f'---=> the DATA POST-PROCESSING is ended!, now the dataset has a total {dataframe.shape[1]} features')
 gc.collect()
return dataframe<load_from_csv>
class TestDatasetMNIST(Dataset):
def __init__(self, data, transform=None):
self.data = data
self.transform = transform
def __len__(self):
return len(self.data)
 def __getitem__(self, index):
image = self.data.iloc[index].values.astype(np.uint8 ).reshape(( 28, 28))
if self.transform is not None:
image = self.transform(image)
 return image
def Kfold_LightGBM(df):
print('===============================================', '
', '
df_subx = pd.read_csv(r'.. /input/homecredit-best-subs/df_subs_3.csv')
 df_sub = df_subx[['SK_ID_CURR', '23']]
df_sub.columns = ['SK_ID_CURR', 'TARGET']
train_df = df[df['TARGET'].notnull() ]
test_df = df[df['TARGET'].isnull() ]
del df
gc.collect()
test_df.TARGET = np.where(df_sub.TARGET > 0.75, 1, 0)
 train_df = pd.concat([train_df, test_df], axis=0)
 train_df = pd.concat([train_df, test_df], axis=0)
 train_df = pd.concat([train_df, test_df], axis=0)
 print(f'Train shape: {train_df.shape}, test shape: {test_df.shape} are loaded.')
 folds = KFold(n_splits=5, shuffle=True, random_state=2020)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feats = [f for f in train_df.columns if f not in ['TARGET', 'SK_ID_CURR', 'SK_ID_BUREAU', 'SK_ID_PREV']]
print(f'only {len(feats)} features from a total {train_df.shape[1]} features are used for ML analysis')
 for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['TARGET'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]
clf = LGBMClassifier(nthread=-1,
n_estimators=5000,
learning_rate=0.01,
max_depth=11,
num_leaves=58,
colsample_bytree=0.613,
subsample=0.708,
max_bin=407,
reg_alpha=3.564,
reg_lambda=4.930,
min_child_weight=6,
min_child_samples=165,
silent=-1,
verbose=-1,)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)], eval_metric='auc', verbose=500, early_stopping_rounds=500)
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del clf, train_x, train_y, valid_x, valid_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))
test_df['TARGET'] = sub_preds
test_df[['SK_ID_CURR', 'TARGET']].to_csv('submission.csv', index=False)
 print('a submission file is created' )<merge>
test_data = TestDatasetMNIST(test_dataset, transform=transform_valid)
 test_loader = DataLoader(test_data, batch_size=BATCH_SIZE, shuffle=False )
df = application()
df = df.merge(bureau_bb() , how='left', on='SK_ID_CURR')
 print('--=> df after merge with bureau:', df.shape)
 df = df.merge(previous_application() , how='left', on='SK_ID_CURR')
 print('--=> df after merge with previous application:', df.shape)
 df = df.merge(pos_cash() , how='left', on='SK_ID_CURR')
 print('--=> df after merge with pos cash :', df.shape)
 df = df.merge(installment() , how='left', on='SK_ID_CURR')
 print('--=> df after merge with installments:', df.shape)
 df = df.merge(credit_card() , how='left', on='SK_ID_CURR')
 print('--=> df after merge with credit card:', df.shape)
 df = data_post_processing(df)
 print('='*50, '
')
 print('---=> df final shape:', df.shape, ' <=---', '
')
 print('=' * 50)
 Kfold_LightGBM(df)
 print('--=> all calculations are done!! <=--' )<create_dataframe>
submissions = [['ImageId', 'Label']]
img_id = 1
with torch.no_grad() :
model.eval()
for x in test_loader:
if train_on_gpu:
x = x.cuda()
z = model(x)
 _, yhat = torch.max(z.data, 1)
 for pred in yhat:
submissions.append([img_id, pred.item() ])
 img_id += 1
print(f"Submission size: {len(submissions)-1}")
 print(f"Test data size: {len(test_dataset)}" )
<save_to_csv><EOS> 
with open('submission.csv', 'w')as submissionFile:
writer = csv.writer(submissionFile)
 writer.writerows(submissions)
 print('Submission Complete!' )