Kaggle competition: Home Credit Default Risk
if use_gpu:
model_ft = model_ft.cuda()
criterion = criterion.cuda()
optimizer = optim.SGD(( model_ft.parameters()), lr=lr,
momentum=momentum, weight_decay=0.0004)
<train_model>
import matplotlib.pyplot as plt
import lightgbm as lgb
import gc
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import confusion_matrix,precision_recall_curve,auc,roc_auc_score,roc_curve,recall_score,classification_report
from sklearn.preprocessing import LabelEncoder
<find_best_params>
app_train = pd.read_csv('.. /input/application_train.csv')
 print('Training data shape: ', app_train.shape)
 app_train.head()
model_ft_test = EfficientNet.from_name('efficientnet-b2')
 num_ftrs = model_ft_test._fc.in_features
model_ft_test._fc = nn.Linear(num_ftrs, class_num)
 net_weight = '.. /input/weight/best_val_efficientnet-b2.pth'
state_dict = torch.load(net_weight)
 model_ft_test.load_state_dict(state_dict)
 model_ft_test.cuda()
model_ft_test.eval()
data_transforms = transforms.Compose([
transforms.Resize(input_size),
transforms.CenterCrop(input_size),
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])] )<features_selection>
app_test = pd.read_csv('.. /input/application_test.csv')
 print('Testing data shape: ', app_test.shape)
 app_test.head()
res = []
hash1 = {}
for imagepath in os.listdir('.. /input/issm2020-ai-challenge/semTest/semTest'):
p = []
image = Image.open('.. /input/issm2020-ai-challenge/semTest/semTest/'+imagepath)
 imgprob = data_transforms(image ).unsqueeze(0)
 imgprob = Variable(imgprob ).cuda()
torch.no_grad()
logit = model_ft_test(imgprob)
 p.append(F.softmax(logit, -1))
if 1:
logit = model_ft_test(torch.flip(imgprob, dims=(2,)).contiguous())
 p.append(F.softmax(logit, -1))
logit = model_ft_test(torch.flip(imgprob, dims=(3,)).contiguous())
 p.append(F.softmax(logit, -1))
logit = model_ft_test(torch.flip(imgprob, dims=(2,3)).contiguous())
 p.append(F.softmax(logit, -1))
logit = model_ft_test(imgprob.permute(0,1,3,2 ).contiguous())
 p.append(F.softmax(logit, -1))
p1 = torch.stack(p ).mean(0)
 predict = p1.argmax(-1)
 if imagepath.endswith('.jpg'):
hash1[int(imagepath.rstrip('.jpg')) ] =(int(predict.cpu().numpy()))
else:
hash1[int(imagepath.rstrip('.JPG')) ] =(int(predict.cpu().numpy()))<save_to_csv>
app_train['TARGET'].value_counts()
print('The proportion of label 1 is %.2f' %(sum(app_train['TARGET']==1)/app_train.shape[0]*100), '%' )
for i in sorted(hash1):
res.append(hash1[i]+1)
 data = {'Id':list(range(1,351)) ,'LABEL':res}
df = pd.DataFrame(data)
 df.to_csv('submission.csv',index=None )<import_modules>
def missing_values_table(df):
mis_val = df.isnull().sum()
mis_val_percent = df.isnull().sum() * 100 / df.shape[0]
mis_val_table = pd.concat([mis_val, mis_val_percent], axis = 1)
 mis_val_table_rename_columns = mis_val_table.rename(columns = {0: 'Missing Values', 1: 'Percentage'})
 mis_val_table_rename_columns = mis_val_table_rename_columns[
mis_val_table_rename_columns.iloc[:, 1]!=0].sort_values('Percentage', ascending=False ).round(1)
 print('The total dataframe has ' + str(df.shape[1])+ ' columns')
 print('There are ' + str(mis_val_table_rename_columns.shape[0])+ ' columns')
 return mis_val_table_rename_columns
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os<define_variables>
missing_values = missing_values_table(app_train)
 missing_values.head(20 )
wd = "/kaggle/input/jamp-hackathon-drive-1/train_set/"<define_variables>
app_train.dtypes.value_counts()
images_dir=os.listdir(wd )<feature_engineering>
app_train.select_dtypes('object' ).apply(pd.Series.nunique, axis=0 )
df=pd.DataFrame()
df['Images']=data
classes=df["Images"].str.split("/", n = 6, expand = True)[5]
df['Label']=classes
df = df.sample(frac=1 ).reset_index(drop=True )<count_values>
app_test.dtypes.value_counts()
df['Label'].value_counts()<split>
app_train = pd.get_dummies(app_train)
 app_test = pd.get_dummies(app_test)
 print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )
train, test = train_test_split(df, test_size=0.2,stratify=df['Label'] )<set_options>
train_labels = app_train['TARGET']
app_train, app_test = app_train.align(app_test, join = 'inner', axis = 1)
 app_train['TARGET'] = train_labels
print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )
get_ipython().magic('matplotlib inline')
<import_modules>
app_train['DAYS_EMPLOYED_ANOM'] = app_train['DAYS_EMPLOYED'] == 365243
app_train['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)
 app_test['DAYS_EMPLOYED_ANOM'] = app_test['DAYS_EMPLOYED'] == 365243
app_test['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)
 print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )
from tqdm import tqdm
import pickle<categorify>
app_train_domain = app_train.copy()
app_test_domain = app_test.copy()
app_train_domain['CREDIT_INCOME_PERCENT'] = app_train_domain['AMT_CREDIT'] / app_train_domain['AMT_INCOME_TOTAL']
app_train_domain['ANNUITY_INCOME_PERCENT'] = app_train_domain['AMT_ANNUITY'] / app_train_domain['AMT_INCOME_TOTAL']
app_train_domain['CREDIT_TERM'] = app_train_domain['AMT_ANNUITY'] / app_train_domain['AMT_CREDIT']
app_train_domain['DAYS_EMPLOYED_PERCENT'] = app_train_domain['DAYS_EMPLOYED'] / app_train_domain['DAYS_BIRTH']
app_test_domain['CREDIT_INCOME_PERCENT'] = app_test_domain['AMT_CREDIT'] / app_test_domain['AMT_INCOME_TOTAL']
app_test_domain['ANNUITY_INCOME_PERCENT'] = app_test_domain['AMT_ANNUITY'] / app_test_domain['AMT_INCOME_TOTAL']
app_test_domain['CREDIT_TERM'] = app_test_domain['AMT_ANNUITY'] / app_test_domain['AMT_CREDIT']
app_test_domain['DAYS_EMPLOYED_PERCENT'] = app_test_domain['DAYS_EMPLOYED'] / app_test_domain['DAYS_BIRTH']
print('Domain Training Features shape: ', app_train_domain.shape)
 print('Domain Testing Features shape: ', app_test_domain.shape )
train_img=[]
for i in tqdm(df['Images']):
temp_img=image.load_img(i,target_size=(224,224))
temp_img=image.img_to_array(temp_img)
 train_img.append(temp_img)
 train_img=np.array(train_img)
 train_img=preprocess_input(train_img)
<feature_engineering>
bureau = pd.read_csv('.. /input/bureau.csv')
 bureau.head()
test_wd = "/kaggle/input/jamp-hackathon-drive-1/test_set/"
test_dir=os.listdir(test_wd)
 test_data=[]
for i in test_dir:
test_data.append(os.path.join(test_wd,i))
test_df=pd.DataFrame()
test_df['Images']=test_data
test_df.head()<normalization>
previous_loan_counts = bureau.groupby('SK_ID_CURR', as_index=False)['SK_ID_BUREAU'].count().rename(columns = {'SK_ID_BUREAU': 'previous_loan_counts'})
 previous_loan_counts.head()
test_img=[]
for i in tqdm(test_df['Images']):
temp_img=image.load_img(i,target_size=(224,224))
temp_img=image.img_to_array(temp_img)
 test_img.append(temp_img)
 test_img=np.array(test_img)
 test_img=preprocess_input(test_img )<choose_model_class>
def agg_numeric(df, group_var, df_name):

 for col in df:
if col != group_var and 'SK_ID' in col:
df = df.drop(columns = col)
 group_ids = df[group_var]
numeric_df = df.select_dtypes('number')
 numeric_df[group_var] = group_ids
agg = numeric_df.groupby(group_var ).agg(['count', 'mean', 'max', 'min', 'sum'] ).reset_index()
columns = [group_var]
for var in agg.columns.levels[0]:
if var != group_var:
for stat in agg.columns.levels[1][:-1]:
columns.append('%s_%s_%s' %(df_name, var, stat))
agg.columns = columns
return agg
bureau_agg = agg_numeric(bureau.drop(columns = ['SK_ID_BUREAU']), group_var = 'SK_ID_CURR', df_name = 'bureau')
 bureau_agg.head()
model = VGG16(weights='imagenet', include_top=False )<predict_on_test>
def count_categorical(df, group_var, df_name):

 categorical = pd.get_dummies(df.select_dtypes('object'))
categorical[group_var] = df[group_var]
categorical = categorical.groupby(group_var ).agg(['sum', 'mean'])
 column_names = []
for var in categorical.columns.levels[0]:
for stat in ['count', 'count_norm']:
column_names.append('%s_%s_%s' %(df_name, var, stat))
categorical.columns = column_names
return categorical
bureau_counts = count_categorical(bureau, group_var = 'SK_ID_CURR', df_name = 'bureau')
 bureau_counts.head()
features_train=model.predict(train_img )<predict_on_test>
train = app_train_domain.merge(previous_loan_counts, on = 'SK_ID_CURR', how = 'left')
 train['previous_loan_counts'] = train['previous_loan_counts'].fillna(0)
 test = app_test_domain.merge(previous_loan_counts, on = 'SK_ID_CURR', how = 'left')
 test['previous_loan_counts'] = test['previous_loan_counts'].fillna(0 )
features_test=model.predict(test_img )<prepare_x_and_y>
train = train.merge(bureau_agg, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_agg, on = 'SK_ID_CURR', how = 'left' )
train_x=features_train.reshape(1027,-1 )<prepare_x_and_y>
train = train.merge(bureau_counts, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_counts, on = 'SK_ID_CURR', how = 'left' )
test_x=features_test.reshape(256,-1 )<split>
print('Before align train.shape: ', train.shape)
 print('Before align test.shape: ', test.shape)
 train_labels = train['TARGET']
train, test = train.align(test, join = 'inner', axis = 1)
 train['TARGET'] = train_labels
print('After align train.shape: ', train.shape)
 print('After align test.shape: ', test.shape )
train_y=np.asarray(df['Label'])
 train_y=pd.get_dummies(train_y)
 train_y=np.array(train_y)
 X_train, X_valid, Y_train, Y_valid=train_test_split(train_x,train_y,test_size=0.3, random_state=42)
<categorify>
def train_with_cv(train_data, test_data, n_folds, seed_varying):
train_ids = train_data['SK_ID_CURR']
test_ids = test_data['SK_ID_CURR']
train_labels = train_data['TARGET']
train_features = train_data.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test_features = test_data.drop(columns = ['SK_ID_CURR'])
 feature_names = list(train_features.columns)
 feature_importance_values = np.zeros(len(feature_names))
train_features = np.array(train_features)
 test_features = np.array(test_features)
 k_fold = KFold(n_splits = n_folds, shuffle = True, random_state = 50+seed_varying)
 test_pred = np.zeros(test_features.shape[0])
 out_of_fold = np.zeros(train_features.shape[0])
 valid_scores = []
train_scores = []
for train_indices, valid_indices in k_fold.split(train_features):
x_train, y_train = train_features[train_indices], train_labels[train_indices]
x_valid, y_valid = train_features[valid_indices], train_labels[valid_indices]
model = lgb.LGBMClassifier(n_estimators=10000, objective = 'binary',
class_weight = 'balanced', learning_rate = 0.05,
reg_alpha = 0.1, reg_lambda = 0.1,
subsample = 0.8, n_jobs = -1, random_state = 50+seed_varying)
 model.fit(x_train, y_train, eval_metric = 'auc',
eval_set = [(x_valid, y_valid),(x_train, y_train)],
eval_names = ['valid', 'train'], categorical_feature = 'auto',
early_stopping_rounds = 100, verbose = -1)
 best_iteration = model.best_iteration_
feature_importance_values += model.feature_importances_ / k_fold.n_splits
test_pred += model.predict_proba(test_features, num_iteration = best_iteration)[:, 1] / k_fold.n_splits
out_of_fold[valid_indices] = model.predict_proba(x_valid, num_iteration = best_iteration)[:, 1]
valid_score = model.best_score_['valid']['auc']
train_score = model.best_score_['train']['auc']
valid_scores.append(valid_score)
 train_scores.append(train_score)
 gc.enable()
del model, x_train, y_train, x_valid, y_valid
gc.collect()
pred_score = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': test_pred})
 feature_importances = pd.DataFrame({'feature': feature_names, 'importance': feature_importance_values})
 valid_auc = roc_auc_score(train_labels, out_of_fold)
 valid_scores.append(valid_auc)
 train_scores.append(np.mean(train_scores))
fold_names = list(range(n_folds))
fold_names.append('overall')
 metrics = pd.DataFrame({'fold': fold_names,
'train': train_scores,
'valid': valid_scores})
 return pred_score, feature_importances, metrics
<choose_model_class><EOS> 
train_times = 3
n_folds = 5
i = 0
metrics_all = np.zeros(( train_times, 2))
for seed_varying in range(train_times):
print('
=======================================================')
 print('The ', seed_varying, ' time of train')
 print('
=======================================================')
 sub, fi, metrics = train_with_cv(train, test, n_folds, seed_varying)
 if i==0:
submission = sub
feat_
else:
submission['TARGET'] += sub['TARGET']
feat_import['importance'] += fi['importance']
metrics_all[i, :] = metrics.iloc[-1, 1:3]
i += 1
metrics_all_average = metrics_all.mean(axis = 0)
 metrics_all = np.row_stack([metrics_all, metrics_all_average])
 train_time_names = list(range(train_times))
train_time_names.append('Average')
 metrics_final = pd.DataFrame({'train_time': train_time_names,
'train': metrics_all[:,0],
'valid': metrics_all[:,1]})
 submission['TARGET'] = submission['TARGET'] / train_times
feat_import['importance'] = feat_import['importance'] / train_times
submission.to_csv('lightgbm_version_9.csv', index = False )