Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<install_modules>
warnings.filterwarnings('ignore')
!pip install iterative-stratification
!pip install tensorflow_addons<load_from_csv>
app_train = pd.read_csv('.. /input/application_train.csv')
 app_test = pd.read_csv('.. /input/application_test.csv')
 print('Training data shape: {}, Testing data shape: {}'.format(app_train.shape,app_test.shape))
df_target_train = pd.read_csv('/kaggle/input/mlclass-dubai-by-ods-lecture-5-hw/df_target_train.csv')
 print('df_target_train:', df_target_train.shape)
 df_sample_submit = pd.read_csv('/kaggle/input/mlclass-dubai-by-ods-lecture-5-hw/df_sample_submit.csv')
 print('df_sample_submit:', df_sample_submit.shape)
 df_tracks = pd.read_csv('/kaggle/input/mlclass-dubai-by-ods-lecture-5-hw/df_tracks.csv')
 print('df_tracks:', df_tracks.shape)
 df_genres = pd.read_csv('/kaggle/input/mlclass-dubai-by-ods-lecture-5-hw/df_genres.csv')
 print('df_genres:', df_genres.shape)
 df_features = pd.read_csv('/kaggle/input/mlclass-dubai-by-ods-lecture-5-hw/df_features.csv')
 print('df_features:', df_features.shape)
 df_target_train.head()<count_unique_values>
df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True )
trainids=df_target_train["track_id"].tolist()
testids=df_sample_submit["track_id"].tolist()
tr_album=df_tracks[(df_tracks["track_id"].isin(trainids)) ]["album:id"].unique()
te_album=df_tracks[(df_tracks["track_id"].isin(testids)) ]["album:id"].unique()
len(tr_album),len(te_album),len([t for t in te_album if t in tr_album]),len([t for t in te_album if t not in tr_album] )<count_unique_values>
app_train.select_dtypes('object' ).apply(pd.Series.nunique, axis = 0 )
trainids=df_target_train["track_id"].tolist()
testids=df_sample_submit["track_id"].tolist()
tr_album=df_tracks[(df_tracks["track_id"].isin(trainids)) ]["artist:id"].unique()
te_album=df_tracks[(df_tracks["track_id"].isin(testids)) ]["artist:id"].unique()
len(tr_album),len(te_album),len([t for t in te_album if t in tr_album]),len([t for t in te_album if t not in tr_album] )<count_unique_values>
print(df['CODE_GENDER'].value_counts())
 print('
Size Before {}'.format(df.shape))
df = df[df['CODE_GENDER'] != 'XNA']
print('Size After {}'.format(df.shape))
trainids=df_target_train["track_id"].tolist()
testids=df_sample_submit["track_id"].tolist()
tr_album=df_tracks[(df_tracks["track_id"].isin(trainids)) ]["track:language_code"].unique()
te_album=df_tracks[(df_tracks["track_id"].isin(testids)) ]["track:language_code"].unique()
len(tr_album),len(te_album),len([t for t in te_album if t in tr_album]),len([t for t in te_album if t not in tr_album]),[t for t in te_album if t not in tr_album]<count_values>
le = LabelEncoder()
le_count = 0
for col in df:
if df[col].dtype == 'object':
if len(list(df[col].unique())) <= 2:
le.fit(df[col])
 df[col] = le.transform(df[col])
 le_count += 1
print('{} variable are label encoded'.format(le_count))
categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= True)
 print('The shape of dataset after One hot encoding: {}'.format(df.shape))
df_tracks["album:type"].value_counts()<data_type_conversions>
df['DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']
df['PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']
<define_variables>
app_train['TARGET'].value_counts()
all_labels=[]
ys=[]
for i,row in df_target_train.iterrows() :
genres=row["track:genres"].split(" ")
 all_labels.extend(genres)
 all_labels=list(set(all_labels))
ys.append(genres)
 print(len(all_labels))
ys[:5]<categorify>
del app_test, app_train
gc.collect()
mlb = MultiLabelBinarizer()
y_mlb=mlb.fit_transform(ys)
 y_mlb.shape<feature_engineering>
def _get_categorical_features(df):
feats = [col for col in list(df.columns)if df[col].dtype == 'object']
return feats
def _factorize_categoricals(df, cats):
for col in cats:
df[col], _ = pd.factorize(df[col])
 return df
def _get_dummies(df, cats):
for col in cats:
df = pd.concat([df, pd.get_dummies(df[col], prefix=col)], axis=1)
 return df
mlb.inverse_transform(y_mlb)[:4]<merge>
bureau = pd.read_csv('.. /input/bureau.csv', nrows = None)
 bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = None )
print(df_target_train.shape,df_features.shape)
 train_X=df_target_train[["track_id"]].merge(df_features,how='inner',on='track_id')
 train_X=df_tracks[["track_id","track:bit_rate","track:duration","track:listens","track:comments","track:interest","album:tracks","artist:favorites"]].merge(train_X,how='inner',on="track_id" ).drop('track_id',axis=1)
 print(train_X.shape)
 print(df_sample_submit.shape)
 test_X=df_sample_submit[["track_id"]].merge(df_features,how='inner',on='track_id')
 test_X=df_tracks[["track_id","track:bit_rate","track:duration","track:listens","track:comments","track:interest","album:tracks","artist:favorites"]].merge(test_X,how='inner',on="track_id" ).drop('track_id',axis=1)
 print(test_X.shape)
<choose_model_class>
bereau_cats = _get_categorical_features(bureau)
 bb_cats = _get_categorical_features(bb)
 bureau = _get_dummies(bureau,bereau_cats)
 bb = _get_dummies(bb,bb_cats )
def build_model() :
num_input = layers.Input(shape=(train_X.shape[1],))
x = layers.Dense(300 )(num_input)
 x = BatchNormalization()(x)
 x = Dropout(0.1 )(x)
 x = Dense(300, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = BatchNormalization()(x)
 x = Dense(64, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = BatchNormalization()(x)
 y = Dense(y_mlb.shape[1], activation="sigmoid" )(x)
 model = Model(inputs=num_input, outputs=y)
 model.compile(loss='binary_crossentropy', optimizer='adam',metrics=[tfa.metrics.F1Score(num_classes=161,average="macro")])
 return model
model=build_model()
model.summary()<categorify>
bureau_avg = bureau.groupby('SK_ID_CURR' ).mean()
bureau_avg['buro_count'] = bureau[['SK_ID_BUREAU','SK_ID_CURR']].groupby('SK_ID_CURR' ).count() ['SK_ID_BUREAU']
bureau_avg.columns = ['b_' + f_ for f_ in bureau_avg.columns]
df = df.merge(right=bureau_avg.reset_index() , how='left', on='SK_ID_CURR')
 del bb, bureau_avg
gc.collect()
def decode(y_pred):
inverse=mlb.inverse_transform(y_pred.round())
 maxes=np.max(y_pred,axis=1)
 for i in range(len(inverse)) :
if(inverse[i]):
pass
else:
args=np.argpartition(y_pred[i], -4)[-4:]
k=y_pred[i]
k[args]=1
y_pred[i]=k
inverse2=mlb.inverse_transform(y_pred.round())
 return inverse2<prepare_x_and_y>
prev = pd.read_csv('.. /input/previous_application.csv', nrows = None )
NFOLDS=8
sc=StandardScaler()
sc.fit(pd.concat([train_X,test_X],axis=0 ).values)
 X=sc.transform(train_X.values)
 X_test=sc.transform(test_X.values)
 y=y_mlb
mskf=MultilabelStratifiedKFold(n_splits=NFOLDS)
 validationdf=pd.DataFrame({"track_id":df_target_train["track_id"]})
 predictiondf=pd.DataFrame({"track_id":df_sample_submit["track_id"]})
 scores=[]
predictions=np.zeros(( X_test.shape[0],y.shape[1]))
for fold,(trn_,val_)in enumerate(mskf.split(X,y)) :
print(fold)
 X_train, X_val = X[trn_], X[val_]
y_train, y_val = y[trn_], y[val_]
model=build_model()
rlr=ReduceLROnPlateau(monitor='val_f1_score',factor=0.1,patience=2, min_lr=1e-12, mode='max', verbose=1)
 es = EarlyStopping(monitor='val_f1_score',mode='max',min_delta=0.00001, patience=5,verbose=1, baseline=None,restore_best_weights=True)
 history= model.fit(
 x=X_train,
y=y_train,
validation_data=(X_val,y_val),
epochs=100,
batch_size=80,
callbacks=[es,rlr],
verbose=1 
)
 predictions+=model.predict(X_test)
 predictions=predictions/NFOLDS 
<string_transform>
prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True )
predictions_decoded=decode(predictions)
 predictions_decoded[:2]<feature_engineering>
prev_app_cats = _get_categorical_features(prev)
 prev = _get_dummies(prev, prev_app_cats )
def format_output(x):
return " ".join(list(x))
df_sample_submit["track:genres"]=predictions_decoded
df_sample_submit["track:genres"]=df_sample_submit["track:genres"].apply(format_output)
 df_sample_submit.head()<count_unique_values>
prev_apps_count = prev[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
prev['SK_ID_PREV'] = prev['SK_ID_CURR'].map(prev_apps_count['SK_ID_PREV'])
 prev_apps_avg = prev.groupby('SK_ID_CURR' ).mean()
prev_apps_avg.columns = ['p_' + col for col in prev_apps_avg.columns]
df = df.merge(right=prev_apps_avg.reset_index() , how='left', on='SK_ID_CURR')
 del prev, prev_apps_avg
gc.collect()
trainids=df_target_train["track_id"].tolist()
testids=df_sample_submit["track_id"].tolist()
tr_album=df_tracks[(df_tracks["track_id"].isin(trainids)) ]["album:id"].unique()
te_album=df_tracks[(df_tracks["track_id"].isin(testids)) ]["album:id"].unique()
print(len(tr_album),len(te_album),len([t for t in te_album if t in tr_album]),len([t for t in te_album if t not in tr_album]))
matching_albums=[t for t in te_album if t in tr_album and t!=-1]
print(len(matching_albums)) 
<merge>
pos = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows = None )
t=df_tracks[["track_id","album:id"]].merge(df_target_train,how='inner',on=["track_id"] ).drop("track_id",axis=1)
 t.head()
albs_genres={}
for alb, grp in t.groupby("album:id"):
if(alb not in matching_albums):
continue
albs_genres[alb]=format_output(mlb.inverse_transform(np.max(mlb.transform(grp["track:genres"].apply(lambda x: x.split(" ")).tolist()),axis=0 ).reshape(1,-1)) [0])
 len(list(albs_genres))<merge>
pos_cats = _get_categorical_features(pos)
 pos = _get_dummies(pos, pos_cats )
df_sample_submit=df_sample_submit.merge(df_tracks[["track_id","album:id"]],on="track_id",how='inner')
 df_sample_submit["track:genres"]=df_sample_submit.apply(lambda x: albs_genres[x["album:id"]] if int(x["album:id"])in albs_genres else x["track:genres"],axis=1)
 df_sample_submit.head(10 )<count_unique_values>
pos_count = pos[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
pos['SK_ID_PREV'] = pos['SK_ID_CURR'].map(pos_count['SK_ID_PREV'])
 pos_avg = pos.groupby('SK_ID_CURR' ).mean()
df = df.merge(right=pos_avg.reset_index() , how='left', on='SK_ID_CURR')
 del pos, pos_avg
gc.collect()
<save_to_csv>
ins = pd.read_csv('.. /input/installments_payments.csv', nrows = None )
df_sample_submit[["track_id","track:genres"]].to_csv("submission.csv",index=None)
 df_sample_submit[["track_id","track:genres"]]<import_modules>
ins_cats = _get_categorical_features(ins)
 ins = _get_dummies(ins, ins_cats )
import numpy as np
import pandas as pd
import os
from subprocess import check_output
import keras
from keras.models import Sequential
from keras.utils import to_categorical
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.layers.normalization import BatchNormalization
from sklearn.model_selection import train_test_split 
<load_from_csv>
ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0 )
data_train = pd.read_csv('.. /input/train42000.csv')
 data_test = pd.read_csv('.. /input/test28000.csv' )<prepare_x_and_y>
cnt_inst = ins[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
ins['SK_ID_PREV'] = ins['SK_ID_CURR'].map(cnt_inst['SK_ID_PREV'])
 avg_inst = ins.groupby('SK_ID_CURR' ).mean()
avg_inst.columns = ['i_' + f_ for f_ in avg_inst.columns]
df = df.merge(right=avg_inst.reset_index() , how='left', on='SK_ID_CURR')
 del ins, avg_inst
gc.collect()
X = np.array(data_train.iloc[:, 1:])
 y = to_categorical(np.array(data_train.iloc[:, 0]))<split>
cc = pd.read_csv('.. /input/credit_card_balance.csv', nrows = None )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=13)
 X_test = np.array(data_test.iloc[:, 0:] )<train_model>
ccbal_cats = _get_categorical_features(cc)
 credit_card_balance = _get_dummies(cc, ccbal_cats )
img_rows, img_cols = 28, 28
input_shape =(img_rows, img_cols, 1)
 X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
 X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
 X_val = X_val.reshape(X_val.shape[0], img_rows, img_cols, 1 )<data_type_conversions>
nb_prevs = credit_card_balance[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
credit_card_balance['SK_ID_PREV'] = credit_card_balance['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 avg_cc_bal = credit_card_balance.groupby('SK_ID_CURR' ).mean()
avg_cc_bal.columns = ['cc_bal_' + f_ for f_ in avg_cc_bal.columns]
df = df.merge(right=avg_cc_bal.reset_index() , how='left', on='SK_ID_CURR')
 del cc, avg_cc_bal
gc.collect()
X_train = X_train.astype('float32')
 X_test = X_test.astype('float32')
 X_val = X_val.astype('float32')
 X_train /= 255
X_test /= 255
X_val /= 255<define_variables>
from lightgbm import LGBMClassifier
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.model_selection import KFold, StratifiedKFold
batch_size = 256
num_classes = 10
epochs = 5<choose_model_class>
train_df = df[df['TARGET'].notnull() ]
test_df = df[df['TARGET'].isnull() ]
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
model = Sequential()
model.add(Conv2D(20, kernel_size=(5, 5),
activation='relu',
kernel_initializer='he_normal',
input_shape=input_shape))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Flatten())
 model.add(Dense(20, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))<choose_model_class>
folds = KFold(n_splits=2, shuffle=True, random_state=1001 )
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam() ,
metrics=['accuracy'] )<train_model>
oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]
history = model.fit(X_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(X_val, y_val))<predict_on_test>
for train_indices, valid_indices in folds.split(train_df[feats],train_df['TARGET']):
train_features, train_labels = train_df[feats].iloc[train_indices], train_df['TARGET'].iloc[train_indices]
valid_features, valid_labels = train_df[feats].iloc[valid_indices], train_df['TARGET'].iloc[valid_indices]
model = LGBMClassifier(n_estimators=5000, objective = 'binary',
class_weight = 'balanced', learning_rate = 0.05,
reg_alpha = 0.1, reg_lambda = 0.1,
subsample = 0.8, n_jobs = -1, random_state = 50)
 model.fit(train_features, train_labels, eval_metric = 'auc',
eval_set = [(valid_features, valid_labels),(train_features, train_labels)],
eval_names = ['valid', 'train'],
early_stopping_rounds = 100, verbose = 200)
 oof_preds[valid_indices] = model.predict_proba(valid_features, num_iteration=model.best_iteration_)[:, 1]
sub_preds += model.predict_proba(test_df[feats], num_iteration=model.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = model.feature_importances_
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 del model, train_features, train_labels, valid_features, valid_labels
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))
<save_to_csv><EOS> 
test_df['TARGET'] = sub_preds
test_df[['SK_ID_CURR', 'TARGET']].to_csv('submission.csv', index= False )