Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<compute_test_metric>
warnings.simplefilter(action='ignore', category=FutureWarning)
 py.init_notebook_mode(connected=True)
 %matplotlib inline
cf.go_offline()
cv_score = accuracy_score(train_df[TARGET], np.argmax(oof, axis=1))
logger.info(f'CV: {cv_score:.5f}' )<categorify>
def one_hot_encoder(df, nan_as_category = True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns
test_transform = A.Compose([
A.Resize(IMSIZE, IMSIZE, p=1),
A.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225)) ,
ToTensorV2() ,
])
 test_dataloader = get_dataloader(
 submission_df[ID].apply(lambda x: os.path.join(PATH['test_image_dir'], x)) ,
submission_df[TARGET],
transform=test_transform,
with_memory_cache=True,
batch_size=params['test_batch_size'],
shuffle=False,
pin_memory=True,
)<predict_on_test>
num_rows = None
nan_as_category = True
predictions = np.zeros(( len(submission_df), 10))
for state_dict in best_state_dicts:
model = Model().to(DEVICE)
 model.load_state_dict(state_dict)
 model.eval()
_predictions = []
for x, _ in test_dataloader:
x = x.to(dtype=torch.float32, device=DEVICE)
 with torch.no_grad() :
y_pred = model(x)
 _predictions.append(y_pred.cpu().numpy())
 _predictions = np.concatenate(_predictions)
 predictions += _predictions / len(best_state_dicts )<prepare_output>
print("Start Train Test................." )
np.save('oof', oof)
 np.save('predictions', predictions)
 saved_best_state_dicts = {}
for i, bsd in enumerate(best_state_dicts):
saved_best_state_dicts[f'f{i}'] = bsd
torch.save(saved_best_state_dicts, 'best_state_dicts.pth')
 submission_df[TARGET] = np.argmax(predictions, axis=1 ).tolist()<save_to_csv>
df = pd.read_csv('.. /input/application_train.csv', nrows= num_rows)
 test_df = pd.read_csv('.. /input/application_test.csv', nrows= num_rows)
 print("Train samples: {}, test samples: {}".format(len(df), len(test_df)))
 df = df.append(test_df ).reset_index()
del test_df
gc.collect()
submission_df.to_csv('submission.csv', index=False)
 FileLink('submission.csv' )<load_from_csv>
for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:
df[bin_feature], uniques = pd.factorize(df[bin_feature])
 df, cat_cols = one_hot_encoder(df, nan_as_category)
 df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True )
train_users_game1 = pd.read_csv('/kaggle/input/ds2019uec-task2/train_users_game1.csv')
 train_users_game2 = pd.read_csv('/kaggle/input/ds2019uec-task2/train_users_game2.csv')
 test_users_game1 = pd.read_csv('/kaggle/input/ds2019uec-task2/test_users_game1.csv')
 test_user_ids = pd.read_csv('/kaggle/input/ds2019uec-task2/test_user_ids.csv')
 game_group2 = pd.read_csv('/kaggle/input/ds2019uec-task2/game_group2.csv')
 sample_submission = pd.read_csv('/kaggle/input/ds2019uec-task2/sample_submission.csv' )<count_duplicates>
df['DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']
users_game1 = pd.concat([train_users_game1, test_users_game1] ).drop(['play_hour'], axis=1 ).drop_duplicates()
users_game1['label'] = 1<concatenate>
a = df['DAYS_EMPLOYED_PERC'].tolist()
a = [x for x in a if str(x)!= 'nan']
b = df['INCOME_CREDIT_PERC'].tolist()
b = [x for x in b if str(x)!= 'nan']
c = df['INCOME_PER_PERSON'].tolist()
c = [x for x in c if str(x)!= 'nan']
d = df['ANNUITY_INCOME_PERC'].tolist()
d = [x for x in d if str(x)!= 'nan']
user_ids = pd.concat([train_users_game1['user_id'], train_users_game2['user_id'], test_user_ids['user_id']] ).unique()<drop_column>
print("End Train Test.................. " )
users_game1_matrix = users_game1.set_index(['user_id', 'game_title'])['label'].unstack().reindex(user_ids ).fillna(0 )<concatenate>
print("Start Bureau................ " )
train_user_ids = pd.concat([train_users_game1['user_id'], train_users_game2['user_id']] ).unique()
test_user_ids = test_user_ids['user_id'].values<compute_test_metric>
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows )
users_similarity = 1 - cosine_distances(users_game1_matrix.loc[test_user_ids], users_game1_matrix.loc[train_user_ids] )<remove_duplicates>
bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category )
users_game2 = train_users_game2.drop(['play_hour', 'predict_game_id'], axis=1 ).drop_duplicates()
users_game2['label'] = 1<filter>
bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(columns= 'SK_ID_BUREAU', inplace= True)
 del bb, bb_agg
gc.collect()
test_users_game2_matrix =(users_similarity @ users_game2_matrix.loc[train_user_ids] )<data_type_conversions>
num_aggregations = {
'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],
'CREDIT_DAY_OVERDUE': ['max', 'mean'],
'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'DAYS_CREDIT_UPDATE': ['min', 'max', 'mean'],
'AMT_ANNUITY': ['max', 'mean'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}
sample_submission['purchased_games'] = test_users_game2_matrix.apply(lambda x: ' '.join(x.sort_values(ascending=False)[:10].index.astype('str')) , axis=1 )<data_type_conversions>
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ] )
sample_submission.loc[test_users_game2_matrix.sum(axis=1)== 0, 'purchased_games'] = ' '.join(train_users_game2['predict_game_id'].value_counts(normalize=True ).index.astype('str')[:10] )<save_to_csv>
active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACT_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLS_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left')
 del closed, closed_agg, bureau
gc.collect()
sample_submission.to_csv('submission.csv', index=None )<import_modules>
print("End Bureau................ " )
import pandas as pd
import numpy as np
from sklearn import tree
import graphviz
from sklearn.model_selection import cross_val_score<load_from_csv>
print("Start previous_application................ " )
df = pd.read_csv('/kaggle/input/predict-the-income-bi-hack/train.csv')
 dft = pd.read_csv('/kaggle/input/predict-the-income-bi-hack/test.csv')
 print('train dataset length',len(df),'
test dataset length',len(dft)) 
<drop_column>
prev = pd.read_csv('.. /input/previous_application.csv', nrows = num_rows )
ids=dft['ID']
df.drop('ID', axis=1,inplace =True)
 dft.drop('ID', axis=1,inplace =True )<categorify>
prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)
 prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True )
df = pd.get_dummies(df, columns=['Work', 'Education', 'Marital_Status', 'Occupation', 'Relationship', 'Race', 'Gender','Nationality'])
 dft = pd.get_dummies(dft, columns=['Work', 'Education', 'Marital_Status', 'Occupation', 'Relationship', 'Race', 'Gender','Nationality'] )<drop_column>
prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']
df.drop('Nationality_Holand-Netherlands',axis=1, inplace=True )<count_values>
num_aggregations = {
'AMT_ANNUITY': ['min', 'max', 'mean'],
'AMT_APPLICATION': ['min', 'max', 'mean'],
'AMT_CREDIT': ['min', 'max', 'mean'],
'APP_CREDIT_PERC': ['min', 'max', 'mean', 'var'],
'AMT_DOWN_PAYMENT': ['min', 'max', 'mean'],
'AMT_GOODS_PRICE': ['min', 'max', 'mean'],
'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
}
df['Income'].value_counts()<feature_engineering>
cat_aggregations = {}
for cat in cat_cols:
cat_aggregations[cat] = ['mean']
prev_agg = prev.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 prev_agg.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist() ] )
df["Income"] = df["Income"].replace({'<=50K':1,'>50K':0} )<create_dataframe>
approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
approved_agg = approved.groupby('SK_ID_CURR' ).agg(num_aggregations)
 approved_agg.columns = pd.Index(['APR_' + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist() ])
 prev_agg = prev_agg.join(approved_agg, how='left')
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
refused_agg = refused.groupby('SK_ID_CURR' ).agg(num_aggregations)
 refused_agg.columns = pd.Index(['REF_' + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist() ])
 prev_agg = prev_agg.join(refused_agg, how='left')
 del refused, refused_agg, approved, approved_agg, prev
gc.collect()
d_train = df.copy()
d_test = dft.copy()
df1 = df<prepare_x_and_y>
print("End previous_application................ " )
X = d_train.drop('Income', axis=1)
 y = d_train['Income']<import_modules>
print("Start POS_CASH_balance................ " )
from sklearn.model_selection import train_test_split<import_modules>
pos = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows = num_rows )
from sklearn.model_selection import train_test_split<split>
pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)
 aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size'],
'SK_DPD': ['max', 'mean'],
'SK_DPD_DEF': ['max', 'mean']
}
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.30,random_state=42 )<choose_model_class>
for cat in cat_cols:
aggregations[cat] = ['mean']
pos_agg = pos.groupby('SK_ID_CURR' ).agg(aggregations)
 pos_agg.columns = pd.Index(['POS_' + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR' ).size()
del pos
gc.collect()
t = tree.DecisionTreeClassifier(criterion='entropy', max_depth=7 )<train_model>
print("Start POS_CASH_balance................ " )
t = t.fit(X_train,y_train )<compute_test_metric>
ins = pd.read_csv('.. /input/installments_payments.csv', nrows = num_rows)
 ins, cat_cols = one_hot_encoder(ins, nan_as_category= True )
t.score(X_test,y_test )<predict_on_test>
ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0 )
sol=t.predict(dft)
 print(sol )<save_to_csv>
aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DPD': ['max', 'mean', 'sum'],
'DBD': ['max', 'mean', 'sum'],
'PAYMENT_PERC': ['max', 'mean', 'sum', 'var'],
'PAYMENT_DIFF': ['max', 'mean', 'sum', 'var'],
'AMT_INSTALMENT': ['max', 'mean', 'sum'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
ins_agg = ins.groupby('SK_ID_CURR' ).agg(aggregations)
 ins_agg.columns = pd.Index(['INS_' + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist() ])
 ins_agg['INS_COUNT'] = ins.groupby('SK_ID_CURR' ).size()
del ins
gc.collect()
with open('income-predicted.csv','w')as fw:
fw.write('ID,Income
')
 ct=24001
for i in sol:
s=""
if i==1:
s="<=50K"
else:
s=">50K"
fw.write(str(ct)+','+str(s)+'
')
 ct+=1<load_from_csv>
print("End POS_CASH_balance................ " )
sdf = pd.read_csv('income-predicted.csv', sep=',')
 sdf.head()<count_values>
print("Start credit_card_balance................ " )
sdf.Income.value_counts()<load_from_csv>
cc = pd.read_csv('.. /input/credit_card_balance.csv', nrows = num_rows )
%%capture
train = pd.read_csv('.. /input/fakenewsvortexbsb/train_df.csv', sep=';', error_bad_lines=False, quoting=3);<categorify>
cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)
 cc.drop(columns = ['SK_ID_PREV'], inplace = True)
 cc_agg = cc.groupby('SK_ID_CURR' ).agg(['min', 'max', 'mean', 'sum', 'var'])
 cc_agg.columns = pd.Index(['CC_' + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist() ])
 cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR' ).size()
del cc
gc.collect()
example = train["manchete"][1]
print(unidecode(example))<categorify>
print("End credit_card_balance................ " )
letters_only=re.sub("[^a-zA-Z]"," ",unidecode(example))
print(letters_only )<string_transform>
with timer("Process bureau and bureau_balance"):
print("Bureau df shape:", bureau_agg.shape)
 df = df.join(bureau_agg, how='left',on='SK_ID_CURR')
 gc.collect()
with timer("Process previous_applications"):
print("Previous applications df shape:", prev_agg.shape)
 df = df.join(prev_agg, how='left', on='SK_ID_CURR')
 gc.collect()
with timer("Process POS-CASH balance"):
print("Pos-cash balance df shape:", pos_agg.shape)
 df = df.join(pos_agg, how='left', on='SK_ID_CURR')
 gc.collect()
with timer("Process installments payments"):
print("Installments payments df shape:", ins_agg.shape)
 df = df.join(ins_agg, how='left', on='SK_ID_CURR')
 gc.collect()
with timer("Process credit card balance"):
print("Credit card balance df shape:", cc_agg.shape)
 df = df.join(cc_agg, how='left', on='SK_ID_CURR')
 gc.collect()
del bureau_agg,prev_agg,pos_agg,ins_agg,cc_agg
gc.collect()
lower_case=letters_only.lower()
words=lower_case.split()<string_transform>
print("Done.;.............. ")
print(stopwords.words("portuguese"))<string_transform>
train_df = df[df['TARGET'].notnull() ]
test_df = df[df['TARGET'].isnull() ]
stop = stopwords.words("portuguese" )<categorify>
train_df = train_df.drop(['index'],axis=1)
 test_df = test_df.drop(['index','TARGET'],axis=1)
 train_df = train_df.fillna(0)
 test_df = test_df.fillna(0 )
lista_stop = [unidecode(x)for x in stop]
print(lista_stop )<define_variables>
label = u'TARGET'
a = list(train_df.columns)
 a.remove(label)
 labels = train_df[label]
data_only = train_df[list(a)]
col_name = data_only.columns
X_train, X_test, y_train, y_test = train_test_split(data_only, labels, test_size=0.1,random_state = 42 )
words=[w for w in words if not w in lista_stop]
print(words )<string_transform>
clf_catboost = CatBoostClassifier(iterations=1200,
learning_rate=0.1,
depth=7,
l2_leaf_reg=40,
bootstrap_type='Bernoulli',
subsample=0.7,
scale_pos_weight=5,
eval_metric='AUC',
metric_period=50,
od_type='Iter',
od_wait=45,
random_seed=15,
allow_writing_files=False)
 clf_catboost.fit(data_only,labels,verbose=True )
def review_to_words(raw_review):
raw_review = unidecode(raw_review)
 raw_review.lstrip('Jovem Pan')
 letters_only=re.sub("[^a-zA-Z]"," ",raw_review)
 words=letters_only.lower().split()
meaningful_words=[w for w in words if not w in lista_stop]
return(' '.join(meaningful_words))<string_transform>
pred = clf_catboost.predict_proba(test_df)
 test_df['TARGET'] = pred[:, 0]
<find_best_params><EOS> 
test_df[['SK_ID_CURR', 'TARGET']].to_csv('submission_catboost1.csv', index= False )