Kaggle competition: Rossmann Store Sales
<SOS> metric: RootMeanSquarePercentageError Kaggle data source: rossmann-store-sales<categorify>
%matplotlib inline
for c in previous_application.columns:
if previous_application[c].dtypes==object:
LE = LabelEncoder()
previous_application[c] = LE.fit_transform(list(previous_application[c].values.astype('str')))
 aggregations = {
'AMT_ANNUITY': ['min', 'max', 'mean'],
'AMT_APPLICATION': ['min', 'max', 'mean'],
'AMT_CREDIT': ['min', 'max', 'mean'],
'APP_CREDIT_PERC': ['min', 'max', 'mean', 'var'],
'AMT_DOWN_PAYMENT': ['min', 'max', 'mean'],
'AMT_GOODS_PRICE': ['min', 'max', 'mean'],
'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'var'],
}
prev_agg = previous_application.groupby('SK_ID_CURR' ).agg({**aggregations})
 prev_agg.columns = pd.Index(['PREV_' + col[0] + "_" + col[1].upper() for col in prev_agg.columns.tolist() ])
 del previous_application
gc.collect()
df = pd.merge(df,prev_agg, how='left', on='SK_ID_CURR')
 del prev_agg
gc.collect()<categorify>
df_train = pd.read_csv(".. /input/train.csv", parse_dates=["Date"], date_parser=pd.to_datetime, low_memory=False)
 df_test = pd.read_csv(".. /input/test.csv", parse_dates=["Date"], date_parser=pd.to_datetime, low_memory=False)
 df_store = pd.read_csv(".. /input/store.csv" )
for c in POS_CASH_balance.columns:
if POS_CASH_balance[c].dtypes==object:
LE = LabelEncoder()
POS_CASH_balance[c] = LE.fit_transform(list(POS_CASH_balance[c].values.astype('str')))
 aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size'],
'SK_DPD': ['max', 'mean', 'var'],
'SK_DPD_DEF': ['max', 'mean', 'var']
}
pos_agg = POS_CASH_balance.groupby('SK_ID_CURR' ).agg(aggregations)
 pos_agg.columns = pd.Index(['POS_' + col[0] + "_" + col[1].upper() for col in pos_agg.columns.tolist() ])
 pos_agg['POS_COUNT'] = POS_CASH_balance.groupby('SK_ID_CURR' ).size()
del POS_CASH_balance
gc.collect()
df = pd.merge(df,pos_agg, how='left', on='SK_ID_CURR')
 del pos_agg
gc.collect()<categorify>
train = df_train.merge(df_store)
 test = df_test.merge(df_store )
for c in installments_payments.columns:
if installments_payments[c].dtypes==object:
LE = LabelEncoder()
installments_payments[c] = LE.fit_transform(list(installments_payments[c].values.astype('str')))
 aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DPD': ['max', 'mean', 'sum'],
'DBD': ['max', 'mean', 'sum'],
'PAYMENT_PERC': ['max', 'mean', 'sum', 'var'],
'PAYMENT_DIFF': ['max', 'mean', 'sum', 'var'],
'AMT_INSTALMENT': ['max', 'mean', 'sum'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']
}
ins_agg = installments_payments.groupby('SK_ID_CURR' ).agg(aggregations)
 ins_agg.columns = pd.Index(['INSTAL_' + col[0] + "_" + col[1].upper() for col in ins_agg.columns.tolist() ])
 ins_agg['INSTAL_COUNT'] = installments_payments.groupby('SK_ID_CURR' ).size()
del installments_payments
gc.collect()
df = pd.merge(df,ins_agg, how='left', on='SK_ID_CURR')
 del ins_agg
gc.collect()<merge>
train = df_train.merge(df_store)
 test = df_test.merge(df_store )
for c in credit_card_balance.columns:
if credit_card_balance[c].dtypes==object:
LE = LabelEncoder()
credit_card_balance[c] = LE.fit_transform(list(credit_card_balance[c].values.astype('str')))
 credit_card_balance.drop(['SK_ID_PREV'], axis= 1, inplace = True)
 cc_agg = credit_card_balance.groupby('SK_ID_CURR' ).agg(['min', 'max', 'mean', 'sum', 'var'])
 cc_agg.columns = pd.Index(['CC_' + col[0] + "_" + col[1].upper() for col in cc_agg.columns.tolist() ])
 cc_agg['CC_COUNT'] = credit_card_balance.groupby('SK_ID_CURR' ).size()
df = pd.merge(df,cc_agg, how='left', on='SK_ID_CURR')
 del credit_card_balance, cc_agg
gc.collect()<drop_column>
def get_Promo2Active(df):
months_map = {v:i+1 for i, v in enumerate(["Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sept", "Oct", "Nov", "Dec"])}
def is_Promo2_active(row):
if row.Promo2 == 0:
return 0
current_week, current_month, current_year = row.Date.week, row.Date.month, row.Date.year
start_week, start_year = row.Promo2SinceWeek, row.Promo2SinceYear
active_months = set([months_map[m] for m in row.PromoInterval.split(",")])
 has_started =(current_year == start_year and current_week >= start_week)or current_year > start_year
return int(has_started and current_month in active_months)
 return df.apply(is_Promo2_active, axis=1 )
df_train, df_test = df.iloc[:len(application_train)], df.iloc[len(application_train):]
del application_train, application_test, df
gc.collect()<prepare_x_and_y>
def get_CompetitionActive(df):
def is_competition_active(row):
if np.isnan(row.CompetitionDistance):
return 0
if np.isnan(row.CompetitionOpenSinceMonth)and np.isnan(row.CompetitionOpenSinceYear):
return 1
current_month, current_year = row.Date.month, row.Date.year
opened_month, opened_year = row.CompetitionOpenSinceMonth, row.CompetitionOpenSinceYear
return int(( current_year == opened_year and current_month >= opened_month)or current_year > opened_year)
 return df.apply(is_competition_active, axis=1 )
folds = StratifiedKFold(n_splits= 10, shuffle=True, random_state=2020)
 sub_preds = np.zeros(df_test.shape[0])
 feats = [f for f in df_train.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(df_train[feats], df_train['TARGET'])) :
train_x, train_y = df_train[feats].iloc[train_idx], df_train['TARGET'].iloc[train_idx]
valid_x, valid_y = df_train[feats].iloc[valid_idx], df_train['TARGET'].iloc[valid_idx]
lgb = LGBMClassifier(nthread=4, n_estimators=12000, learning_rate=0.02, num_leaves=31,
colsample_bytree=0.85,subsample=0.9, max_depth=8, reg_alpha=0.0415, reg_lambda=0.073,
min_split_gain=0.022, min_child_weight=39.32, silent=-1, verbose=-1)
 lgb.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'auc', verbose= 200, early_stopping_rounds= 100)
 sub_preds += lgb.predict_proba(df_test[feats], num_iteration=lgb.best_iteration_)[:, 1] / folds.n_splits
del lgb, train_x, train_y, valid_x, valid_y
gc.collect()
df_test['TARGET'] = sub_preds<save_to_csv>
train["Promo2Active"] = get_Promo2Active(train)
 train["CompetitionActive"] = get_CompetitionActive(train )
df_test[['SK_ID_CURR', 'TARGET']].to_csv('submission', index= False )<feature_engineering>
train[train.Promo2 == 1].groupby("Promo2Active" ).agg({"Sales": "mean"} ).plot(kind="bar" )
pd.set_option('display.max_columns', None)
 %matplotlib inline
bureau_balance = pd.read_csv('.. /input/bureau_balance.csv')
 bureau_balance['STATUS_mod'] = bureau_balance.STATUS.map({'0':0, '1':1, '2':2, '3':3, '4':4, '5':5, 'X':np.nan, 'C':0} ).map(lambda x: 0 if x=='C' else x ).interpolate(method = 'linear')
 bureau_balance['write_off'] = bureau_balance.STATUS.map(lambda x: 1 if x=='5' else 0)
 bureau_balance['adj_score'] =(bureau_balance.MONTHS_BALANCE-bureau_balance.MONTHS_BALANCE.min() +1)*bureau_balance.STATUS_mod
bb_month_count = bureau_balance.groupby('SK_ID_BUREAU')['MONTHS_BALANCE'].count()
bb_dpd_sum = bureau_balance.groupby('SK_ID_BUREAU')['STATUS_mod'].sum()
bb_write_off = bureau_balance.groupby('SK_ID_BUREAU')['write_off'].sum()
bb_dpd_sum_2_year = bureau_balance.loc[bureau_balance.MONTHS_BALANCE>=-24].groupby('SK_ID_BUREAU')['STATUS_mod'].sum()
bb_write_off_2_year = bureau_balance.loc[bureau_balance.MONTHS_BALANCE>=-24].groupby('SK_ID_BUREAU')['write_off'].sum()
bb_adj_score = bureau_balance.groupby('SK_ID_BUREAU')['adj_score'].sum()
bb_feature = pd.DataFrame({'bb_month_count':bb_month_count, 'bb_dpd_sum':bb_dpd_sum, 'bb_write_off':bb_write_off, 'bb_dpd_sum_2_year':bb_dpd_sum_2_year,
'bb_write_off_2_year':bb_write_off_2_year, 'bb_adj_score': bb_adj_score} ).reset_index().fillna(0)
 del bb_month_count, bb_dpd_sum, bb_write_off, bb_dpd_sum_2_year, bb_write_off_2_year, bb_adj_score, bureau_balance
gc.collect()
bureau = pd.read_csv('.. /input/bureau.csv')
 bureau = bureau.sort_values(['SK_ID_CURR', 'DAYS_CREDIT'])
 bureau['ADJ_DAYS'] =(bureau.DAYS_CREDIT - bureau.DAYS_CREDIT.min())/(bureau.DAYS_CREDIT.max() - bureau.DAYS_CREDIT.min())+ 0.5
bur_ncount = bureau.groupby('SK_ID_CURR')['SK_ID_BUREAU'].count()
bur_act_count = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['SK_ID_BUREAU'].count()
bur_bad_count = bureau.loc[bureau.CREDIT_ACTIVE=='Bad debt'].groupby('SK_ID_CURR')['SK_ID_BUREAU'].count()
bur_sold_count = bureau.loc[bureau.CREDIT_ACTIVE=='Sold out'].groupby('SK_ID_CURR')['SK_ID_BUREAU'].count()
bur_recent_application = -bureau.groupby('SK_ID_CURR')['DAYS_CREDIT'].max()
bur_eariliest_application = -bureau.groupby('SK_ID_CURR')['DAYS_CREDIT'].min()
bur_max_enddate = -bureau.groupby('SK_ID_CURR')['DAYS_CREDIT_ENDDATE'].max()
bureau['application_interval'] = bureau.groupby('SK_ID_CURR')['DAYS_CREDIT'].diff(-1)
 missing_iter = iter(bureau.groupby('SK_ID_CURR')['DAYS_CREDIT'].max())
 bureau.application_interval = bureau.application_interval.map(lambda x: -next(missing_iter)if np.isnan(x)else -x)
 bur_avg_intervel = bureau.groupby('SK_ID_CURR')['application_interval'].mean()
bur_sd_intervel = bureau.groupby('SK_ID_CURR')['application_interval'].agg('std' ).fillna(0)
 bur_max_overdue_days = bureau.groupby('SK_ID_CURR')['CREDIT_DAY_OVERDUE'].max()
bur_active_total_overdue_days = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['CREDIT_DAY_OVERDUE'].sum()
bur_active_max_overdue_days = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['CREDIT_DAY_OVERDUE'].max()
bureau['DAYS_CREDIT_ISPAST'] = bureau.DAYS_CREDIT_ENDDATE.map(lambda x: 1 if x < 0 else 0)
 bur_avg_remaining_days = bureau.loc[bureau.DAYS_CREDIT_ENDDATE>0].groupby('SK_ID_CURR')['DAYS_CREDIT_ENDDATE'].mean()
bureau['ADJ_AMT_CREDIT_MAX_OVERDUE'] = bureau.ADJ_DAYS * bureau.AMT_CREDIT_MAX_OVERDUE
bur_total_max_overdue_adj = bureau.groupby('SK_ID_CURR')['ADJ_AMT_CREDIT_MAX_OVERDUE'].sum()
bur_avg_max_overdue = bureau.groupby('SK_ID_CURR')['AMT_CREDIT_MAX_OVERDUE'].mean()
bur_overall_max_overdue = bureau.groupby('SK_ID_CURR')['AMT_CREDIT_MAX_OVERDUE'].max()
bureau['ADJ_CNT_CREDIT_PROLONG'] = bureau.ADJ_DAYS * bureau.CNT_CREDIT_PROLONG
bur_avg_prelonged = bureau.groupby('SK_ID_CURR')['ADJ_CNT_CREDIT_PROLONG'].mean().fillna(0)
 bur_max_prelonged = bureau.groupby('SK_ID_CURR')['CNT_CREDIT_PROLONG'].max().fillna(0)
 bur_total_prelonged_adj = bureau.groupby('SK_ID_CURR')['ADJ_CNT_CREDIT_PROLONG'].sum().fillna(0)
 bureau['ADJ_AMT_CREDIT_SUM_DEBT'] = bureau.ADJ_DAYS * bureau.AMT_CREDIT_SUM
bur_total_amount_adj = bureau.groupby('SK_ID_CURR')['ADJ_AMT_CREDIT_SUM_DEBT'].sum()
bur_avg_amount = bureau.groupby('SK_ID_CURR')['AMT_CREDIT_SUM'].mean()
bur_active_total_amount = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['AMT_CREDIT_SUM'].sum()
bur_active_avg_amount = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['AMT_CREDIT_SUM'].mean()
bur_active_total_debt = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['AMT_CREDIT_SUM_DEBT'].sum()
bur_active_avg_debt = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['AMT_CREDIT_SUM_DEBT'].mean()
bur_active_total_limit = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['AMT_CREDIT_SUM_LIMIT'].sum()
bur_active_avg_limit = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['AMT_CREDIT_SUM_LIMIT'].mean()
bur_active_total_overdue = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['AMT_CREDIT_SUM_OVERDUE'].sum()
bur_active_avg_overdue = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['AMT_CREDIT_SUM_OVERDUE'].mean()
bur_active_ratio_debt_credit =(bur_active_total_debt / bur_active_total_amount.map(lambda x: x+0.1))
bur_active_ratio_overdue_debt =(bur_active_total_overdue / bur_active_total_debt.map(lambda x: x+0.1))
bur_avg_update = bureau.groupby('SK_ID_CURR')['DAYS_CREDIT_UPDATE'].mean()
bur_recent_update = bureau.groupby('SK_ID_CURR')['DAYS_CREDIT_UPDATE'].max()
bur_avg_annuity = bureau.groupby('SK_ID_CURR')['AMT_ANNUITY'].mean()
bur_total_annuity = bureau.groupby('SK_ID_CURR')['AMT_ANNUITY'].sum()
bur_active_total_annuity = bureau.loc[bureau.CREDIT_ACTIVE=='Active'].groupby('SK_ID_CURR')['AMT_ANNUITY'].sum()
bureau['term'] = bureau.AMT_CREDIT_SUM / bureau.AMT_ANNUITY
bur_avg_term = bureau.loc[bureau.term < float('inf')].groupby('SK_ID_CURR')['term'].mean()
bureau_num_feature = pd.DataFrame({'bur_ncount':bur_ncount, 'bur_act_count':bur_act_count, 'bur_bad_count':bur_bad_count, 'bur_sold_count':bur_sold_count,
'bur_recent_application':bur_recent_application,'bur_eariliest_application':bur_eariliest_application, 'bur_max_enddate':bur_max_enddate,
'bur_avg_intervel':bur_avg_intervel, 'bur_sd_intervel':bur_sd_intervel,
'bur_max_overdue_days':bur_max_overdue_days, 'bur_active_total_overdue_days':bur_active_total_overdue_days, 'bur_active_max_overdue_days':bur_active_max_overdue_days,
'bur_avg_remaining_days':bur_avg_remaining_days, 'bur_total_max_overdue_adj':bur_total_max_overdue_adj, 'bur_avg_max_overdue':bur_avg_max_overdue, 'bur_overall_max_overdue':bur_overall_max_overdue,
'bur_avg_prelonged':bur_avg_prelonged, 'bur_max_prelonged':bur_max_prelonged, 'bur_total_prelonged_adj':bur_total_prelonged_adj,
'bur_total_amount_adj':bur_total_amount_adj, 'bur_avg_amount':bur_avg_amount,
'bur_active_total_amount':bur_active_total_amount, 'bur_active_avg_amount':bur_active_avg_amount, 'bur_active_total_debt':bur_active_total_debt, 'bur_active_avg_debt':bur_active_avg_debt,
'bur_active_total_limit':bur_active_total_limit, 'bur_active_avg_limit':bur_active_avg_limit, 'bur_active_total_overdue':bur_active_total_overdue, 'bur_active_avg_overdue':bur_active_avg_overdue,
'bur_active_ratio_debt_credit':bur_active_ratio_debt_credit, 'bur_active_ratio_overdue_debt':bur_active_ratio_overdue_debt,
'bur_avg_update':bur_avg_update, 'bur_recent_update':bur_recent_update,
'bur_avg_annuity':bur_avg_annuity, 'bur_total_annuity':bur_total_annuity, 'bur_active_total_annuity':bur_active_total_annuity, 'bur_avg_term':bur_avg_term} ).reset_index()
fill0_list = ['bur_act_count', 'bur_bad_count', 'bur_sold_count', 'bur_active_total_overdue_days', 'bur_active_max_overdue_days', 'bur_active_total_amount', 'bur_active_avg_amount',
'bur_active_total_debt', 'bur_active_avg_debt', 'bur_active_total_limit', 'bur_active_avg_limit', 'bur_active_total_overdue', 'bur_active_avg_overdue',
'bur_active_ratio_debt_credit', 'bur_active_ratio_overdue_debt', 'bur_active_total_annuity']
bureau_num_feature[fill0_list] = bureau_num_feature[fill0_list].fillna(0)
 bureau_cat = pd.get_dummies(bureau[['SK_ID_CURR','CREDIT_ACTIVE', 'CREDIT_CURRENCY', 'CREDIT_TYPE']], prefix='bur')
 bureau_cat_feature = bureau_cat.groupby('SK_ID_CURR' ).mean().reset_index()
del bureau_cat
gc.collect()
bureau_bb = bureau[['SK_ID_CURR','SK_ID_BUREAU']].merge(bb_feature, on='SK_ID_BUREAU', how='left')
 bb_avg_month = bureau_bb.groupby('SK_ID_CURR')['bb_month_count'].mean()
bb_total_overdue_month = bureau_bb.groupby('SK_ID_CURR')['bb_dpd_sum'].sum()
bb_total_writeoff = bureau_bb.groupby('SK_ID_CURR')['bb_write_off'].sum()
bb_max_overdue_month = bureau_bb.groupby('SK_ID_CURR')['bb_dpd_sum'].max()
bb_max_writeoff = bureau_bb.groupby('SK_ID_CURR')['bb_write_off'].max()
bb_total_overdue_month_2year = bureau_bb.groupby('SK_ID_CURR')['bb_dpd_sum_2_year'].sum()
bb_max_overdue_month_2year= bureau_bb.groupby('SK_ID_CURR')['bb_dpd_sum_2_year'].max()
bb_total_writeoff_2year = bureau_bb.groupby('SK_ID_CURR')['bb_write_off_2_year'].sum()
bb_max_writeoff_2year = bureau_bb.groupby('SK_ID_CURR')['bb_write_off_2_year'].max()
bb_max_score = bureau_bb.groupby('SK_ID_CURR')['bb_adj_score'].max()
bb_total_score = bureau_bb.groupby('SK_ID_CURR')['bb_adj_score'].sum()
bb_avg_score = bureau_bb.groupby('SK_ID_CURR')['bb_adj_score'].mean()
bureau_bb_feature = pd.DataFrame({'bb_avg_month':bb_avg_month,
'bb_total_overdue_month':bb_total_overdue_month, 'bb_total_writeoff':bb_total_writeoff, 'bur_sold_count':bb_max_overdue_month,'bb_max_writeoff':bb_max_writeoff,
'bb_total_overdue_month_2year':bb_total_overdue_month_2year, 'bb_total_writeoff_2year':bb_total_writeoff_2year,
'bb_max_overdue_month_2year':bb_max_overdue_month_2year,'bb_max_writeoff_2year':bb_max_writeoff_2year,
'bb_max_score':bb_max_score, 'bb_total_score':bb_total_score, 'bb_avg_score':bb_avg_score} ).reset_index()
			
bureau_feature = bureau_num_feature.merge(bureau_cat_feature, on='SK_ID_CURR' ).merge(bureau_bb_feature, on='SK_ID_CURR', how='left')
 print(bureau_feature.shape)
 bureau_feature.to_csv('bureau_feature.csv', index=False )<load_from_csv>
train = df_train.merge(df_store)
 test = df_test.merge(df_store)
 train["Promo2Active"] = get_Promo2Active(train)
 train["CompetitionActive"] = get_CompetitionActive(train)
 test["Promo2Active"] = get_Promo2Active(test)
 test["CompetitionActive"] = get_CompetitionActive(test )
pd.set_option('display.max_columns', None)
 %matplotlib inline
warnings.filterwarnings("ignore")
 application_train = pd.read_csv('.. /input/application_train.csv')
 credit_card_balance = pd.read_csv('.. /input/credit_card_balance.csv')
 credit_card_balance = credit_card_balance.sort_values(['SK_ID_CURR', 'SK_ID_PREV', 'MONTHS_BALANCE'])
 ccb_prev_count = credit_card_balance.groupby('SK_ID_CURR')['SK_ID_PREV'].nunique()
credit_card_balance['PAY_MONTH'] = credit_card_balance.CNT_INSTALMENT_MATURE_CUM.map(lambda x: 1 if x > 0 else 0)
 ccb_temp = credit_card_balance.groupby(['SK_ID_CURR','SK_ID_PREV'])['PAY_MONTH'].sum().reset_index()
ccb_avg_inst_card = ccb_temp.groupby('SK_ID_CURR')['PAY_MONTH'].mean()
ccb_total_inst_card = ccb_temp.groupby('SK_ID_CURR')['PAY_MONTH'].sum()
ccb_temp = credit_card_balance.groupby(['SK_ID_CURR','SK_ID_PREV'])['AMT_CREDIT_LIMIT_ACTUAL'].mean().reset_index()
ccb_avg_limit_card = ccb_temp.groupby('SK_ID_CURR')['AMT_CREDIT_LIMIT_ACTUAL'].mean()
ccb_max_limit_card = credit_card_balance.groupby('SK_ID_CURR')['AMT_CREDIT_LIMIT_ACTUAL'].max()
ccb_total_limit_card = ccb_temp.groupby('SK_ID_CURR')['AMT_CREDIT_LIMIT_ACTUAL'].sum()
ccb_temp = credit_card_balance.loc[credit_card_balance.CNT_DRAWINGS_CURRENT>0].groupby(['SK_ID_CURR','MONTHS_BALANCE'])['AMT_DRAWINGS_CURRENT', 'CNT_DRAWINGS_CURRENT'].sum().reset_index()
ccb_temp['avg_drawing_amount'] =(ccb_temp.AMT_DRAWINGS_CURRENT / ccb_temp.CNT_DRAWINGS_CURRENT ).fillna(0)
 ccb_avg_drawing_amount = ccb_temp.groupby('SK_ID_CURR')['avg_drawing_amount'].mean().fillna(0)
 ccb_count_rej = credit_card_balance.groupby(['SK_ID_CURR'])['NAME_CONTRACT_STATUS'].agg(lambda x: np.sum(x=='Refused'))
last_month_credit = credit_card_balance.groupby(['SK_ID_CURR','SK_ID_PREV'])['MONTHS_BALANCE'].max().reset_index()
last_month_credit = last_month_credit.merge(credit_card_balance, on=['SK_ID_CURR','SK_ID_PREV', 'MONTHS_BALANCE'])
 ccb_cur_total_receivable = last_month_credit.groupby('SK_ID_CURR')['AMT_TOTAL_RECEIVABLE'].sum()
ccb_cur_total_limit = last_month_credit.loc[last_month_credit.NAME_CONTRACT_STATUS == 'Active'].groupby('SK_ID_CURR')['AMT_CREDIT_LIMIT_ACTUAL'].sum()
ccb_cur_total_payment = last_month_credit.groupby('SK_ID_CURR')['AMT_INST_MIN_REGULARITY'].sum()
ccb_cur_total_balance = last_month_credit.groupby('SK_ID_CURR')['AMT_BALANCE'].sum()
ccb_temp = credit_card_balance.loc[credit_card_balance.MONTHS_BALANCE>=-12]
ccb_drawing_amount_1y = ccb_temp.groupby('SK_ID_CURR')['AMT_DRAWINGS_CURRENT'].sum()
ccb_drawing_times_1y = ccb_temp.groupby('SK_ID_CURR')['CNT_DRAWINGS_CURRENT'].sum()
ccb_temp = credit_card_balance.loc[credit_card_balance.MONTHS_BALANCE>=-6]
ccb_drawing_amount_6m = ccb_temp.groupby('SK_ID_CURR')['AMT_DRAWINGS_CURRENT'].sum()
ccb_drawing_times_6m = ccb_temp.groupby('SK_ID_CURR')['CNT_DRAWINGS_CURRENT'].sum()
ccb_temp = credit_card_balance[['SK_ID_CURR', 'SK_ID_PREV', 'SK_DPD', 'SK_DPD_DEF']].groupby(['SK_ID_CURR','SK_ID_PREV'])['SK_DPD','SK_DPD_DEF'].max().reset_index()
ccb_max_dpd_days = ccb_temp.groupby('SK_ID_CURR')['SK_DPD'].max()
ccb_total_dpd_days = ccb_temp.groupby('SK_ID_CURR')['SK_DPD'].sum()
ccb_max_largedpd_days = ccb_temp.groupby('SK_ID_CURR')['SK_DPD_DEF'].max()
ccb_total_largedpd_days = ccb_temp.groupby('SK_ID_CURR')['SK_DPD_DEF'].sum()
ccb_feature = pd.DataFrame({'ccb_prev_count':ccb_prev_count, 'ccb_avg_inst_card':ccb_avg_inst_card, 'ccb_avg_limit_card':ccb_avg_limit_card, 'ccb_total_inst_card':ccb_total_inst_card, 'ccb_count_rej': ccb_count_rej,
'ccb_avg_limit_card':ccb_avg_limit_card, 'ccb_max_limit_card':ccb_max_limit_card, 'ccb_total_limit_card':ccb_total_limit_card, 'ccb_avg_drawing_amount':ccb_avg_drawing_amount,
'ccb_cur_total_receivable':ccb_cur_total_receivable, 'ccb_cur_total_limit':ccb_cur_total_limit, 'ccb_cur_total_payment':ccb_cur_total_payment, 'ccb_cur_total_balance':ccb_cur_total_balance,
'ccb_drawing_amount_1y':ccb_drawing_amount_1y, 'ccb_drawing_times_1y':ccb_drawing_times_1y, 'ccb_drawing_amount_6m':ccb_drawing_amount_6m, 'ccb_drawing_times_6m':ccb_drawing_times_6m,
'ccb_max_dpd_days':ccb_max_dpd_days, 'ccb_total_dpd_days':ccb_total_dpd_days, 'ccb_max_largedpd_days':ccb_max_largedpd_days, 'ccb_total_largedpd_days':ccb_total_largedpd_days} ).reset_index()
ccb_feature.to_csv('ccb_feature.csv', index=False )<load_from_csv>
train["DayOfYear"] = train.Date.apply(lambda x: x.timetuple().tm_yday)
 test["DayOfYear"] = test.Date.apply(lambda x: x.timetuple().tm_yday )
pd.set_option('display.max_columns', None)
 %matplotlib inline
warnings.filterwarnings("ignore")
 installments_payments = pd.read_csv('.. /input/installments_payments.csv')
 installments_payments = installments_payments.sort_values(['SK_ID_CURR', 'SK_ID_PREV', 'NUM_INSTALMENT_NUMBER'])
 previous_application = pd.read_csv(".. /input/previous_application.csv")
 previous_application = previous_application.sort_values(['SK_ID_CURR', 'DAYS_DECISION'])
 recent_record = installments_payments.merge(installments_payments.groupby('SK_ID_CURR')['SK_ID_PREV'].max().reset_index().drop('SK_ID_CURR', axis=1), on='SK_ID_PREV')
 ip_recent_term = recent_record.groupby('SK_ID_CURR')['NUM_INSTALMENT_NUMBER'].max()
ip_recent_total_actual_payment = recent_record.groupby('SK_ID_CURR')['AMT_PAYMENT'].sum()
ip_recent_total_required_payment = recent_record.groupby('SK_ID_CURR')['AMT_INSTALMENT'].sum()
recent_record['is_late'] =(recent_record.DAYS_INSTALMENT < recent_record.DAYS_ENTRY_PAYMENT ).map(lambda x: 1 if x==True else 0 ).fillna(0)
 recent_record['is_less'] =(recent_record.AMT_INSTALMENT > recent_record.AMT_PAYMENT ).map(lambda x: 1 if x==True else 0 ).fillna(0)
 ip_recent_total_late_times = recent_record.groupby('SK_ID_CURR')['is_late'].sum()
ip_recent_total_less_times = recent_record.groupby('SK_ID_CURR')['is_less'].sum()
ip_temp1 = recent_record.loc[recent_record.is_late==1]
ip_temp2 = recent_record.loc[recent_record.is_less==1]
ip_temp1['total_late'] = ip_temp1.DAYS_ENTRY_PAYMENT - ip_temp1.DAYS_INSTALMENT
ip_temp2['total_less'] = ip_temp2.AMT_INSTALMENT - ip_temp2.AMT_PAYMENT
ip_recent_total_late_days = ip_temp1.groupby('SK_ID_CURR')['total_late'].sum()
ip_recent_total_less_amount = ip_temp2.groupby('SK_ID_CURR')['total_less'].sum()
del ip_temp1, ip_temp2
gc.collect()
ip_prev_count = installments_payments.groupby('SK_ID_CURR')['SK_ID_PREV'].nunique()
ip_payment_count = installments_payments.groupby('SK_ID_CURR')['SK_ID_PREV'].count()
installments_payments['IS_CREDIT'] = installments_payments.NUM_INSTALMENT_VERSION.map(lambda x: 1 if x==0 else 0)
 ip_creditcard_user = installments_payments.groupby('SK_ID_CURR')['IS_CREDIT'].sum().map(lambda x: 1 if x>0 else 0)
 ip_creditcard_count = installments_payments.groupby(['SK_ID_CURR','SK_ID_PREV'])['IS_CREDIT'].sum().map(lambda x: 1 if x>0 else 0 ).reset_index().groupby('SK_ID_CURR')['IS_CREDIT'].sum()
ip_temp =(installments_payments.groupby(['SK_ID_CURR','SK_ID_PREV'])['NUM_INSTALMENT_VERSION'].nunique() -1 ).reset_index()
ip_total_change_times = ip_temp.groupby('SK_ID_CURR')['NUM_INSTALMENT_VERSION'].sum()
ip_avg_change_times = ip_temp.groupby('SK_ID_CURR')['NUM_INSTALMENT_VERSION'].mean()
ip_temp = installments_payments.groupby(['SK_ID_CURR', 'SK_ID_PREV'])['NUM_INSTALMENT_NUMBER'].max().reset_index()
ip_avg_instl = ip_temp.groupby('SK_ID_CURR')['NUM_INSTALMENT_NUMBER'].mean()
ip_max_instl = ip_temp.groupby('SK_ID_CURR')['NUM_INSTALMENT_NUMBER'].max()
installments_payments['is_late'] =(installments_payments.DAYS_INSTALMENT < installments_payments.DAYS_ENTRY_PAYMENT ).map(lambda x: 1 if x==True else 0 ).fillna(0)
 installments_payments['is_less'] =(installments_payments.AMT_INSTALMENT > installments_payments.AMT_PAYMENT ).map(lambda x: 1 if x==True else 0 ).fillna(0)
 ip_total_late_times = installments_payments.groupby('SK_ID_CURR')['is_late'].sum()
ip_total_less_times = installments_payments.groupby('SK_ID_CURR')['is_less'].sum()
ip_temp1 = installments_payments.loc[installments_payments.is_late==1]
ip_temp2 = installments_payments.loc[installments_payments.is_less==1]
ip_temp1['total_late'] = ip_temp1.DAYS_ENTRY_PAYMENT - ip_temp1.DAYS_INSTALMENT
ip_temp2['total_less'] = ip_temp2.AMT_INSTALMENT - ip_temp2.AMT_PAYMENT
ip_total_late_days = ip_temp1.groupby('SK_ID_CURR')['total_late'].sum()
ip_total_less_amount = ip_temp2.groupby('SK_ID_CURR')['total_less'].sum()
del ip_temp1, ip_temp2
gc.collect()
ip_total_actual_payment = installments_payments.groupby('SK_ID_CURR')['AMT_PAYMENT'].sum()
ip_total_required_payment = installments_payments.groupby('SK_ID_CURR')['AMT_INSTALMENT'].sum()
ip_1y = installments_payments.loc[installments_payments.DAYS_ENTRY_PAYMENT>-365]
ip_payment_count_1y = ip_1y.groupby('SK_ID_CURR')['SK_ID_PREV'].count()
ip_creditcard_count_1y = ip_1y.groupby(['SK_ID_CURR','SK_ID_PREV'])['IS_CREDIT'].sum().map(lambda x: 1 if x>0 else 0 ).reset_index().groupby('SK_ID_CURR')['IS_CREDIT'].sum()
ip_total_late_times_1y = ip_1y.groupby('SK_ID_CURR')['is_late'].sum()
ip_total_less_times_1y = ip_1y.groupby('SK_ID_CURR')['is_less'].sum()
ip_temp = ip_1y.groupby(['SK_ID_CURR', 'SK_ID_PREV'])['NUM_INSTALMENT_NUMBER'].max().reset_index()
ip_avg_instl_1y = ip_temp.groupby('SK_ID_CURR')['NUM_INSTALMENT_NUMBER'].mean()
ip_temp1 = ip_1y.loc[ip_1y.is_late==1]
ip_temp2 = ip_1y.loc[ip_1y.is_less==1]
ip_temp1['total_late'] = ip_temp1.DAYS_ENTRY_PAYMENT - ip_temp1.DAYS_INSTALMENT
ip_temp2['total_less'] = ip_temp2.AMT_INSTALMENT - ip_temp2.AMT_PAYMENT
ip_total_late_days_1y = ip_temp1.groupby('SK_ID_CURR')['total_late'].sum()
ip_total_less_amount_1y = ip_temp2.groupby('SK_ID_CURR')['total_less'].sum()
del ip_temp1, ip_temp2
gc.collect()
ip_total_actual_payment_1y = ip_1y.groupby('SK_ID_CURR')['AMT_PAYMENT'].sum()
ip_total_required_payment_1y = ip_1y.groupby('SK_ID_CURR')['AMT_INSTALMENT'].sum()
ip_6m = installments_payments.loc[installments_payments.DAYS_ENTRY_PAYMENT>-180]
ip_payment_count_6m = ip_6m.groupby('SK_ID_CURR')['SK_ID_PREV'].count()
ip_creditcard_count_6m = ip_6m.groupby(['SK_ID_CURR','SK_ID_PREV'])['IS_CREDIT'].sum().map(lambda x: 1 if x>0 else 0 ).reset_index().groupby('SK_ID_CURR')['IS_CREDIT'].sum()
ip_total_late_times_6m = ip_6m.groupby('SK_ID_CURR')['is_late'].sum()
ip_total_less_times_6m = ip_6m.groupby('SK_ID_CURR')['is_less'].sum()
ip_temp = ip_6m.groupby(['SK_ID_CURR', 'SK_ID_PREV'])['NUM_INSTALMENT_NUMBER'].max().reset_index()
ip_avg_instl_6m = ip_temp.groupby('SK_ID_CURR')['NUM_INSTALMENT_NUMBER'].mean()
ip_temp1 = ip_6m.loc[ip_6m.is_late==1]
ip_temp2 = ip_6m.loc[ip_6m.is_less==1]
ip_temp1['total_late'] = ip_temp1.DAYS_ENTRY_PAYMENT - ip_temp1.DAYS_INSTALMENT
ip_temp2['total_less'] = ip_temp2.AMT_INSTALMENT - ip_temp2.AMT_PAYMENT
ip_total_late_days_6m = ip_temp1.groupby('SK_ID_CURR')['total_late'].sum()
ip_total_less_amount_6m = ip_temp2.groupby('SK_ID_CURR')['total_less'].sum()
del ip_temp1, ip_temp2
gc.collect()
ip_total_actual_payment_6m = ip_6m.groupby('SK_ID_CURR')['AMT_PAYMENT'].sum()
ip_total_required_payment_6m = ip_6m.groupby('SK_ID_CURR')['AMT_INSTALMENT'].sum()
ip_feature = pd.DataFrame({'ip_prev_count':ip_prev_count, 'ip_payment_count':ip_payment_count, 'ip_creditcard_user':ip_creditcard_user, 'ip_creditcard_count':ip_creditcard_count,
'ip_total_change_times':ip_total_change_times, 'ip_avg_change_times':ip_avg_change_times, 'ip_avg_instl':ip_avg_instl, 'ip_max_instl':ip_max_instl,
'ip_total_late_times':ip_total_late_times, 'ip_total_less_times':ip_total_less_times, 'ip_total_late_days':ip_total_late_days, 'ip_total_less_amount':ip_total_less_amount,
'ip_total_actual_payment':ip_total_actual_payment, 'ip_total_required_payment':ip_total_required_payment,
'ip_recent_term':ip_recent_term, 'ip_recent_total_actual_payment':ip_recent_total_actual_payment, 'ip_recent_total_required_payment':ip_recent_total_required_payment,
'ip_recent_total_late_times':ip_recent_total_late_times, 'ip_recent_total_less_times':ip_recent_total_less_times,
'ip_recent_total_late_days':ip_recent_total_late_days, 'ip_recent_total_less_amount':ip_recent_total_less_amount} ).fillna(0)
 ip_1y = pd.DataFrame({'ip_payment_count_1y':ip_payment_count_1y, 'ip_creditcard_count_1y': ip_creditcard_count_1y, 'ip_avg_instl_1y':ip_avg_instl_1y,
'ip_total_late_times_1y':ip_total_late_times_1y, 'ip_total_less_times_1y':ip_total_less_times_1y, 'ip_total_late_days_1y':ip_total_late_days_1y,
'ip_total_less_amount_1y':ip_total_less_amount_1y, 'ip_total_actual_payment_1y':ip_total_actual_payment_1y, 'ip_total_required_payment_1y':ip_total_required_payment_1y} ).reset_index().fillna(0)
 ip_6m = pd.DataFrame({'ip_payment_count_6m':ip_payment_count_6m, 'ip_creditcard_count_6m': ip_creditcard_count_6m, 'ip_avg_instl_6m':ip_avg_instl_6m,
'ip_total_late_times_6m':ip_total_late_times_6m, 'ip_total_less_times_6m':ip_total_less_times_6m, 'ip_total_late_days_6m':ip_total_late_days_6m,
'ip_total_less_amount_6m':ip_total_less_amount_6m, 'ip_total_actual_payment_6m':ip_total_actual_payment_6m, 'ip_total_required_payment_6m':ip_total_required_payment_6m} ).reset_index().fillna(0)
 ip_feature = ip_feature.merge(ip_1y, on='SK_ID_CURR', how='left' ).merge(ip_6m, on='SK_ID_CURR', how='left')
 ip_feature['ip_active_1y'] = ip_feature.ip_total_late_times_1y.notnull().map(lambda x: 1 if x==True else 0)
 ip_feature['ip_active_6m'] = ip_feature.ip_total_late_times_6m.notnull().map(lambda x: 1 if x==True else 0)
 ACCOUNT_1Y = installments_payments.groupby('SK_ID_PREV')['DAYS_ENTRY_PAYMENT'].max().map(lambda x: 1 if x>=-365 else 0)
 ACCOUNT_6M = installments_payments.groupby('SK_ID_PREV')['DAYS_ENTRY_PAYMENT'].max().map(lambda x: 1 if x>=-180 else 0)
 ACTIVE_ACCOUNT = pd.DataFrame({'ACCOUNT_1Y':ACCOUNT_1Y, 'ACCOUNT_6M':ACCOUNT_6M} ).reset_index()
pa_ip = previous_application[['SK_ID_CURR', 'SK_ID_PREV', 'NAME_CONTRACT_TYPE']].merge(ACTIVE_ACCOUNT, on='SK_ID_PREV', how='left')
 COUNT_1Y = pd.get_dummies(pa_ip.loc[pa_ip.ACCOUNT_1Y==1, ['SK_ID_CURR', 'NAME_CONTRACT_TYPE']], prefix='ip_count_1y_' ).groupby('SK_ID_CURR' ).sum().reset_index()
COUNT_6M = pd.get_dummies(pa_ip.loc[pa_ip.ACCOUNT_6M==1, ['SK_ID_CURR', 'NAME_CONTRACT_TYPE']], prefix='ip_count_6m_' ).groupby('SK_ID_CURR' ).sum().reset_index()
ip_feature = ip_feature.merge(COUNT_1Y, on='SK_ID_CURR', how='left' ).merge(COUNT_6M, on='SK_ID_CURR', how='left')
 for i in range(-6, 0):
ip_feature.iloc[:, i] = ip_feature.iloc[:, i].fillna(0)
 ip_feature.shape
ip_feature.to_csv('ip_feature.csv', index=False )<load_from_csv>
min_date = train.Date.min()
def date_to_day_number(df):
return(df.Date - min_date ).apply(lambda x: x.days )
pd.set_option('display.max_columns', None)
 %matplotlib inline
warnings.filterwarnings("ignore")
 application_train = pd.read_csv('.. /input/application_train.csv')
 POS_CASH_balance = pd.read_csv('.. /input/POS_CASH_balance.csv')
 POS_CASH_balance = POS_CASH_balance.sort_values(['SK_ID_CURR', 'SK_ID_PREV', 'MONTHS_BALANCE'])
 pcb_prev_count = POS_CASH_balance.groupby('SK_ID_CURR')['SK_ID_PREV'].nunique()
pcb_avg_month = POS_CASH_balance.groupby('SK_ID_CURR')['MONTHS_BALANCE'].count() /pcb_prev_count
pcb_recent_active = POS_CASH_balance.groupby('SK_ID_CURR')['MONTHS_BALANCE'].max()
pcb_temp_inst_change_time = POS_CASH_balance[['SK_ID_PREV', 'CNT_INSTALMENT']].groupby('SK_ID_PREV')['CNT_INSTALMENT'].nunique().map(lambda x: x - 1 ).reset_index().rename(columns={'CNT_INSTALMENT':'pcb_prev_inst_change_time'})
 pcb_temp = POS_CASH_balance.groupby(['SK_ID_CURR', 'SK_ID_PREV'])['MONTHS_BALANCE'].count().reset_index()
pcb_temp = pcb_temp.merge(pcb_temp_inst_change_time, on='SK_ID_PREV')
 pcb_inst_change_time = pcb_temp.groupby('SK_ID_CURR')['pcb_prev_inst_change_time'].sum()
pcb_temp = POS_CASH_balance.groupby(['SK_ID_CURR', 'SK_ID_PREV'])['CNT_INSTALMENT'].mean().reset_index()
pcb_avg_inst = pcb_temp.groupby(['SK_ID_CURR'])['CNT_INSTALMENT'].mean()
pcb_temp = POS_CASH_balance.groupby(['SK_ID_CURR', 'SK_ID_PREV'])['MONTHS_BALANCE'].max().reset_index()
pcb_temp = pcb_temp.merge(POS_CASH_balance[['SK_ID_CURR', 'SK_ID_PREV', 'MONTHS_BALANCE', 'NAME_CONTRACT_STATUS']], on=['SK_ID_CURR', 'SK_ID_PREV', 'MONTHS_BALANCE'])
 pcb_temp['active_1'] = pcb_temp.MONTHS_BALANCE.map(lambda x: 1 if x>=-4 else 0)
 pcb_temp['active_2'] = pcb_temp.NAME_CONTRACT_STATUS.map(lambda x: 1 if x=='Active' else 0)
 pcb_temp['active'] = pcb_temp.active_1 * pcb_temp.active_2
pcb_active_inst = pcb_temp.groupby('SK_ID_CURR')['active'].count()
pcb_temp = POS_CASH_balance[['SK_ID_CURR', 'SK_ID_PREV', 'SK_DPD', 'SK_DPD_DEF']].groupby(['SK_ID_CURR','SK_ID_PREV'])['SK_DPD','SK_DPD_DEF'].max().reset_index()
pcb_max_dpd_days = pcb_temp.groupby('SK_ID_CURR')['SK_DPD'].max()
pcb_total_dpd_days = pcb_temp.groupby('SK_ID_CURR')['SK_DPD'].sum()
pcb_max_largedpd_days = pcb_temp.groupby('SK_ID_CURR')['SK_DPD_DEF'].max()
pcb_total_largedpd_days = pcb_temp.groupby('SK_ID_CURR')['SK_DPD_DEF'].sum()
pcb_temp = POS_CASH_balance.loc[POS_CASH_balance.MONTHS_BALANCE>=-12, ['SK_ID_CURR', 'SK_ID_PREV', 'SK_DPD', 'SK_DPD_DEF']].groupby(['SK_ID_CURR','SK_ID_PREV'])['SK_DPD','SK_DPD_DEF'].max().reset_index()
pcb_max_dpd_days_1y = pcb_temp.groupby('SK_ID_CURR')['SK_DPD'].max()
pcb_total_dpd_days_1y = pcb_temp.groupby('SK_ID_CURR')['SK_DPD'].sum()
pcb_max_largedpd_days_1y = pcb_temp.groupby('SK_ID_CURR')['SK_DPD_DEF'].max()
pcb_total_largedpd_days_1y = pcb_temp.groupby('SK_ID_CURR')['SK_DPD_DEF'].sum()
pcb_temp = POS_CASH_balance.loc[POS_CASH_balance.MONTHS_BALANCE>=-24, ['SK_ID_CURR', 'SK_ID_PREV', 'SK_DPD', 'SK_DPD_DEF']].groupby(['SK_ID_CURR','SK_ID_PREV'])['SK_DPD','SK_DPD_DEF'].max().reset_index()
pcb_max_dpd_days_2y = pcb_temp.groupby('SK_ID_CURR')['SK_DPD'].max()
pcb_total_dpd_days_2y = pcb_temp.groupby('SK_ID_CURR')['SK_DPD'].sum()
pcb_max_largedpd_days_2y = pcb_temp.groupby('SK_ID_CURR')['SK_DPD_DEF'].max()
pcb_total_largedpd_days_2y = pcb_temp.groupby('SK_ID_CURR')['SK_DPD_DEF'].sum()
pcb_num = pd.DataFrame({'pcb_prev_count':pcb_prev_count, 'pcb_avg_month':pcb_avg_month, 'pcb_recent_active':pcb_recent_active, 'pcb_inst_change_time':pcb_inst_change_time,
'pcb_avg_inst':pcb_avg_inst, 'pcb_active_inst':pcb_active_inst,
'pcb_max_dpd_days':pcb_max_dpd_days, 'pcb_total_dpd_days':pcb_total_dpd_days, 'pcb_max_largedpd_days':pcb_max_largedpd_days, 'pcb_total_largedpd_days':pcb_total_largedpd_days,
'pcb_max_dpd_days_1y':pcb_max_dpd_days_1y, 'pcb_total_dpd_days_1y':pcb_total_dpd_days_1y, 'pcb_max_largedpd_days_1y':pcb_max_largedpd_days_1y,
'pcb_total_largedpd_days_1y':pcb_total_largedpd_days_1y, 'pcb_max_dpd_days_2y':pcb_max_dpd_days_2y, 'pcb_total_dpd_days_2y':pcb_total_dpd_days_2y,
'pcb_max_largedpd_days_2y':pcb_max_largedpd_days_2y, 'pcb_total_largedpd_days_2y':pcb_total_largedpd_days_2y,} ).reset_index()
					
pcb_temp = POS_CASH_balance.groupby(['SK_ID_CURR', 'SK_ID_PREV'])['MONTHS_BALANCE'].max().reset_index()
pcb_temp = pcb_temp.merge(POS_CASH_balance[['SK_ID_CURR', 'SK_ID_PREV', 'MONTHS_BALANCE', 'NAME_CONTRACT_STATUS']], on=['SK_ID_CURR', 'SK_ID_PREV', 'MONTHS_BALANCE'])
 pcb_temp_dummy = pd.get_dummies(pcb_temp, prefix='pcb_end_as')
 pcb_end_as_dummy = pcb_temp_dummy.loc[pcb_temp_dummy.pcb_end_as_Active!=1].drop(['SK_ID_PREV','MONTHS_BALANCE', 'pcb_end_as_Active'], axis=1 ).groupby('SK_ID_CURR' ).sum().reset_index()
del pcb_temp_dummy
gc.collect()
pcb_feature = pcb_num.merge(pcb_end_as_dummy, on='SK_ID_CURR', how='left')
 pcb_feature['pcb_no_dpd'] = pcb_feature.pcb_total_dpd_days.map(lambda x: 0 if x== 0 else 1)
 pcb_feature['pcb_no_largedpd'] = pcb_feature.pcb_total_largedpd_days.map(lambda x: 0 if x== 0 else 1)
 pcb_feature['pcb_no_dpd_1y'] = pcb_feature.pcb_total_dpd_days_1y.map(lambda x: 0 if x== 0 else 1)
 pcb_feature['pcb_no_largedpd_1y'] = pcb_feature.pcb_total_largedpd_days_1y.map(lambda x: 0 if x== 0 else 1)
 pcb_feature['pcb_no_dpd_2y'] = pcb_feature.pcb_total_dpd_days_2y.map(lambda x: 0 if x== 0 else 1)
 pcb_feature['pcb_no_largedpd_2y'] = pcb_feature.pcb_total_largedpd_days_2y.map(lambda x: 0 if x== 0 else 1)
 pcb_feature.to_csv('pcb_feature.csv', index=False )<load_from_csv>
train["Day"] = date_to_day_number(train)
 test["Day"] = date_to_day_number(test )
pd.set_option('display.max_columns', None)
 %matplotlib inline
warnings.filterwarnings("ignore")
 previous_application = pd.read_csv(".. /input/previous_application.csv")
 previous_application = previous_application.sort_values(['SK_ID_CURR', 'DAYS_DECISION'])
 cat_col = []
for i in range(len(previous_application.columns)) :
if previous_application.iloc[:, i].dtype == 'object':
cat_col.append(i)
 cat_pa = previous_application.iloc[:, cat_col]
cat_pa = pd.concat([previous_application[['HOUR_APPR_PROCESS_START', 'NFLAG_LAST_APPL_IN_DAY', 'NFLAG_INSURED_ON_APPROVAL']],cat_pa], axis=1 ).fillna('XNA')
 cat_pa.HOUR_APPR_PROCESS_START = cat_pa.HOUR_APPR_PROCESS_START.astype('object')
 cat_pa.NFLAG_LAST_APPL_IN_DAY = cat_pa.NFLAG_LAST_APPL_IN_DAY.astype('object')
 cat_pa.NFLAG_INSURED_ON_APPROVAL = cat_pa.NFLAG_INSURED_ON_APPROVAL.astype('object')
 cat_NAME_CONTRACT_TYPE = pd.get_dummies(cat_pa.NAME_CONTRACT_TYPE, prefix='pa_NAME_CONTRACT_TYPE' ).drop('pa_NAME_CONTRACT_TYPE_XNA', axis=1)
 cat_NAME_CASH_LOAN_PURPOSE = pd.get_dummies(cat_pa.NAME_CASH_LOAN_PURPOSE, prefix='pa_NAME_CASH_LOAN_PURPOSE' ).drop('pa_NAME_CASH_LOAN_PURPOSE_XNA', axis=1)
 cat_NAME_CONTRACT_STATUS = pd.get_dummies(cat_pa.NAME_CONTRACT_STATUS, prefix='pa_NAME_CONTRACT_STATUS')
 cat_NAME_PAYMENT_TYPE = pd.get_dummies(cat_pa.NAME_PAYMENT_TYPE, prefix='pa_NAME_PAYMENT_TYPE' ).drop('pa_NAME_PAYMENT_TYPE_XNA', axis=1)
 cat_CODE_REJECT_REASON = pd.get_dummies(cat_pa.CODE_REJECT_REASON, prefix='pa_CODE_REJECT_REASON' ).drop('pa_CODE_REJECT_REASON_XNA', axis=1)
 cat_NAME_CLIENT_TYPE = pd.get_dummies(cat_pa.NAME_CLIENT_TYPE, prefix='pa_NAME_CLIENT_TYPE' ).drop('pa_NAME_CLIENT_TYPE_XNA', axis=1)
 cat_NAME_PORTFOLIO = pd.get_dummies(cat_pa.NAME_PORTFOLIO, prefix='pa_NAME_PORTFOLIO' ).drop('pa_NAME_PORTFOLIO_XNA', axis=1)
 cat_NAME_PRODUCT_TYPE = pd.get_dummies(cat_pa.NAME_PRODUCT_TYPE, prefix='pa_NAME_PRODUCT_TYPE' ).drop('pa_NAME_PRODUCT_TYPE_XNA', axis=1)
 cat_NAME_YIELD_GROUP = pd.get_dummies(cat_pa.NAME_YIELD_GROUP, prefix='pa_NAME_YIELD_GROUP' ).drop('pa_NAME_YIELD_GROUP_XNA', axis=1)
 cat_PRODUCT_COMBINATION = pd.get_dummies(cat_pa.PRODUCT_COMBINATION, prefix='pa_PRODUCT_COMBINATION' ).drop('pa_PRODUCT_COMBINATION_XNA', axis=1)
 cat_NFLAG_INSURED_ON_APPROVAL = pd.get_dummies(cat_pa.NFLAG_INSURED_ON_APPROVAL, prefix='pa_NFLAG_INSURED_ON_APPROVAL' ).drop('pa_NFLAG_INSURED_ON_APPROVAL_XNA', axis=1)
 cat_pa_dummy = pd.concat([previous_application[['SK_ID_PREV', 'SK_ID_CURR']], cat_NAME_CONTRACT_TYPE, cat_NAME_CASH_LOAN_PURPOSE, cat_NAME_CONTRACT_STATUS, cat_NAME_PAYMENT_TYPE, cat_CODE_REJECT_REASON,
cat_NAME_CLIENT_TYPE, cat_NAME_PORTFOLIO, cat_NAME_PRODUCT_TYPE, cat_NAME_YIELD_GROUP, cat_PRODUCT_COMBINATION, cat_NFLAG_INSURED_ON_APPROVAL], axis=1)
 del cat_NAME_CONTRACT_TYPE, cat_NAME_CASH_LOAN_PURPOSE, cat_NAME_CONTRACT_STATUS, cat_NAME_PAYMENT_TYPE, cat_CODE_REJECT_REASON, cat_NAME_CLIENT_TYPE,
cat_NAME_PORTFOLIO, cat_NAME_PRODUCT_TYPE, cat_NAME_YIELD_GROUP, cat_PRODUCT_COMBINATION, cat_NFLAG_INSURED_ON_APPROVAL, cat_pa
gc.collect()
pa_cat = cat_pa_dummy.drop('SK_ID_PREV', axis=1 ).groupby('SK_ID_CURR' ).mean().reset_index()
num_col = []
for i in range(len(previous_application.columns)) :
if(previous_application.iloc[:, i].dtype == 'int64')or(previous_application.iloc[:, i].dtype == 'float64'):
num_col.append(i)
 num_pa = previous_application.iloc[:, num_col].drop(['HOUR_APPR_PROCESS_START', 'NFLAG_LAST_APPL_IN_DAY', 'NFLAG_INSURED_ON_APPROVAL'], axis=1)
 for i in range(-5, 0):
num_pa.iloc[:, i] = num_pa.iloc[:, i].map(lambda x: np.nan if x==365243.0 else x)
 num_pa['app_vs_actual_less'] =(num_pa.AMT_APPLICATION < num_pa.AMT_CREDIT ).map(lambda x: 1 if x==True else 0)
 num_pa['app_vs_actual_more'] =(num_pa.AMT_APPLICATION > num_pa.AMT_CREDIT ).map(lambda x: 1 if x==True else 0)
 a = num_pa.DAYS_DECISION - num_pa.DAYS_DECISION.min() + 1
num_pa['ADJ_SCORE'] =(a-a.min())/(a.max() -a.min())+ 0.5
num_pa['ADJ_AMT_ANNUITY'] = num_pa['ADJ_SCORE']*num_pa['AMT_ANNUITY']
num_pa['ADJ_AMT_APPLICATION'] = num_pa['ADJ_SCORE']*num_pa['AMT_APPLICATION']
num_pa['ADJ_AMT_CREDIT'] = num_pa['ADJ_SCORE']*num_pa['AMT_CREDIT']
num_pa['FREQ'] =(num_pa['DAYS_LAST_DUE']-num_pa['DAYS_FIRST_DUE'])/num_pa['CNT_PAYMENT']
pa_prev_count = num_pa.groupby('SK_ID_CURR')['SK_ID_PREV'].count()
pa_avg_annuity = num_pa.groupby('SK_ID_CURR')['AMT_ANNUITY'].mean()
pa_avg_application = num_pa.groupby('SK_ID_CURR')['AMT_APPLICATION'].mean()
pa_avg_actual_credit = num_pa.groupby('SK_ID_CURR')['AMT_CREDIT'].mean()
pa_max_application = num_pa.groupby('SK_ID_CURR')['AMT_APPLICATION'].max()
pa_max_actual_credit = num_pa.groupby('SK_ID_CURR')['AMT_CREDIT'].max()
pa_total_annuity = num_pa.groupby('SK_ID_CURR')['AMT_ANNUITY'].sum()
pa_total_application = num_pa.groupby('SK_ID_CURR')['AMT_APPLICATION'].sum()
pa_total_actual_credit = num_pa.groupby('SK_ID_CURR')['AMT_CREDIT'].sum()
pa_not_full_credit_times = num_pa.groupby('SK_ID_CURR')['app_vs_actual_less'].sum()
pa_not_full_credit_rate = num_pa.groupby('SK_ID_CURR')['app_vs_actual_less'].mean()
pa_get_more_credit_times = num_pa.groupby('SK_ID_CURR')['app_vs_actual_more'].sum()
pa_get_more_credit_rate = num_pa.groupby('SK_ID_CURR')['app_vs_actual_more'].mean()
pa_total_annuity_adj = num_pa.groupby('SK_ID_CURR')['ADJ_AMT_ANNUITY'].sum()
pa_total_application_adj = num_pa.groupby('SK_ID_CURR')['ADJ_AMT_APPLICATION'].sum()
pa_total_actual_credit_adj = num_pa.groupby('SK_ID_CURR')['ADJ_AMT_CREDIT'].sum()
pa_avg_down_payment = num_pa.groupby('SK_ID_CURR')['AMT_DOWN_PAYMENT'].mean()
pa_max_down_payment = num_pa.groupby('SK_ID_CURR')['AMT_DOWN_PAYMENT'].max()
pa_total_down_payment = num_pa.groupby('SK_ID_CURR')['AMT_DOWN_PAYMENT'].sum()
pa_avg_goods_price = num_pa.groupby('SK_ID_CURR')['AMT_GOODS_PRICE'].mean()
pa_max_goods_price = num_pa.groupby('SK_ID_CURR')['AMT_GOODS_PRICE'].max()
pa_total_goods_price = num_pa.groupby('SK_ID_CURR')['AMT_GOODS_PRICE'].sum()
pa_avg_down_payment_rate = num_pa.groupby('SK_ID_CURR')['RATE_DOWN_PAYMENT'].mean()
pa_max_down_payment_rate = num_pa.groupby('SK_ID_CURR')['RATE_DOWN_PAYMENT'].max()
pa_total_down_payment_rate = num_pa.groupby('SK_ID_CURR')['RATE_DOWN_PAYMENT'].sum()
pa_avg_selling_area = num_pa.groupby('SK_ID_CURR')['SELLERPLACE_AREA'].mean()
pa_max_selling_area = num_pa.groupby('SK_ID_CURR')['SELLERPLACE_AREA'].max()
pa_total_selling_area = num_pa.groupby('SK_ID_CURR')['SELLERPLACE_AREA'].sum()
pa_avg_term = num_pa.groupby('SK_ID_CURR')['CNT_PAYMENT'].mean()
pa_max_term = num_pa.groupby('SK_ID_CURR')['CNT_PAYMENT'].max()
pa_total_term = num_pa.groupby('SK_ID_CURR')['CNT_PAYMENT'].sum()
pa_most_frequent_term = num_pa.groupby('SK_ID_CURR')['CNT_PAYMENT'].agg(pd.Series.mode ).map(lambda x: np.mean(x))
pa_recent_decision_day = num_pa.groupby('SK_ID_CURR')['DAYS_DECISION'].max()
pa_earliest_decision_day = num_pa.groupby('SK_ID_CURR')['DAYS_DECISION'].min()
pa_usage_length = pa_recent_decision_day - pa_earliest_decision_day
num_pa['application_interval'] = num_pa.groupby('SK_ID_CURR')['DAYS_DECISION'].diff(-1)
 missing_iter = iter(num_pa.groupby('SK_ID_CURR')['DAYS_DECISION'].max())
 num_pa.application_interval = num_pa.application_interval.map(lambda x: -next(missing_iter)if np.isnan(x)else -x)
 pa_avg_intervel = num_pa.groupby('SK_ID_CURR')['application_interval'].mean()
pa_sd_intervel = num_pa.groupby('SK_ID_CURR')['application_interval'].agg('std' ).fillna(0)
 pa_first_due_day = num_pa.groupby('SK_ID_CURR')['DAYS_FIRST_DUE'].min()
pa_last_due_day = num_pa.groupby('SK_ID_CURR')['DAYS_LAST_DUE'].max()
pa_last_termination_day = num_pa.groupby('SK_ID_CURR')['DAYS_TERMINATION'].max()
pa_lastdue_termination_range = pa_last_termination_day - pa_last_due_day
pa_avg_freq = num_pa.groupby('SK_ID_CURR')['FREQ'].mean()
pa_min_freq = num_pa.groupby('SK_ID_CURR')['FREQ'].min()
pa_num = pd.DataFrame({'pa_prev_count':pa_prev_count, 'pa_avg_annuity':pa_avg_annuity, 'pa_avg_application':pa_avg_application, 'pa_avg_actual_credit':pa_avg_actual_credit,
'pa_max_application':pa_max_application, 'pa_max_actual_credit':pa_max_actual_credit, 'pa_total_annuity':pa_total_annuity, 'pa_total_application':pa_total_application, 'pa_total_actual_credit':pa_total_actual_credit,
'pa_not_full_credit_times':pa_not_full_credit_times, 'pa_not_full_credit_rate':pa_not_full_credit_rate, 'pa_get_more_credit_times':pa_get_more_credit_times, 'pa_get_more_credit_rate':pa_get_more_credit_rate,
'pa_total_annuity_adj':pa_total_annuity_adj, 'pa_total_application_adj':pa_total_application_adj, 'pa_total_actual_credit_adj':pa_total_actual_credit_adj,
'pa_avg_down_payment':pa_avg_down_payment, 'pa_max_down_payment':pa_max_down_payment, 'pa_total_down_payment':pa_total_down_payment,
'pa_avg_goods_price':pa_avg_goods_price, 'pa_max_goods_price':pa_max_goods_price, 'pa_total_goods_price':pa_total_goods_price,
'pa_avg_down_payment_rate':pa_avg_down_payment_rate, 'pa_max_down_payment_rate':pa_max_down_payment_rate, 'pa_total_down_payment_rate':pa_total_down_payment_rate,
'pa_avg_selling_area':pa_avg_selling_area, 'pa_max_selling_area':pa_max_selling_area, 'pa_total_selling_area':pa_total_selling_area,
'pa_avg_term':pa_avg_term, 'pa_max_term':pa_max_term, 'pa_total_term':pa_total_term, 'pa_most_frequent_term':pa_most_frequent_term,
'pa_recent_decision_day':pa_recent_decision_day, 'pa_earliest_decision_day':pa_earliest_decision_day, 'pa_usage_length': pa_usage_length,
'pa_avg_intervel':pa_avg_intervel, 'pa_sd_intervel':pa_sd_intervel, 'pa_first_due_day':pa_first_due_day, 'pa_last_due_day':pa_last_due_day,
'pa_last_termination_day':pa_last_termination_day, 'pa_lastdue_termination_range':pa_lastdue_termination_range, 'pa_avg_freq':pa_avg_freq, 'pa_min_freq':pa_min_freq} ).reset_index()
pa_feature = pa_num.merge(pa_cat, on='SK_ID_CURR')
 del pa_num, pa_cat
gc.collect()
pa_feature.to_csv('pa_feature.csv', index=False )<feature_engineering>
train.sort_values("Day", inplace=True)
 test.sort_values("Day", inplace=True )
bureau = pd.read_csv('.. /input/bureau.csv')
 bureau_balance = pd.read_csv('.. /input/bureau_balance.csv')
 med = bureau.AMT_CREDIT_SUM.median()
bureau.AMT_CREDIT_SUM = bureau.AMT_CREDIT_SUM.fillna(med)
 med = bureau.AMT_CREDIT_SUM_DEBT.median()
bureau.AMT_CREDIT_SUM_DEBT = bureau.AMT_CREDIT_SUM_DEBT.fillna(med)
 bureau['OVERDUE_DEBT_RATIO'] = bureau.AMT_CREDIT_SUM_OVERDUE/(bureau.AMT_CREDIT_SUM_DEBT+1)
 bureau['DEBT_TOTAL_RATIO'] = bureau.AMT_CREDIT_SUM_DEBT/(bureau.AMT_CREDIT_SUM+1)
 bureau_balance['INT_STATUS'] = bureau_balance.STATUS.replace('X', 0.1 ).replace('C', 0 ).astype('int64')
 bur_max_bad_level = bureau_balance.groupby('SK_ID_BUREAU')['INT_STATUS'].max().reset_index()
cluter_bur = bureau[['SK_ID_BUREAU', 'CREDIT_DAY_OVERDUE', 'OVERDUE_DEBT_RATIO', 'DEBT_TOTAL_RATIO', 'CNT_CREDIT_PROLONG']].merge(bur_max_bad_level, on='SK_ID_BUREAU', how='left' ).fillna(0)
 X = cluter_bur.drop(['SK_ID_BUREAU'], axis=1)
 X = Normalizer().fit_transform(X)
 gmm = GaussianMixture(n_components=2, verbose=5, max_iter=100, init_params='kmeans')
 gmm.fit(X)
 group_prob = gmm.predict_proba(X)
 group_prob = np.round(group_prob, decimals=2)
 bur_cluster = pd.concat([bureau[['SK_ID_CURR', 'SK_ID_BUREAU']], pd.DataFrame({'cluster':group_prob[:, 0]})], axis=1)
 bur_cluster = bur_cluster.groupby('SK_ID_CURR')['cluster'].mean().reset_index()
bur_cluster.to_csv('bur_cluster.csv', index=False)
<load_from_csv>
def rmspe(y_true, y_pred):
w = np.zeros(y_true.shape, dtype=float)
 ind = y_true != 0
w[ind] = 1./(y_true[ind]**2)
 return np.sqrt(np.mean(w *(y_true - y_pred)**2))
rmspe_scorer = make_scorer(rmspe, greater_is_better=False )
a = time.time()
row1=None
row2=None
row3=None
app_train = pd.read_csv('.. /input/application_train.csv', nrows=row1 ).sort_values('SK_ID_CURR')
 app_test = pd.read_csv('.. /input/application_test.csv', nrows=row1 ).sort_values('SK_ID_CURR')
 bureau = pd.read_csv('.. /input/bureau.csv', nrows=row2 ).sort_values(['SK_ID_CURR', 'SK_ID_BUREAU'])
 bureau_balance = pd.read_csv('.. /input/bureau_balance.csv', nrows=row3 ).sort_values(['SK_ID_BUREAU', 'MONTHS_BALANCE'])
 previous = pd.read_csv('.. /input/previous_application.csv', nrows=row3 ).sort_values(['SK_ID_CURR', 'SK_ID_PREV'])
 cash = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows=row3 ).sort_values(['SK_ID_CURR', 'SK_ID_PREV', 'MONTHS_BALANCE'])
 credit = pd.read_csv('.. /input/credit_card_balance.csv', nrows=row3 ).sort_values(['SK_ID_CURR', 'SK_ID_PREV', 'MONTHS_BALANCE'])
 installments = pd.read_csv('.. /input/installments_payments.csv', nrows=row3 ).sort_values(['SK_ID_CURR', 'SK_ID_PREV'])
 app_train = app_train[['SK_ID_CURR']]
app_test = app_test[['SK_ID_CURR']]
bureau = bureau[['SK_ID_CURR', 'SK_ID_BUREAU', 'DAYS_CREDIT', 'AMT_CREDIT_MAX_OVERDUE', 'CNT_CREDIT_PROLONG',
'AMT_CREDIT_SUM', 'AMT_CREDIT_SUM_DEBT', 'AMT_CREDIT_SUM_LIMIT', 'AMT_CREDIT_SUM_OVERDUE']]
bureau_balance.STATUS = bureau_balance.STATUS.map({'C':0, 'X':0.1, '1':1, '2':2, '3':3, '4':4, '5':5})
 previous = previous[['SK_ID_CURR', 'SK_ID_PREV', 'AMT_ANNUITY', 'AMT_APPLICATION', 'AMT_CREDIT', 'AMT_GOODS_PRICE', 'NAME_CONTRACT_STATUS', 'DAYS_DECISION',
'CNT_PAYMENT', 'NAME_YIELD_GROUP', 'NFLAG_INSURED_ON_APPROVAL','SELLERPLACE_AREA']]
previous.NAME_CONTRACT_STATUS = previous.NAME_CONTRACT_STATUS.map(lambda x: 1 if x=='Refused' else 0)
 previous.NAME_YIELD_GROUP = previous.NAME_YIELD_GROUP.map({'XNA':0, 'low_noraml':1, 'low_action':1, 'middle':2, 'high':3})
 cash = cash[['SK_ID_PREV', 'MONTHS_BALANCE', 'SK_DPD', 'SK_DPD_DEF']]
credit = credit[['SK_ID_PREV', 'MONTHS_BALANCE', 'SK_DPD', 'SK_DPD_DEF', 'AMT_BALANCE', 'AMT_CREDIT_LIMIT_ACTUAL', 'AMT_INST_MIN_REGULARITY', 'AMT_TOTAL_RECEIVABLE']]
installments['date_diff'] = installments.DAYS_INSTALMENT - installments.DAYS_ENTRY_PAYMENT
installments['amount_diff'] = installments.AMT_INSTALMENT - installments.AMT_PAYMENT
installments = installments[['SK_ID_PREV', 'DAYS_INSTALMENT', 'amount_diff', 'date_diff']]
def replace_day_outliers(df):

 for col in df.columns:
if "DAYS" in col:
df[col] = df[col].replace({365243: np.nan})
 return df
app_train = replace_day_outliers(app_train)
 app_test = replace_day_outliers(app_test)
 bureau = replace_day_outliers(bureau)
 bureau_balance = replace_day_outliers(bureau_balance)
 previous = replace_day_outliers(previous)
 cash = replace_day_outliers(cash)
 credit = replace_day_outliers(credit)
 installments = replace_day_outliers(installments)
 start_date = pd.Timestamp("2018-01-01")
 for col in ['DAYS_CREDIT']:
bureau[col] = pd.to_timedelta(bureau[col], 'D')
 bureau['bureau_credit_application_date'] = start_date + bureau['DAYS_CREDIT']
bureau_balance['MONTHS_BALANCE'] = pd.to_timedelta(bureau_balance['MONTHS_BALANCE'], 'M')
 bureau_balance['bureau_balance_date'] = start_date + bureau_balance['MONTHS_BALANCE']
bureau = bureau_balance.drop(columns = ['DAYS_CREDIT'])
 bureau_balance = bureau_balance.drop(columns = ['MONTHS_BALANCE'])
 for col in ['DAYS_DECISION']:
previous[col] = pd.to_timedelta(previous[col], 'D')
 previous['previous_decision_date'] = start_date + previous['DAYS_DECISION']
previous = previous.drop(columns = ['DAYS_DECISION'])
 cash['MONTHS_BALANCE'] = pd.to_timedelta(cash['MONTHS_BALANCE'], 'M')
 cash['cash_balance_date'] = start_date + cash['MONTHS_BALANCE']
cash = cash.drop(columns = ['MONTHS_BALANCE'])
 credit['MONTHS_BALANCE'] = pd.to_timedelta(credit['MONTHS_BALANCE'], 'M')
 credit['credit_balance_date'] = start_date + credit['MONTHS_BALANCE']
credit = credit.drop(columns = ['MONTHS_BALANCE'])
 installments['DAYS_INSTALMENT'] = pd.to_timedelta(installments['DAYS_INSTALMENT'], 'D')
 installments['installments_due_date'] = start_date + installments['DAYS_INSTALMENT']
installments = installments.drop(columns = ['DAYS_INSTALMENT'])
 es = ft.EntitySet(id = 'clients')
 es = es.entity_from_dataframe(entity_id = 'app_train', dataframe = app_train,
index = 'SK_ID_CURR')
 es = es.entity_from_dataframe(entity_id = 'app_test', dataframe = app_test,
index = 'SK_ID_CURR')
 es = es.entity_from_dataframe(entity_id = 'bureau', dataframe = bureau,
index = 'SK_ID_BUREAU', time_index='bureau_credit_application_date')
 es = es.entity_from_dataframe(entity_id = 'bureau_balance', dataframe = bureau_balance,
make_index = True, index = 'bb_index',
time_index = 'bureau_balance_date')
 es = es.entity_from_dataframe(entity_id = 'previous', dataframe = previous,
index = 'SK_ID_PREV', time_index = 'previous_decision_date')
 es = es.entity_from_dataframe(entity_id = 'cash', dataframe = cash,
make_index = True, index = 'cash_index',
time_index = 'cash_balance_date')
 es = es.entity_from_dataframe(entity_id = 'installments', dataframe = installments,
make_index = True, index = 'installments_index',
time_index = 'installments_due_date')
 es = es.entity_from_dataframe(entity_id = 'credit', dataframe = credit,
make_index = True, index = 'credit_index',
time_index = 'credit_balance_date')
 r_app_bureau = ft.Relationship(es['app_train']['SK_ID_CURR'], es['bureau']['SK_ID_CURR'])
 r_test_app_bureau = ft.Relationship(es['app_test']['SK_ID_CURR'], es['bureau']['SK_ID_CURR'])
 r_bureau_balance = ft.Relationship(es['bureau']['SK_ID_BUREAU'], es['bureau_balance']['SK_ID_BUREAU'])
 r_app_previous = ft.Relationship(es['app_train']['SK_ID_CURR'], es['previous']['SK_ID_CURR'])
 r_test_app_previous = ft.Relationship(es['app_test']['SK_ID_CURR'], es['previous']['SK_ID_CURR'])
 r_previous_cash = ft.Relationship(es['previous']['SK_ID_PREV'], es['cash']['SK_ID_PREV'])
 r_previous_installments = ft.Relationship(es['previous']['SK_ID_PREV'], es['installments']['SK_ID_PREV'])
 r_previous_credit = ft.Relationship(es['previous']['SK_ID_PREV'], es['credit']['SK_ID_PREV'])
 es = es.add_relationships([r_app_bureau, r_test_app_bureau, r_bureau_balance,
						 r_app_previous, r_test_app_previous, r_previous_cash,
						 r_previous_installments, r_previous_credit])
 						
del app_train, app_test, bureau, bureau_balance, cash, credit, previous, installments
gc.collect()
print('prepare time:', str(time.time() -a))
a = time.time()
time_features, time_feature_names = ft.dfs(entityset = es, target_entity = 'app_train',
max_depth = 2,
agg_primitives = ['trend'],
features_only = False, verbose = True,
chunk_size = 30000,
ignore_entities = ['app_test'])
 print('feature time:,', str(time.time() -a))
time_features.reset_index().to_csv('trend3_train.csv', index=False)
 time_features_test, time_feature_names = ft.dfs(entityset = es, target_entity = 'app_test',
max_depth = 2,
agg_primitives = ['trend'],
features_only = False, verbose = True,
chunk_size = 25000,
ignore_entities = ['app_train'])
 time_features_test.reset_index().to_csv('trend4_test.csv', index=False)
<load_from_csv>
mean_error = []
for day in range(2, train_baseline.Day.max() + 1):
val = train_baseline[train_baseline.Day == day]
p = val.Last_Week_Sales.values
error = rmspe(val.Sales.values, p)
 mean_error.append(error)
 print('Mean Error = %.5f' % np.mean(mean_error))
full_df = pd.read_csv(".. /input/home-credit-default-risk/application_train.csv")
 test_df = pd.read_csv(".. /input/home-credit-default-risk/application_test.csv")
 test_df 
<categorify>
class LabelEncoderPipelineFriendly(LabelEncoder):
def fit(self, X, y=None):

 super(LabelEncoderPipelineFriendly, self ).fit(X)
 def transform(self, X, y=None):
return super(LabelEncoderPipelineFriendly, self ).transform(X ).reshape(-1, 1)
 def fit_transform(self, X, y=None):
return super(LabelEncoderPipelineFriendly, self ).fit(X ).transform(X ).reshape(-1, 1 )
full_df = pd.get_dummies(full_df)
 test_df = pd.get_dummies(test_df)
 full_df, test_df = full_df.align(test_df, join = 'inner', axis = 1)
 test_df.head(10)
 test_df.count()<normalization>
def prepare_pipeline(df):
def get_DayOfWeek(df):
return df["DayOfWeek"]
def get_Open(df):
return df[["Open"]]
def get_Promo(df):
return df[["Promo"]]
def get_StateHoliday(df):
return df["StateHoliday"]
def get_SchoolHoliday(df):
return df[["SchoolHoliday"]]
def get_StoreType(df):
return df["StoreType"]
def get_Assortment(df):
return df["Assortment"]
def get_Promo2Active(df):
return df[["Promo2Active"]]
def get_CompetitionActive(df):
return df[["CompetitionActive"]]
def get_CompetitionDistance(df):
return df[["CompetitionDistance"]]
def get_DayOfYear(df):
return df["DayOfYear"]
p = make_union(*[
make_pipeline(FunctionTransformer(get_DayOfWeek, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder()),
make_pipeline(FunctionTransformer(get_Open, validate=False),
Imputer(strategy="most_frequent")) ,
make_pipeline(FunctionTransformer(get_Promo, validate=False)) ,
make_pipeline(FunctionTransformer(get_StateHoliday, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder()),
make_pipeline(FunctionTransformer(get_SchoolHoliday, validate=False)) ,
make_pipeline(FunctionTransformer(get_StoreType, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder()),
make_pipeline(FunctionTransformer(get_Assortment, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder()),
make_pipeline(FunctionTransformer(get_Promo2Active, validate=False)) ,
make_pipeline(FunctionTransformer(get_CompetitionActive, validate=False)) ,
make_pipeline(FunctionTransformer(get_CompetitionDistance, validate=False),
Imputer() ,
StandardScaler()),
make_pipeline(FunctionTransformer(get_DayOfYear, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder())
 ])
 return p
<train_model>
pipeline = prepare_pipeline(train )
def model(features, test_features, encoding = 'ohe', n_folds = 5):

 train_ids = features['SK_ID_CURR']
test_ids = test_features['SK_ID_CURR']
labels = features['TARGET']
features = features.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test_features = test_features.drop(columns = ['SK_ID_CURR'])
 if encoding == 'ohe':
features = pd.get_dummies(features)
 test_features = pd.get_dummies(test_features)
 features, test_features = features.align(test_features, join = 'inner', axis = 1)
 cat_indices = 'auto'
elif encoding == 'le':
label_encoder = LabelEncoder()
cat_indices = []
for i, col in enumerate(features):
if features[col].dtype == 'object':
features[col] = label_encoder.fit_transform(np.array(features[col].astype(str)).reshape(( -1,)))
 test_features[col] = label_encoder.transform(np.array(test_features[col].astype(str)).reshape(( -1,)))
 cat_indices.append(i)
 else:
raise ValueError("Encoding must be either 'ohe' or 'le'")
 print('Training Data Shape: ', features.shape)
 print('Testing Data Shape: ', test_features.shape)
 feature_names = list(features.columns)
 features = np.array(features)
 test_features = np.array(test_features)
 k_fold = KFold(n_splits = n_folds, shuffle = True, random_state = 50)
 feature_importance_values = np.zeros(len(feature_names))
test_predictions = np.zeros(test_features.shape[0])
 out_of_fold = np.zeros(features.shape[0])
 valid_scores = []
train_scores = []
for train_indices, valid_indices in k_fold.split(features):
train_features, train_labels = features[train_indices], labels[train_indices]
valid_features, valid_labels = features[valid_indices], labels[valid_indices]
model = lgb.LGBMClassifier(n_estimators=500, objective = 'binary',
class_weight = 'balanced', learning_rate = 0.5,
reg_alpha = 0.7, reg_lambda = 0.5,
subsample = 0.5, n_jobs = -1, random_state = 30)
 model.fit(train_features, train_labels, eval_metric = 'auc',
eval_set = [(valid_features, valid_labels),(train_features, train_labels)],
eval_names = ['valid', 'train'], categorical_feature = cat_indices,
early_stopping_rounds = 25, verbose = 200)
 best_iteration = model.best_iteration_
feature_importance_values += model.feature_importances_ / k_fold.n_splits
test_predictions += model.predict_proba(test_features, num_iteration = best_iteration)[:, 1] / k_fold.n_splits
out_of_fold[valid_indices] = model.predict_proba(valid_features, num_iteration = best_iteration)[:, 1]
valid_score = model.best_score_['valid']['auc']
train_score = model.best_score_['train']['auc']
valid_scores.append(valid_score)
 train_scores.append(train_score)
 gc.enable()
del model, train_features, valid_features
gc.collect()
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': test_predictions})
 feature_importances = pd.DataFrame({'feature': feature_names, 'importance': feature_importance_values})
 valid_auc = roc_auc_score(labels, out_of_fold)
 valid_scores.append(valid_auc)
 train_scores.append(np.mean(train_scores))
fold_names = list(range(n_folds))
fold_names.append('overall')
 metrics = pd.DataFrame({'fold': fold_names,
'train': train_scores,
'valid': valid_scores})
 return submission, feature_importances, metrics<save_to_csv>
x_train, y_train = pipeline.fit_transform(train), train.Sales
x_test = pipeline.transform(test )
submission, fi, metrics = model(full_df, test_df)
 print('Baseline metrics')
 print(metrics)
 submission.to_csv('baseline_lgb.csv', index = False)
<import_modules>
params = {"boosting_type" : ["gbdt"],
"learning_rate": [0.1],
"n_estimators": [200],
"objective": ["regression"],
"reg_alpha": [1.0],
"reg_lambda": [1.0],
"random_state": [0],
"n_jobs": [-1]
}
import numpy as np
import pandas as pd<import_modules>
gs = GridSearchCV(LGBMRegressor() , params, scoring=rmspe_scorer, cv=2, n_jobs=1)
 gs.fit(x_train, y_train )
from tqdm.notebook import tqdm
import random
import gc
import time<import_modules>
prediction = gs.predict(x_test )
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import StratifiedKFold
from sklearn.ensemble import ExtraTreesClassifier<import_modules>
pd.DataFrame({"Id": test.Id, "Sales": prediction} ).to_csv("submission.csv", sep=",", index=False )
import lightgbm as lgb<set_options>
def prepare_pipeline_ts(df, min_shift, max_shift):
def get_shifted_date(df, for_sales=False):
return(df.Date.min() + pd.DateOffset(days_to_shift))
def get_DayOfWeek(df):
return df["DayOfWeek"]
def get_Open(df):
return df[["Open"]]
def get_Promo(df):
return df[["Promo"]]
def get_StateHoliday(df):
return df["StateHoliday"]
def get_SchoolHoliday(df):
return df[["SchoolHoliday"]]
def get_StoreType(df):
return df["StoreType"]
def get_Assortment(df):
return df["Assortment"]
def get_Promo2Active(df):
return df[["Promo2Active"]]
def get_CompetitionActive(df):
return df[["CompetitionActive"]]
def get_CompetitionDistance(df):
return df[["CompetitionDistance"]]
def get_DayOfYear(df):
return df["DayOfYear"]
def get_previous_sales(df):
sales = df[["Store", "Sales"]].copy()
for day in range(min_shift, max_shift + 1):
sales["Last-{}_Day_Sales".format(day)] = sales.groupby("Store")["Sales"].shift(day)
 sales["Last-{}_Day_Diff".format(day)] = sales.groupby("Store")["Last-{}_Day_Sales".format(day)].diff()
return sales.drop(["Store", "Sales"], axis=1)
 p = make_union(*[
make_pipeline(FunctionTransformer(get_DayOfWeek, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder()),
make_pipeline(FunctionTransformer(get_Open, validate=False),
Imputer(strategy="most_frequent")) ,
make_pipeline(FunctionTransformer(get_Promo, validate=False)) ,
make_pipeline(FunctionTransformer(get_StateHoliday, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder()),
make_pipeline(FunctionTransformer(get_SchoolHoliday, validate=False)) ,
make_pipeline(FunctionTransformer(get_StoreType, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder()),
make_pipeline(FunctionTransformer(get_Assortment, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder()),
make_pipeline(FunctionTransformer(get_Promo2Active, validate=False)) ,
make_pipeline(FunctionTransformer(get_CompetitionActive, validate=False)) ,
make_pipeline(FunctionTransformer(get_CompetitionDistance, validate=False),
Imputer() ,
StandardScaler()),
make_pipeline(FunctionTransformer(get_DayOfYear, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder()),
make_pipeline(FunctionTransformer(get_previous_sales, validate=False),
Imputer() ,
StandardScaler())
 ])
 return p
gc.enable()<load_from_csv>
test_size = len(test)
 min_shift =(test.Date.max() - test.Date.min() ).days
max_shift = 180
to_drop = len(train[train.Date < train.Date.min() + pd.DateOffset(max_shift)] )
train_data = pd.read_csv('/kaggle/input/home-credit-default-risk/application_train.csv',
na_values=['XNA', 'XAP'], na_filter=True)
 test_data = pd.read_csv('/kaggle/input/home-credit-default-risk/application_test.csv',
na_values=['XNA', 'XAP'], na_filter=True )<count_values>
full = pd.concat([train, test], ignore_index=True)
 pipeline_ts = prepare_pipeline_ts(train, min_shift, max_shift)
 full_transformed = pipeline_ts.fit_transform(full)
 x_train_ts, y_train_ts = full_transformed[to_drop:-test_size], train.Sales[to_drop:]
x_test_ts = full_transformed[-test_size:]
train_counts = train_data.count().sort_values() /len(train_data)
 test_counts = test_data.count().sort_values() /len(test_data )<define_variables>
gs_ts = GridSearchCV(LGBMRegressor() , params, scoring=rmspe_scorer, cv=2, n_jobs=1)
 gs_ts.fit(x_train_ts, y_train_ts )
cols = set(train_counts[(train_counts < 1)&(train_counts > 0.99)].index)- set(test_counts[(test_counts < 1)&(test_counts > 0.9)].index )<correct_missing_values>
prediction_ts = gs_ts.predict(x_test_ts )
train_data.dropna(subset=cols, inplace=True )<prepare_x_and_y>
pd.DataFrame({"Id": test.Id, "Sales": prediction_ts} ).to_csv("submission_ts.csv", sep=",", index=False )