Kaggle competition: Rossmann Store Sales
del(train_data )<load_from_csv>
train=pd.read_csv('/kaggle/input/rossmann-store-sales/train.csv')
 test=pd.read_csv('/kaggle/input/rossmann-store-sales/test.csv')
 store=pd.read_csv('/kaggle/input/rossmann-store-sales/store.csv' )
bureau = pd.read_csv('.. /input/home-credit-default-risk/bureau.csv',
na_values=['XNA', 'XAP'], na_filter=True )<feature_engineering>
train['StateHoliday'].value_counts()
application_data['IS_IN_BUREAU'] = 0<feature_engineering>
train['Store'].value_counts().head(20 )
application_data.loc[application_data.SK_ID_CURR.isin(bureau.SK_ID_CURR.unique()), 'IS_IN_BUREAU'] = 1<sort_values>
train['Store'].value_counts().tail(20 )
appl_counts = application_data.count().sort_values() /len(application_data)
 appl_counts[(appl_counts < 1)]<feature_engineering>
train['DayOfWeek'].value_counts()
application_data['HAS_SOCIAL_CIRCLE'] = 0<feature_engineering>
train['Open'].value_counts()
application_data.loc[~application_data.OBS_30_CNT_SOCIAL_CIRCLE.isnull() , 'HAS_SOCIAL_CIRCLE'] = 1<feature_engineering>
train['Promo'].value_counts()
application_data['AMT_CREDIT_FRAC'] = application_data.AMT_CREDIT / application_data.AMT_INCOME_TOTAL<feature_engineering>
train['Date']=pd.to_datetime(train['Date'],format='%Y-%m-%d' )
application_data['AMT_CREDIT_FRAC'] = application_data.AMT_ANNUITY / application_data.AMT_CREDIT<feature_engineering>
train.isna().sum()
application_data['AMT_GOODS_FRAC'] = application_data.AMT_GOODS_PRICE / application_data.AMT_CREDIT<feature_engineering>
test.isna().sum()
application_data['AMT_ANNUITY_FRAC'] = application_data.AMT_ANNUITY / application_data.AMT_INCOME_TOTAL<feature_engineering>
store.isna().sum()
application_data['AMT_DPD_DEF'] = application_data.DEF_30_CNT_SOCIAL_CIRCLE + application_data.OBS_30_CNT_SOCIAL_CIRCLE<load_from_csv>
store1=train[train['Store']==1]
bureau_balance = pd.read_csv('.. /input/home-credit-default-risk/bureau_balance.csv',
na_values=['XNA', 'XAP'], na_filter=True )<count_unique_values>
store1[store1['Sales']==0]
bureau = bureau[bureau.SK_ID_CURR.isin(application_data.SK_ID_CURR.unique())]<categorify>
test_store1=test[test['Store']==1]
test_store1['Date']=pd.to_datetime(test_store1['Date'],format='%Y-%m-%d' )
bureau_balance = pd.get_dummies(bureau_balance )<sort_values>
test_store1['Open'].value_counts()
bureau_balance = bureau_balance.sort_values(['SK_ID_BUREAU', 'MONTHS_BALANCE'] )<create_dataframe>
store[store['Store']==1].T
temp = bureau_balance.groupby('SK_ID_BUREAU' ).size().to_frame()
temp = temp.rename(columns={0: 'COUNT'})
 temp.reset_index(inplace=True )<groupby>
store[~store['Promo2SinceYear'].isna() ].iloc[0]
bureau_balance = bureau_balance.groupby('SK_ID_BUREAU' ).agg({'last', 'sum', 'mean'} )<string_transform>
store=pd.read_csv('/kaggle/input/rossmann-store-sales/store.csv' )
bureau_balance.columns = bureau_balance.columns.map('_'.join )<filter>
store['Promo2SinceWeek']=store['Promo2SinceWeek'].fillna(0 )
bureau_balance.reset_index(inplace=True )<merge>
store['Promo2SinceYear']=store['Promo2SinceYear'].fillna(store['Promo2SinceYear'].mode().iloc[0] )
bureau_balance = bureau_balance.merge(temp, how='left', on='SK_ID_BUREAU' )<rename_columns>
store['PromoInterval']=store['PromoInterval'].fillna(store['PromoInterval'].mode().iloc[0] )
bureau_balance.columns = bureau_balance.columns.map(lambda x : 'BLN_' + x if x != 'SK_ID_BUREAU' else x )<merge>
store['CompetitionDistance']=store['CompetitionDistance'].fillna(store['CompetitionDistance'].max())
 store['CompetitionOpenSinceMonth']=store['CompetitionOpenSinceMonth'].fillna(store['CompetitionOpenSinceMonth'].mode().iloc[0])
 store['CompetitionOpenSinceYear']=store['CompetitionOpenSinceYear'].fillna(store['CompetitionOpenSinceYear'].mode().iloc[0] )
bureau = bureau.merge(bureau_balance, how='left', on='SK_ID_BUREAU' )<drop_column>
store.isna().sum()
bureau.drop(columns='SK_ID_BUREAU', inplace=True )<drop_column>
df=train.merge(store,on='Store',how='left' )
del(bureau_balance )<sort_values>
df.isna().sum()
bureau = bureau.sort_values(['SK_ID_CURR', 'DAYS_CREDIT'] )<categorify>
df['day']=df['Date'].dt.day
df['year']=df['Date'].dt.year
df['month']=df['Date'].dt.month
bureau = pd.get_dummies(bureau, dummy_na=True )<create_dataframe>
df['StateHoliday']=df['StateHoliday'].apply(lambda x:'0' if x==0 or x=='0' else x )
temp = bureau.groupby('SK_ID_CURR' ).size().to_frame()
temp = temp.rename(columns={0: 'COUNT'})
 temp.reset_index(inplace=True )<groupby>
df['StateHoliday']=df['StateHoliday'].map({'0':0,'a':1,'b':2,'c':3})
 df['StateHoliday']=df['StateHoliday'].astype(int )
bureau = bureau.groupby('SK_ID_CURR' ).agg({'sum', 'mean', 'max'} )<string_transform>
df['StoreType'].value_counts()
bureau.columns = bureau.columns.map('_'.join )<merge>
df['StoreType']=df['StoreType'].map({'a':0,'b':1,'c':2,'d':3} )
bureau = bureau.merge(temp, how='left', on='SK_ID_CURR' )<merge>
df['Assortment']=df['Assortment'].map({'a':0,'b':1,'c':2} )
application_data = application_data.merge(bureau, how='left', on='SK_ID_CURR' )<drop_column>
df['Assortment']=df['Assortment'].astype(int )
del(bureau )<load_from_csv>
df['PromoInterval'].value_counts()
prev_application = pd.read_csv('/kaggle/input/home-credit-default-risk/previous_application.csv',
na_values=['XNA', 'XAP'], na_filter=True )<categorify>
df['PromoInterval']=df['PromoInterval'].map({'Jan,Apr,Jul,Oct':0,'Feb,May,Aug,Nov':1,'Mar,Jun,Sept,Dec':2} )
prev_application = pd.get_dummies(prev_application, dummy_na=True )<drop_column>
df['PromoInterval']=df['PromoInterval'].astype(int )
prev_application.drop(columns='SK_ID_PREV', inplace=True )<sort_values>
df=df.drop('Date',1 )
prev_application = prev_application.sort_values(['SK_ID_CURR', 'DAYS_DECISION'] )<create_dataframe>
y=np.log1p(df['Sales'] )
temp = prev_application.groupby('SK_ID_CURR' ).size().to_frame()
temp = temp.rename(columns={0: 'COUNT'})
 temp.reset_index(inplace=True )<groupby>
X=df.drop(['Sales','Customers'],1 )
prev_application = prev_application.groupby('SK_ID_CURR' ).agg(['max', 'sum', 'mean'] )<string_transform>
X_train,X_val,y_train,y_val=train_test_split(X,y,random_state=1,test_size=0.2 )
prev_application.columns = prev_application.columns.map('_'.join )<merge>
dt=DecisionTreeRegressor(max_depth=11,random_state=1 ).fit(X_train,y_train )
prev_application = prev_application.merge(temp, how='left', on='SK_ID_CURR' )<rename_columns>
y_pred_val=dt.predict(X_val )
prev_application.columns = prev_application.columns.map(lambda x : 'PREV_' + x if x != 'SK_ID_CURR' else x )<merge>
print(r2_score(y_val,y_pred_val))
print(np.sqrt(mean_squared_error(y_val,y_pred_val)) )
application_data = application_data.merge(prev_application, how='left', on='SK_ID_CURR' )<drop_column>
y_val_exp=np.exp(y_val)-1
y_pred_val_exp=np.exp(y_pred_val)-1
np.sqrt(mean_squared_error(y_val_exp,y_pred_val_exp))
del(prev_application )<load_from_csv>
r2_score(y_val_exp,y_pred_val_exp )
pos_cash_balance = pd.read_csv('/kaggle/input/home-credit-default-risk/POS_CASH_balance.csv',
na_values=['XNA', 'XAP'], na_filter=True )<sort_values>
!pip install pydotplus
pos_cash_balance = pos_cash_balance.sort_values(['SK_ID_CURR', 'SK_ID_PREV', 'MONTHS_BALANCE'] )<create_dataframe>
def draw_tree(model, columns):
graphviz_path = 'C:\Program Files(x86)\Graphviz2.38/bin/'
os.environ["PATH"] += os.pathsep + graphviz_path
dot_data = StringIO()
tree.export_graphviz(model,
out_file=dot_data,
feature_names=columns)
 graph = pydotplus.graph_from_dot_data(dot_data.getvalue())
 return Image(graph.create_png() )
temp = pos_cash_balance.groupby(['SK_ID_CURR', 'SK_ID_PREV'] ).size().to_frame()
temp = temp.rename(columns={0: 'BLN_COUNT'})
 temp.reset_index(inplace=True )<categorify>
avg_cust=df.groupby(['Store'])[['Customers']].mean().astype(int)
 test1=test.merge(avg_cust,on='Store',how='left' )
pos_cash_balance = pd.get_dummies(pos_cash_balance, dummy_na=True )<groupby>
test_merged=test1.merge(store,on='Store',how='left' )
pos_cash_balance = pos_cash_balance.groupby(['SK_ID_PREV', 'SK_ID_CURR'] ).agg(['sum', 'mean', 'max'] )<string_transform>
test_merged['Open'].fillna(1,inplace=True )
pos_cash_balance.columns = pos_cash_balance.columns.map('_'.join )<filter>
test_merged.isna().sum()
pos_cash_balance.reset_index(inplace=True )<merge>
test_merged['Date']=pd.to_datetime(test_merged['Date'],format='%Y-%m-%d')
 test_merged['day']=test_merged['Date'].dt.day
test_merged['month']=test_merged['Date'].dt.month
test_merged['year']=test_merged['Date'].dt.year
pos_cash_balance = pos_cash_balance.merge(temp, how='left', on=['SK_ID_CURR', 'SK_ID_PREV'] )<drop_column>
test_merged=test_merged.drop('Date',1 )
pos_cash_balance.drop(columns='SK_ID_PREV', inplace=True )<create_dataframe>
test_merged['StateHoliday']=test_merged['StateHoliday'].apply(lambda x:'0' if x==0 or x=='0' else x )
temp = pos_cash_balance.groupby('SK_ID_CURR' ).size().to_frame()
temp = temp.rename(columns={0: 'COUNT'})
 temp.reset_index(inplace=True )<groupby>
test_merged['StateHoliday'].value_counts()
pos_cash_balance = pos_cash_balance.groupby(['SK_ID_CURR'] ).agg(['sum', 'mean', 'max'] )<string_transform>
test_merged['StoreType'].value_counts()
pos_cash_balance.columns = pos_cash_balance.columns.map('_'.join )<filter>
test_merged['Assortment'].value_counts()
pos_cash_balance.reset_index(inplace=True )<merge>
test_merged['PromoInterval'].value_counts()
pos_cash_balance = pos_cash_balance.merge(temp, how='left', on='SK_ID_CURR' )<rename_columns>
test_merged['StateHoliday']=test_merged['StateHoliday'].map({'0':0,'a':1})
 test_merged['StateHoliday']=test_merged['StateHoliday'].astype(int)
 test_merged['StoreType']=test_merged['StoreType'].map({'a':0,'b':1,'c':2,'d':3})
 test_merged['StoreType']=test_merged['StoreType'].astype(int)
 test_merged['Assortment']=test_merged['Assortment'].map({'a':0,'b':1,'c':2})
 test_merged['Assortment']=test_merged['Assortment'].astype(int)
 test_merged['PromoInterval']=test_merged['PromoInterval'].map({'Jan,Apr,Jul,Oct':0,'Feb,May,Aug,Nov':1,'Mar,Jun,Sept,Dec':2})
 test_merged['PromoInterval']=test_merged['PromoInterval'].astype(int )
pos_cash_balance.columns = pos_cash_balance.columns.map(lambda x : 'CSH_' + x if x != 'SK_ID_CURR' else x )<merge>
test_merged1=test_merged.drop('Id',1 )
application_data = application_data.merge(pos_cash_balance, how='left', on='SK_ID_CURR' )<drop_column>
y_pred=dt.predict(test_merged1[X_train.columns] )
del(pos_cash_balance )<load_from_csv>
y_pred_exp=np.exp(y_pred)-1
credit_card_balance = pd.read_csv('/kaggle/input/home-credit-default-risk/credit_card_balance.csv',
na_values=['XNA', 'XAP'], na_filter=True )<categorify>
submission_pred=pd.DataFrame(test_merged['Id'],columns=['Id'] )
credit_card_balance = pd.get_dummies(credit_card_balance, dummy_na=True )<sort_values>
submission_pred['Sales']=y_pred_exp
credit_card_balance = credit_card_balance.sort_values(['SK_ID_CURR', 'SK_ID_PREV', 'MONTHS_BALANCE'] )<create_dataframe>
submission_pred['Id']=np.arange(1,len(submission_pred)+1 )
temp = credit_card_balance.groupby(['SK_ID_CURR', 'SK_ID_PREV'] ).size().to_frame()
temp = temp.rename(columns={0: 'COUNT'})
 temp.reset_index(inplace=True )<groupby>
submission_pred.to_csv('Submission.csv',index=False )
credit_card_balance = credit_card_balance.groupby(['SK_ID_PREV', 'SK_ID_CURR'] ).agg(['sum', 'mean', 'max'] )<string_transform>
def ToWeight(y):
w = np.zeros(y.shape, dtype=float)
 ind = y != 0
w[ind] = 1./(y[ind]**2)
 return w
def rmspe(y, yhat):
w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y - yhat)**2))
return rmspe
rmse_val=np.sqrt(mean_squared_error(y_val_exp,y_pred_val_exp))
rmspe_val=rmspe(y_val_exp,y_pred_val_exp)
 print(rmse_val,rmspe_val )
credit_card_balance.columns = credit_card_balance.columns.map('_'.join )<filter>
params={'max_depth':list(range(5,20)) }
base_model=DecisionTreeRegressor()
cv_model=GridSearchCV(base_model,param_grid=params,return_train_score=True ).fit(X_train,y_train )
credit_card_balance.reset_index(inplace=True )<merge>
df_cv_results=pd.DataFrame(cv_model.cv_results_ ).sort_values(by='mean_test_score',ascending=False )
credit_card_balance = credit_card_balance.merge(temp, how='left', on=['SK_ID_CURR', 'SK_ID_PREV'] )<drop_column>
df_cv_results=pd.DataFrame(cv_model.cv_results_ ).sort_values(by='mean_test_score',ascending=False)[['param_max_depth','mean_test_score','mean_train_score']]
df_cv_results
credit_card_balance.drop(columns='SK_ID_PREV', inplace=True )<groupby>
def get_rmspe_score(model,input_values,y_actual):
y_predicted=model.predict(input_values)
 y_actual=np.exp(y_actual)-1
y_predicted=np.exp(y_predicted)-1
score=rmspe(y_actual,y_predicted)
 return score
params={'max_depth':list(range(5,8)) }
base_model=DecisionTreeRegressor()
cv_model=GridSearchCV(base_model,param_grid=params,return_train_score=True,scoring=get_rmspe_score ).fit(X_train,y_train)
 pd.DataFrame(cv_model.cv_results_)[['params','mean_test_score','mean_train_score']]
credit_card_balance = credit_card_balance.groupby(['SK_ID_CURR'] ).agg(['sum', 'mean'] )<string_transform>
model_ada=AdaBoostRegressor(n_estimators=5 ).fit(X_train,y_train )
credit_card_balance.columns = credit_card_balance.columns.map('_'.join )<filter>
model_ada.estimators_[0]
credit_card_balance.reset_index(inplace=True )<rename_columns>
def draw_tree(model, columns):
graphviz_path = 'C:\Program Files(x86)\Graphviz2.38/bin/'
os.environ["PATH"] += os.pathsep + graphviz_path
dot_data = StringIO()
tree.export_graphviz(model,
out_file=dot_data,
feature_names=columns)
 graph = pydotplus.graph_from_dot_data(dot_data.getvalue())
 return Image(graph.create_png() )
credit_card_balance.columns = credit_card_balance.columns.map(lambda x : 'CRD_' + x if x != 'SK_ID_CURR' else x )<merge>
import xgboost as xgb
application_data = application_data.merge(credit_card_balance, how='left', on='SK_ID_CURR' )<drop_column>
dtrain=xgb.DMatrix(X_train,y_train)
 dvalidate=xgb.DMatrix(X_val,y_val)
 param={'max_depth':5,'eta':1,'ojective':'reg:linear'}
model_xg=xgb.train(param,dtrain,200)
 pred_y=model_xg.predict(dvalidate)
 val_y_inv=np.exp(y_val)-1
pred_y_inv=np.exp(pred_y)-1
rmspe_val=rmspe(val_y_inv,pred_y_inv)
 print(rmspe_val )
del(credit_card_balance )<load_from_csv>
test_merged=test_merged.drop(['Id','Customers'],1 )
installments_payments = pd.read_csv('/kaggle/input/home-credit-default-risk/installments_payments.csv',
na_values=['XNA', 'XAP'], na_filter=True )<sort_values>
y_pred_xg=model_xg.predict(xgb.DMatrix(test_merged[X_train.columns]))
installments_payments = installments_payments.sort_values(['SK_ID_CURR', 'SK_ID_PREV', 'DAYS_ENTRY_PAYMENT'] )<create_dataframe>
y_pred_xg_exp=np.exp(y_pred_xg)-1
<correct_missing_values><EOS> 
submission_predicted1 = pd.DataFrame({'Id': test['Id'], 'Sales': y_pred_xg_exp})
 testop0=(test[test['Open']==0]['Open'] ).index
Sales1=[]
for i in range(41088):
if i in testop0:
Sales1.append(0)
 else:
Sales1.append(submission_predicted1['Sales'][i])
 submission_predicted1['Sales']=Sales1
print(submission_predicted1.head())
 submission_predicted1.to_csv('submission.csv', index=False )