Kaggle competition: COVID19 Global Forecasting (Week 2)
bureau['AMT_CREDIT_SUM_LIMIT_v1'] = bureau['AMT_CREDIT_SUM_LIMIT'].replace(0, np.nan)
 tmp = bureau[(bureau['CREDIT_TYPE'].isin(["Credit card"])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM_LIMIT_v1'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_limit_bureau'] = tmp_merge['des1']
df['max_limit_bureau'] = tmp_merge['des2']
df['mean_limit_bureau'] = tmp_merge['des3']
df['sum_limit_bureau'] = tmp_merge['des4'] 
<categorify>
init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore' )
bureau['AMT_CREDIT_MAX_OVERDUE_v1'] = bureau['AMT_CREDIT_MAX_OVERDUE'].replace(0,np.nan)
 tmp = bureau[(bureau['CREDIT_TYPE'].isin(["Consumer credit"])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT_MAX_OVERDUE_v1'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_overdue_bureau'] = tmp_merge['des1'].fillna(0)
 df['max_overdue_bureau'] = tmp_merge['des2'].fillna(0)
 df['mean_overdue_bureau'] = tmp_merge['des3'].fillna(0)
 df['sum_overdue_bureau'] = tmp_merge['des4'].fillna(0 )<merge>
ndf=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv')
 mdfw=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv' )
bureau['tmp'] = bureau['AMT_CREDIT_SUM_DEBT']/bureau['AMT_CREDIT_SUM']
tmp = bureau.groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_ratio_debt_credit_bureau'] = tmp_merge['des1']
df['max_ratio_debt_credit_bureau'] = tmp_merge['des2']
df['mean_ratio_debt_credit_bureau'] = tmp_merge['des3']
df['sum_ratio_debt_credit_bureau'] = tmp_merge['des4']<merge>
def parser(x):
try:
return datetime.strptime(x,'%Y-%m-%d')
 except:
return datetime.strptime(x,' %Y-%m-%d')
 for i in range(0,len(ndf)) :
ndf.Date[i]=parser(ndf.Date[i])
 for i in range(0,len(mdfw)) :
mdfw.Date[i]=parser(mdfw.Date[i] )
bureau['tmp'] = bureau['AMT_ANNUITY'].fillna(0)
 tmp = bureau.groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_annuity_bureau_v2'] = tmp_merge['des1']
df['max_annuity_bureau_v2'] = tmp_merge['des2']
df['mean_annuity_bureau_v2'] = tmp_merge['des3']
df['sum_annuity_bureau_v2'] = tmp_merge['des4']<load_from_csv>
pdfw=ndf[ndf['Province_State'].isna() ==True]
countries=list(pdfw.Country_Region.unique())
 mdfw.head()
t1=mdfw[['Country_Region','ConfirmedCases','Date']]
t1.index.name=''
p1=pd.DataFrame(t1[t1['Country_Region']=='Afghanistan'] ).set_index('Date' ).loc['2020-03-28']
p=pd.DataFrame(p1 ).T
i='Afghanistan'
install = pd.read_csv(".. /input/installments_payments.csv" )<merge>
for i in countries:
if i not in ['Denmark', 'France', 'Netherlands', 'United Kingdom']:
t1=mdfw[['Country_Region','ConfirmedCases','Date']]
t1.index.name=''
t1=t1[t1['Country_Region']==i]
w1=t1[t1['Date']>=datetime.strptime('2020-03-19', '%Y-%m-%d')]
w1=w1[t1['Date']<=datetime.strptime('2020-03-28', '%Y-%m-%d')].set_index('Date')
 w1.index.name=''
cc1=ndf[ndf['Country_Region']==i].set_index('Date')
 cc=mdfw[mdfw['Country_Region']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('Fatalities',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.ConfirmedCases.diff()
cc.EverydayCC.iloc[0]=0.0
try:
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['ConfirmedCases','EverydayCC'])
 final.drop('ConfirmedCases',axis=1)
 final['ConfirmedCases']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Country_Region']=i
w1=w1.append(final)
 w1['Province_State']=cc1['Province_State']
w1['ForecastId']=cc1['ForecastId']
p =p.append(w1)
 except:
print('Exception')
 else:
kkk=mdfw[mdfw['Country_Region']==i ]
lol=kkk[kkk.Province_State.isna() ]
t1=lol[['Country_Region','ConfirmedCases','Date']]
t1.index.name=''
o1=pd.DataFrame(t1[t1['Country_Region']==i] ).set_index('Date')
 o1.index.name=''
t1=o1.loc['2020-03-19':'2020-03-28']
cc2=ndf[ndf['Country_Region']==i].set_index('Date')
 cc1=cc2[cc2['Province_State'].isna() ]
cc=lol[lol['Country_Region']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('Fatalities',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.ConfirmedCases.diff()
cc.EverydayCC.iloc[0]=0.0
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['ConfirmedCases','EverydayCC'])
 final.drop('ConfirmedCases',axis=1)
 final['ConfirmedCases']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Country_Region']=i
t1=t1.append(final)
 t1['Country_Region']=cc1['Country_Region']
t1['ForecastId']=cc1['ForecastId']
p =p.append(t1)
tmp = install.groupby(['SK_ID_PREV','SK_ID_CURR'] ).agg({'NUM_INSTALMENT_NUMBER':["count","max"]} ).reset_index()
tmp.columns = ['SK_ID_PREV','SK_ID_CURR','count','max']
tmp['delta'] = tmp['count'] - tmp['max']
tmp = tmp.merge(previous[['SK_ID_PREV','CNT_PAYMENT','DAYS_LAST_DUE','NAME_CONTRACT_TYPE']], on=['SK_ID_PREV'], how='left')
 tmp_1 = tmp.groupby(['SK_ID_CURR'])['delta'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp_1.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp_1, on=['SK_ID_CURR'], how='left')
 df['min_delta_num_install'] = tmp_merge['des1']
df['max_delta_num_install'] = tmp_merge['des2']
df['mean_delta_num_install'] = tmp_merge['des3']
df['sum_delta_num_install'] = tmp_merge['des4']<merge>
dp1=p[1:]
tmp = install.groupby(['SK_ID_PREV','SK_ID_CURR'] ).agg({'NUM_INSTALMENT_VERSION':["count","max"]} ).reset_index()
tmp.columns = ['SK_ID_PREV','SK_ID_CURR','count','max']
tmp = tmp.merge(previous[['SK_ID_PREV','CNT_PAYMENT','DAYS_LAST_DUE','NAME_CONTRACT_TYPE']], on=['SK_ID_PREV'], how='left')
 tmp_1 = tmp.groupby(['SK_ID_CURR'])['max'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp_1.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp_1, on=['SK_ID_CURR'], how='left')
 df['min_max_version_install'] = tmp_merge['des1']
df['max_max_version_install'] = tmp_merge['des2']
df['mean_max_version_install'] = tmp_merge['des3']
df['sum_max_version_install'] = tmp_merge['des4']<merge>
t1=mdfw[['Country_Region','Fatalities','Date']]
t1.index.name=''
t1=pd.DataFrame(t1[t1['Country_Region']=='Afghanistan'].iloc[0] ).T
p=t1.set_index('Date')
tmp = install.groupby(['SK_ID_PREV','SK_ID_CURR'] ).agg({'NUM_INSTALMENT_NUMBER':["count","max"]} ).reset_index()
tmp.columns = ['SK_ID_PREV','SK_ID_CURR','count','max']
tmp['delta'] = tmp['count']/tmp['max']
tmp = tmp.merge(previous[['SK_ID_PREV','CNT_PAYMENT','DAYS_LAST_DUE','NAME_CONTRACT_TYPE']], on=['SK_ID_PREV'], how='left')
 tmp_1 = tmp.groupby(['SK_ID_CURR'])['delta'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp_1.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp_1, on=['SK_ID_CURR'], how='left')
 df['min_ratio_num_install'] = tmp_merge['des1']
df['max_ratio_num_install'] = tmp_merge['des2']
df['mean_ratio_num_install'] = tmp_merge['des3']
df['sum_ratio_num_install'] = tmp_merge['des4']<remove_duplicates>
for i in countries:
if i not in ['Denmark', 'France', 'Netherlands', 'United Kingdom']:
t1=mdfw[['Country_Region','Fatalities','Date']]
t1.index.name=''
t1=t1[t1['Country_Region']==i]
w1=t1[t1['Date']>=datetime.strptime('2020-03-19', '%Y-%m-%d')]
w1=w1[t1['Date']<=datetime.strptime('2020-03-28', '%Y-%m-%d')].set_index('Date')
 w1.index.name=''
cc1=ndf[ndf['Country_Region']==i].set_index('Date')
 cc=mdfw[mdfw['Country_Region']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('ConfirmedCases',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.Fatalities.diff()
cc.EverydayCC.iloc[0]=0.0
try:
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['Fatalities','EverydayCC'])
 final.drop('Fatalities',axis=1)
 final['Fatalities']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Country_Region']=i
w1=w1.append(final)
 w1['Province_State']=cc1['Province_State']
w1['ForecastId']=cc1['ForecastId']
p =p.append(w1)
 except:
print('Exception')
 else:
kkk=mdfw[mdfw['Country_Region']==i ]
lol=kkk[kkk.Province_State.isna() ]
t1=lol[['Country_Region','Fatalities','Date']]
t1.index.name=''
o1=pd.DataFrame(t1[t1['Country_Region']==i] ).set_index('Date')
 o1.index.name=''
t1=o1.loc['2020-03-19':'2020-03-28']
cc2=ndf[ndf['Country_Region']==i].set_index('Date')
 cc1=cc2[cc2['Province_State'].isna() ]
cc=lol[lol['Country_Region']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('ConfirmedCases',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.Fatalities.diff()
cc.EverydayCC.iloc[0]=0.0
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['Fatalities','EverydayCC'])
 final.drop('Fatalities',axis=1)
 final['Fatalities']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Country_Region']=i
t1=t1.append(final)
 t1['Country_Region']=cc1['Country_Region']
t1['ForecastId']=cc1['ForecastId']
p =p.append(t1)
tmp = install[['SK_ID_PREV','SK_ID_CURR','NUM_INSTALMENT_NUMBER','AMT_INSTALMENT']].drop_duplicates()<rename_columns>
dp2=p[1:]
tmp = tmp.groupby(['SK_ID_PREV','SK_ID_CURR'])['AMT_INSTALMENT'].sum().reset_index()
tmp.columns = ['SK_ID_PREV','SK_ID_CURR','need_to_pay']<rename_columns>
for i in pro:
t1=mdfw[['Province_State','ConfirmedCases','Date']]
t1.index.name=''
t1=t1[t1['Province_State']==i]
w1=t1[t1['Date']>=datetime.strptime('2020-03-19', '%Y-%m-%d')]
w1=w1[t1['Date']<=datetime.strptime('2020-03-28', '%Y-%m-%d')].set_index('Date')
 w1.index.name=''
cc1=ndf[ndf['Province_State']==i].set_index('Date')
 cc=mdfw[mdfw['Province_State']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('Fatalities',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.ConfirmedCases.diff()
cc.EverydayCC.iloc[0]=0.0
try:
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['ConfirmedCases','EverydayCC'])
 final.drop('ConfirmedCases',axis=1)
 final['ConfirmedCases']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Province_State']=i
w1=w1.append(final)
 w1['Country_Region']=cc1['Country_Region']
w1['ForecastId']=cc1['ForecastId']
p =p.append(w1)
 except:
print('Exception' )
tmp_1 = install.groupby(['SK_ID_PREV'])['AMT_PAYMENT'].sum().reset_index()
tmp_1.columns = ['SK_ID_PREV','paid']<merge>
dp3=p[1:]
tmp = tmp.merge(tmp_1, on=['SK_ID_PREV'], how='left' )<merge>
for i in pro:
t1=mdfw[['Province_State','Fatalities','Date']]
t1.index.name=''
t1=t1[t1['Province_State']==i]
w1=t1[t1['Date']>=datetime.strptime('2020-03-19', '%Y-%m-%d')]
w1=w1[t1['Date']<=datetime.strptime('2020-03-28', '%Y-%m-%d')].set_index('Date')
 w1.index.name=''
cc1=ndf[ndf['Province_State']==i].set_index('Date')
 cc=mdfw[mdfw['Province_State']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('ConfirmedCases',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.Fatalities.diff()
cc.EverydayCC.iloc[0]=0.0
try:
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['Fatalities','EverydayCC'])
 final.drop('Fatalities',axis=1)
 final['Fatalities']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Province_State']=i
w1=w1.append(final)
 w1['Country_Region']=cc1['Country_Region']
w1['ForecastId']=cc1['ForecastId']
p =p.append(w1)
 except:
print('Exception' )
payment_history = tmp
payment_history['ratio'] = payment_history['paid']/payment_history['need_to_pay']
payment_history['delta'] = payment_history['need_to_pay'] - payment_history['paid']
payment_history = payment_history.merge(previous[['SK_ID_PREV','AMT_ANNUITY','CNT_PAYMENT','NAME_CONTRACT_TYPE']], \
on = ['SK_ID_PREV'], how='left')
 payment_history['all_credit'] = payment_history['AMT_ANNUITY'] * payment_history['CNT_PAYMENT']
payment_history['ratio'] = payment_history['paid']/payment_history['all_credit']
payment_history['delta'] = payment_history['all_credit'] - payment_history['paid']
tmp = install.groupby(['SK_ID_PREV'])['NUM_INSTALMENT_VERSION'].mean().reset_index()
tmp.columns = ['SK_ID_PREV','mean_version']
payment_history = payment_history.merge(tmp, on=['SK_ID_PREV'], how='left' )<merge>
dp4=p[1:]
tmp = payment_history[payment_history['mean_version'] > 0].groupby(['SK_ID_CURR'])['ratio'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_ratio_paid_install'] = tmp_merge['des1']
df['max_ratio_paid_install'] = tmp_merge['des2']
df['mean_ratio_paid_install'] = tmp_merge['des3']
df['sum_ratio_paid_install'] = tmp_merge['des4']<merge>
dp1=dp1.reset_index()
dp2=dp2.reset_index()
dp5=pd.merge(dp1,dp2,on=['Country_Region','Province_State','ForecastId','index'])
 dp3=dp3.reset_index()
dp4=dp4.reset_index()
dp6=pd.merge(dp3,dp4,on=['Country_Region','Province_State','ForecastId','index'])
 dp7=dp5.append(dp6)
 dp8=dp7.drop(['index','Country_Region','Province_State'],axis=1 ).sort_values('ForecastId' )
<merge><EOS> 
dp8['ForecastId']=dp8['ForecastId'].apply(lambda x: int(x))
dp8['a']=dp8['ConfirmedCases']
dp8['ConfirmedCases']=dp8['ForecastId']
dp8['ForecastId']=dp8['a']
dp8.drop('a',axis=1,inplace=True)
 dp8.columns=['ForecastId','ConfirmedCases','Fatalities']
dp8.to_csv('submission.csv',index=False )