Kaggle competition: COVID19 Global Forecasting (Week 2)
previous['AMT_APPLICATION'].replace(0,np.nan, inplace = True)
 previous['AMT_CREDIT'].replace(0,np.nan, inplace = True)
 previous['AMT_GOODS_PRICE'].replace(0,np.nan,inplace =True)
 previous['RATE_DOWN_PAYMENT'].replace(0, np.nan, inplace = True)
 previous['AMT_ANNUITY'].replace(0, np.nan, inplace = True)
 previous['CNT_PAYMENT'].replace(0, np.nan, inplace = True )<merge>
data= pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv" )
for i in ['Revolving loans','Cash loans', 'Consumer loans']:
tmp = previous[(previous['NAME_CONTRACT_TYPE'] == i)&(previous['DAYS_LAST_DUE'] == 365243)]
for df in [train,test]:
tmp1 = tmp.groupby(['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index()
tmp1.columns = ['SK_ID_CURR','des']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['count_notfinish_' + "_".join(i.lower().split())] = tmp_merge['des'].fillna(0 )<groupby>
data['Province_State']=data['Province_State'].fillna('')
 test['Province_State']=test['Province_State'].fillna('' )
for i in ['Revolving loans','Cash loans', 'Consumer loans']:
tmp = previous[(previous['NAME_CONTRACT_TYPE'] == i)&(previous['DAYS_LAST_DUE'] == 365243)]
for df in [train,test]:
tmp1 = tmp.groupby(['SK_ID_CURR'])['AMT_CREDIT'].agg({"returns": [np.mean, np.sum]})\ 
.reset_index()
tmp1.columns = ['SK_ID_CURR','des1','des2']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['mean_notfinish_' + "_".join(i.lower().split())] = tmp_merge['des1'].fillna(0)
 df['sum_notfinish_' + "_".join(i.lower().split())] = tmp_merge['des2'].fillna(0 )<groupby>
import matplotlib.pyplot as plt
for i in ['Revolving loans','Cash loans', 'Consumer loans']:
tmp = previous[(previous['NAME_CONTRACT_TYPE'] == i)&(previous['DAYS_TERMINATION'] == 365243)]
for df in [train,test]:
tmp1 = tmp.groupby(['SK_ID_CURR'])['AMT_ANNUITY'].agg({"returns": [np.mean, np.sum]})\ 
.reset_index()
tmp1.columns = ['SK_ID_CURR','des1','des2']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['mean_annuity_notfinish_' + "_".join(i.lower().split())] = tmp_merge['des1'].fillna(0)
 df['sum_annuity_notfinish_' + "_".join(i.lower().split())] = tmp_merge['des2'].fillna(0 )<feature_engineering>
datetime_str = '01/22/20 00:00:00'
datetime_object = datetime.strptime(datetime_str, '%m/%d/%y %H:%M:%S')
 data['days']=pd.to_datetime(data['Date'] ).sub(datetime_object)/np.timedelta64(1, 'D')
 test['days']=pd.to_datetime(test['Date'] ).sub(datetime_object)/np.timedelta64(1, 'D' )
previous['SELLERPLACE_AREA'].replace(0, np.nan, inplace = True)
 previous['SELLERPLACE_AREA'].replace(-1, np.nan, inplace = True)
 previous['DAYS_TERMINATION'].replace(365243, np.nan, inplace = True)
 previous['DAYS_LAST_DUE'].replace(365243, np.nan, inplace = True)
 previous['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace = True)
 previous['sooner'] =(previous['DAYS_LAST_DUE_1ST_VERSION'] - previous['DAYS_LAST_DUE'])/(previous['DAYS_LAST_DUE_1ST_VERSION']-previous['DAYS_DECISION'])
 previous['duration'] = previous['DAYS_TERMINATION'] - previous['DAYS_DECISION']
previous['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace = True )<feature_engineering>
data.loc[(data['Province_State']=='')&(data['Country_Region']=='India'),:].sort_values(by="Date" )
train['DAYS_EMPLOYED'] = train['DAYS_EMPLOYED'].replace(365243, np.nan)
 test['DAYS_EMPLOYED'] = test['DAYS_EMPLOYED'].replace(365243, np.nan)
 tmp = train[train['DAYS_LAST_PHONE_CHANGE'] >= 0].index
train['DAYS_LAST_PHONE_CHANGE'].iloc[tmp] = np.nan
tmp = test[test['DAYS_LAST_PHONE_CHANGE'] >= 0].index
test['DAYS_LAST_PHONE_CHANGE'].iloc[tmp] = np.nan
for df in [train, test]:
df['ORGANIZATION_TYPE_v2'] = df['ORGANIZATION_TYPE']
for i in range(1,4):
df['ORGANIZATION_TYPE_v2'].replace('Business Entity Type ' + str(i), 'Business', inplace = True)
 for i in range(1,14):
df['ORGANIZATION_TYPE_v2'].replace('Industry: type ' + str(i), 'Industry', inplace = True)
 for i in range(1,8):
df['ORGANIZATION_TYPE_v2'].replace('Trade: type ' + str(i), 'Trade', inplace = True)
 for i in range(1,8):
df['ORGANIZATION_TYPE_v2'].replace('Transport: type ' + str(i), 'Transport', inplace = True)
 df['ORGANIZATION_TYPE_v2'].replace('Other','XNA', inplace = True )<groupby>
from statsmodels.tsa.arima_model import ARIMA
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['NAME_CONTRACT_TYPE'].isin(['Consumer loans','Cash loans', 'Revolving loans'])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_credit_master'] = tmp_merge['des1']
df['max_amt_credit_master'] = tmp_merge['des2']
df['mean_amt_credit_master'] = tmp_merge['des3']<groupby>
data['Date']=pd.to_datetime(data['Date'])
 test['Date']=pd.to_datetime(test['Date'] )
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans'])) ].groupby(['SK_ID_CURR'])['AMT_APPLICATION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_app'] = tmp_merge['des1']
df['max_amt_app'] = tmp_merge['des2']
df['mean_amt_app'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['NAME_CONTRACT_TYPE'].isin(['Consumer loans'])) ].groupby(['SK_ID_CURR'])['AMT_APPLICATION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_app_v1'] = tmp_merge['des1']
df['max_amt_app_v1'] = tmp_merge['des2']
df['mean_amt_app_v1'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['NAME_CONTRACT_TYPE'].isin(['Revolving loans'])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_card'] = tmp_merge['des1']
df['max_amt_card'] = tmp_merge['des2']
df['mean_amt_card'] = tmp_merge['des3']<groupby>
pd.DataFrame(data.loc[data['Country_Region']=='Afghanistan',['ConfirmedCases']] ).reset_index(drop=True )
tmp = previous[(previous['NAME_CONTRACT_STATUS'].isin(['Refused','Canceled'])) &(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans'])) ].groupby(['SK_ID_CURR'])['AMT_APPLICATION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_app_fail'] = tmp_merge['des1']
df['max_amt_app_fail'] = tmp_merge['des2']
df['mean_amt_app_fail'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_STATUS'].isin(['Refused','Canceled'])) &(previous['NAME_CONTRACT_TYPE'].isin(['Consumer loans'])) ].groupby(['SK_ID_CURR'])['AMT_APPLICATION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_app_v1_fail'] = tmp_merge['des1']
df['max_amt_app_v1_fail'] = tmp_merge['des2']
df['mean_amt_app_v1_fail'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_STATUS'].isin(['Refused','Canceled'])) &(previous['NAME_CONTRACT_TYPE'].isin(['Revolving loans'])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_card_fail'] = tmp_merge['des1']
df['max_amt_card_fail'] = tmp_merge['des2']
df['mean_amt_card_fail'] = tmp_merge['des3']<groupby>
data.isna().sum(axis=0 )
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans'])) ].groupby(['SK_ID_CURR'])['RATE_DOWN_PAYMENT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_goods'] = tmp_merge['des1']
df['max_amt_goods'] = tmp_merge['des2']
df['mean_amt_goods'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Consumer loans'])) ].groupby(['SK_ID_CURR'])['RATE_DOWN_PAYMENT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_goods_v1'] = tmp_merge['des1']
df['max_amt_goods_v1'] = tmp_merge['des2']
df['mean_amt_goods_v1'] = tmp_merge['des3'] 
<groupby>
data['ConfirmedCases'][data['Country_Region']==''][51:]
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].groupby(['SK_ID_CURR'])['AMT_ANNUITY']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_annuity'] = tmp_merge['des1']
df['max_amt_annuity'] = tmp_merge['des2']
df['mean_amt_annuity'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Revolving loans'])) ].groupby(['SK_ID_CURR'])['AMT_ANNUITY']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_card_annuity'] = tmp_merge['des1']
df['max_amt_card_annuity'] = tmp_merge['des2']
df['mean_amt_card_annuity'] = tmp_merge['des3']<groupby>
data['ConfirmedCases'][data['Country_Region']=='India'].value_counts()
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].groupby(['SK_ID_CURR'])['CNT_PAYMENT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_cntpay'] = tmp_merge['des1']
df['max_cntpay'] = tmp_merge['des2']
df['mean_cntpay'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Consumer loans'])) ].groupby(['SK_ID_CURR'])['CNT_PAYMENT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_cntpay_v1'] = tmp_merge['des1']
df['max_cntpay_v1'] = tmp_merge['des2']
df['mean_cntpay_v1'] = tmp_merge['des3'] 
<merge>
pd.DataFrame(data.loc[data['Country_Region']=='India',['ConfirmedCases']] )
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['AMT_APPLICATION'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_APPLICATION']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_app'] = tmp_merge['des']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['AMT_APPLICATION'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_CREDIT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_credit'] = tmp_merge['des']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['AMT_CREDIT'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Revolving loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_CREDIT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_card'] = tmp_merge['des']<merge>
datetime_str = '03/22/20 00:00:00'
datetime_object = datetime.strptime(datetime_str, '%m/%d/%y %H:%M:%S' )
tmp = previous[(previous['NAME_CONTRACT_STATUS'] != 'Approved')&(previous['AMT_APPLICATION'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_APPLICATION']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_app_fail'] = tmp_merge['des']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] != 'Approved')&(previous['AMT_APPLICATION'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_CREDIT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_credit_fail'] = tmp_merge['des']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] != 'Approved')&(previous['AMT_CREDIT'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Revolving loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_CREDIT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_card_fail'] = tmp_merge['des']<merge>
from datetime import timedelta
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['RATE_DOWN_PAYMENT'] > 0)].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','RATE_DOWN_PAYMENT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_ratedown'] = tmp_merge['des']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] != 'Approved')&(previous['RATE_DOWN_PAYMENT'] > 0)].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','RATE_DOWN_PAYMENT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_ratedown_fail'] = tmp_merge['des']<merge>
import math
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Consumer loans','Cash loans'])) &(previous['CNT_PAYMENT'] > 0)].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','CNT_PAYMENT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_cntpay'] = tmp_merge['des'] 
<merge>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
tmp = previous[previous['AMT_CREDIT'] > 0]
for i in ['Cash loans','Consumer loans','Revolving loans']:
for df in [train,test]:
tmp1 = tmp[tmp['NAME_CONTRACT_TYPE'] == i].groupby(['SK_ID_CURR'])['AMT_CREDIT'].count().reset_index()
tmp1.columns = ['SK_ID_CURR','des']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['count_' + "_".join(i.lower().split())] = tmp_merge['des']<merge>
def evaluate_arima_model(X,forecast_days, arima_order):
X=[x for x in X]
train_size = int(len(X)* 0.9)
 train, test1 = X[0:train_size], X[train_size:]
history=train
model = ARIMA(history, order=arima_order)
 model_fit = model.fit(disp=0)
 predictions = list()
predictions =model_fit.forecast(steps=len(test1)) [0]
model = ARIMA(X, order=arima_order)
 model_fit = model.fit(disp=0)
 if np.isnan(model_fit.forecast(steps=forecast_days)[0] ).sum() >0:
return float('inf')
 error = rmsle(test1, predictions)
 return error
tmp = previous[previous['AMT_CREDIT'].isnull() ]
for i in ['Cash loans','Consumer loans','Revolving loans']:
for df in [train,test]:
tmp1 = tmp[tmp['NAME_CONTRACT_TYPE'] == i].groupby(['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index()
tmp1.columns = ['SK_ID_CURR','des']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['count_null_' + "_".join(i.lower().split())] = tmp_merge['des']<groupby>
def evaluate_models(dataset,forcast_days, p_values, d_values, q_values):
best_score, best_cfg = float("inf"),(0,0,0)
 for p in p_values:
for d in d_values:
for q in q_values:
order =(p,d,q)
 try:
mse = evaluate_arima_model(dataset,forcast_days, order)
 if mse < best_score:
best_score, best_cfg = mse, order
except:
continue
print('Best ARIMA%s MSE=%.3f' %(best_cfg, best_score))
return best_cfg, best_score
tmp = previous[previous['AMT_CREDIT'] > 0].groupby(['SK_ID_CURR'])['DAYS_DECISION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_day_decision'] = tmp_merge['des1']
df['max_day_decision'] = tmp_merge['des2']
df['mean_day_decision'] = tmp_merge['des3']<groupby>
warnings.filterwarnings('ignore' )
tmp = previous[previous['NAME_CONTRACT_STATUS'] != 'Approved'].groupby(['SK_ID_CURR'])['DAYS_DECISION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_day_decision_fail'] = tmp_merge['des1']
df['max_day_decision_fail'] = tmp_merge['des2']
df['mean_day_decision_fail'] = tmp_merge['des3']<merge>
test['ConfirmedCases']=0
test['Fatalities']=0
tmp = previous[(~previous['NAME_CASH_LOAN_PURPOSE'].isin(['XAP','XNA'])) ]
for df in [train,test]:
tmp1 = tmp.groupby(['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index()
tmp1.columns = ['SK_ID_CURR','des']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['count_clear_reason'] = tmp_merge['des']<merge>
sliced_data=data.loc[(data['Province_State']=='')&(data['Country_Region']=='India'),:]
tmp = previous.groupby(['SK_ID_CURR'])['DAYS_TERMINATION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_day_termination'] = tmp_merge['des1']
df['max_day_termination'] = tmp_merge['des2']
df['mean_day_termination'] = tmp_merge['des3']<merge>
from pandas import read_csv
from pandas import datetime
from matplotlib import pyplot
from statsmodels.tsa.arima_model import ARIMA
from sklearn.metrics import mean_squared_error
from math import sqrt
from time import time
from sklearn.metrics import mean_squared_error
tmp = previous.groupby(['SK_ID_CURR'])['DAYS_LAST_DUE_1ST_VERSION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_day_lastdue'] = tmp_merge['des1']
df['max_day_lastdue'] = tmp_merge['des2']
df['mean_day_lastdue'] = tmp_merge['des3']<merge>
country='India'
state=''
sliced_data=data.loc[(data['Province_State']==state)&(data['Country_Region']==country),:]
test_sliced=test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:]
sliced_data=sliced_data.drop_duplicates()
sliced_data=sliced_data.reset_index(drop=True)
 sliced_data=sliced_data.sort_values(by='Date')
 if sliced_data.loc[sliced_data['ConfirmedCases']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['ConfirmedCases']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['ConfirmedCases'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(10),range(7),range(7))
preds=[]
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions=pd.DataFrame()
predictions['Date']=dates
predictions['ConfirmedCases']=preds
test_sliced=test_sliced.merge(sliced_data[['Date','ConfirmedCases']], on='Date',how='left')
 test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
test_sliced=test_sliced.merge(predictions, on='Date',how='left')
 test_sliced['ConfirmedCases_x'][test_sliced['ConfirmedCases_x'].isna() ]=test_sliced['ConfirmedCases_y'][test_sliced['ConfirmedCases_x'].isna() ]
test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_x']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
sliced_data_bck=sliced_data.copy()
if sliced_data.loc[sliced_data['Fatalities']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['Fatalities']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['Fatalities'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model=None
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions_f=pd.DataFrame()
predictions_f['Date']=dates
predictions_f['Fatalities']=preds
test_sliced=test_sliced.merge(sliced_data_bck[['Date','Fatalities']], on='Date',how='left')
 test_sliced['Fatalities']=test_sliced['Fatalities_y']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test_sliced=test_sliced.merge(predictions_f, on='Date',how='left')
 test_sliced['Fatalities_x'][test_sliced['Fatalities_x'].isna() ]=test_sliced['Fatalities_y'][test_sliced['Fatalities_x'].isna() ]
test_sliced['Fatalities']=test_sliced['Fatalities_x']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test=test.merge(test_sliced,on='ForecastId',how='left')
 test['ConfirmedCases_x'][test['ConfirmedCases_y'].notna() ]=test['ConfirmedCases_y'][test['ConfirmedCases_y'].notna() ]
test['Fatalities_x'][test['Fatalities_y'].notna() ]=test['Fatalities_y'][test['Fatalities_y'].notna() ]
new_cols=[]
for col in test.columns:
if col[-2:]=='_y':
del test[col]
elif col[-2:]=='_x':
new_cols.append(col[:-2])
 else:
new_cols.append(col)
 test.columns=new_cols
test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:].head()
plt.plot('Date', 'ConfirmedCases', data=sliced_data, color='blue', linewidth=2)
 plt.plot('Date','ConfirmedCases',data=test_sliced,color='orange',linewidth=2)
 plt.plot('Date', 'Fatalities', data=sliced_data, color='purple', linewidth=2)
 plt.plot('Date','Fatalities',data=test_sliced,color='red',linewidth=2)
 plt.show()
tmp = previous[~previous['DAYS_LAST_DUE_1ST_VERSION'].isnull() ].sort_values(by=['SK_ID_CURR','DAYS_LAST_DUE_1ST_VERSION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-2 ).reset_index()
tmp = tmp[['SK_ID_CURR','DAYS_LAST_DUE_1ST_VERSION']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['2nd_day_lastdue'] = tmp_merge['des']<merge>
test.loc[(test['Province_State']==state)&(test['Country_Region']==country),['Country_Region','Date','ConfirmedCases','Fatalities']]
tmp = previous.groupby(['SK_ID_CURR'])['sooner']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_sooner'] = tmp_merge['des1']
df['max_sooner'] = tmp_merge['des2']
df['mean_sooner'] = tmp_merge['des3']<merge>
test['ConfirmedCases']=0
test['Fatalities']=0
tmp = previous.groupby(['SK_ID_CURR'])['SELLERPLACE_AREA']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_seller'] = tmp_merge['des1']
df['max_seller'] = tmp_merge['des2']
df['mean_seller'] = tmp_merge['des3']<merge>
for country in countries:
for state in countries[country]:
print('Country : '+country,'State : '+state)
 sliced_data=data.loc[(data['Province_State']==state)&(data['Country_Region']==country),:]
test_sliced=test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:]
sliced_data=sliced_data.drop_duplicates()
sliced_data=sliced_data.reset_index(drop=True)
 sliced_data=sliced_data.sort_values(by='Date')
 if sliced_data.loc[sliced_data['ConfirmedCases']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['ConfirmedCases']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['ConfirmedCases'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions=pd.DataFrame()
predictions['Date']=dates
predictions['ConfirmedCases']=preds
test_sliced=test_sliced.merge(sliced_data[['Date','ConfirmedCases']], on='Date',how='left')
 test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
test_sliced=test_sliced.merge(predictions, on='Date',how='left')
 test_sliced['ConfirmedCases_x'][test_sliced['ConfirmedCases_x'].isna() ]=test_sliced['ConfirmedCases_y'][test_sliced['ConfirmedCases_x'].isna() ]
test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_x']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
sliced_data_bck=sliced_data.copy()
if sliced_data.loc[sliced_data['Fatalities']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['Fatalities']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['Fatalities'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model=None
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions_f=pd.DataFrame()
predictions_f['Date']=dates
predictions_f['Fatalities']=preds
test_sliced=test_sliced.merge(sliced_data_bck[['Date','Fatalities']], on='Date',how='left')
 test_sliced['Fatalities']=test_sliced['Fatalities_y']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test_sliced=test_sliced.merge(predictions_f, on='Date',how='left')
 test_sliced['Fatalities_x'][test_sliced['Fatalities_x'].isna() ]=test_sliced['Fatalities_y'][test_sliced['Fatalities_x'].isna() ]
test_sliced['Fatalities']=test_sliced['Fatalities_x']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test=test.merge(test_sliced,on='ForecastId',how='left')
 test['ConfirmedCases_x'][test['ConfirmedCases_y'].notna() ]=test['ConfirmedCases_y'][test['ConfirmedCases_y'].notna() ]
test['Fatalities_x'][test['Fatalities_y'].notna() ]=test['Fatalities_y'][test['Fatalities_y'].notna() ]
new_cols=[]
for col in test.columns:
if col[-2:]=='_y':
del test[col]
elif col[-2:]=='_x':
new_cols.append(col[:-2])
 else:
new_cols.append(col)
 test.columns=new_cols
print(test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:].head())
 plt.plot('Date', 'ConfirmedCases', data=sliced_data, color='blue', linewidth=2)
 plt.plot('Date','ConfirmedCases',data=test_sliced,color='orange',linewidth=2)
 plt.plot('Date', 'Fatalities', data=sliced_data, color='purple', linewidth=2)
 plt.plot('Date','Fatalities',data=test_sliced,color='red',linewidth=2)
 plt.show()
<feature_engineering><EOS> 
sumb=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv')
 output=pd.DataFrame()
output['ForecastId']=test['ForecastId'].astype(int)
 output['ConfirmedCases']=test['ConfirmedCases'].astype(int)
 output['Fatalities']=test['Fatalities'].astype(int)
 output.to_csv('submission.csv',index=False )