Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<merge>
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )
tmp = install.groupby(['SK_ID_PREV'])['AMT_PAYMENT'].max().reset_index()
tmp.columns = ['SK_ID_PREV','max_install']
payment_history = payment_history.merge(tmp, on=['SK_ID_PREV'], how='left' )<groupby>
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )
payment_history['tmp'] = payment_history['max_install']/payment_history['AMT_ANNUITY']
tmp = payment_history[payment_history['mean_version'] > 0].groupby(['SK_ID_CURR'])['tmp']<merge>
train[['ConfirmedCases', 'Fatalities']] = train.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases', 'Fatalities']].transform('cummax')
tmp = install.groupby(['SK_ID_PREV'])['NUM_INSTALMENT_NUMBER'].max().reset_index()
tmp.columns = ['SK_ID_PREV','max_num_install']
payment_history = payment_history.merge(tmp, on=['SK_ID_PREV'], how='left' )<merge>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
adjusted_X_pred = X_pred[feature_use].values.reshape(-1, 1)
 model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(adjusted_X_train,adjusted_y_train_confirmed)
 y_hat_confirmed = model.predict(adjusted_X_pred)
 model.fit(adjusted_X_train,adjusted_y_train_fatalities)
 y_hat_fatalities = model.predict(adjusted_X_pred)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_1 = df_val.copy()
tmp = install[install['AMT_INSTALMENT'] > install['AMT_PAYMENT']]
tmp = tmp.groupby(['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index()
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['count_small_payment'] = tmp_merge['des'].fillna(0)
 for i in [0, 5, 10, 15, 20, 25, 30, 40, 50, 60]:
print(i)
 tmp = install[(install['DAYS_ENTRY_PAYMENT'] - install['DAYS_INSTALMENT'])> i]
tmp = tmp.groupby(['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index()
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['count_late_payment_' + str(i)] = tmp_merge['des'].fillna(0)
<merge>
RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values )
install['tmp'] = install['AMT_PAYMENT']/install['AMT_INSTALMENT']
for i in range(10):
print(i)
 tmp = install[(install['tmp'] > i/10)&(install['tmp'] <(( i+1)/10)) ]
tmp = tmp.groupby(['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index()
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['count_ratio_payment_' + str(i)] = tmp_merge['des'].fillna(0)
<merge>
RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values )
tmp = install.groupby(['SK_ID_PREV','NUM_INSTALMENT_NUMBER'])['DAYS_INSTALMENT'].count().reset_index()
tmp = tmp[tmp['DAYS_INSTALMENT'] > 1]
tmp.columns = ['SK_ID_PREV','NUM_INSTALMENT_NUMBER','count_dup']
install = install.merge(tmp, on = ['SK_ID_PREV','NUM_INSTALMENT_NUMBER'], how='left')
 dup_install = install[install['count_dup'] > 1]
dup_install.reset_index(drop=True, inplace = True)
<groupby>
country = "Vietnam"
df_val = df_val_1
df_val[df_val['Country_Region'] == country].groupby(['Date','Country_Region'] ).sum().reset_index()
tmp = install[(install['AMT_PAYMENT'] < install['AMT_INSTALMENT'])&(install['DAYS_ENTRY_PAYMENT'] < install['DAYS_INSTALMENT'])]
tmp['ratio'] = tmp['AMT_PAYMENT']/tmp['AMT_INSTALMENT']
tmp = dup_install.groupby(['SK_ID_CURR'])['AMT_PAYMENT'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp_1.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp_1, on=['SK_ID_CURR'], how='left')
 df['min_special_install'] = tmp_merge['des1']
df['max_special_install'] = tmp_merge['des2']
df['mean_special_install'] = tmp_merge['des3']
df['sum_special_install'] = tmp_merge['des4']<sort_values>
animator.save('confirm_animation.gif', writer='imagemagick', fps=2)
 display(Image(url='confirm_animation.gif'))
dup_install.sort_values(by=['SK_ID_PREV','NUM_INSTALMENT_NUMBER','DAYS_ENTRY_PAYMENT'] )<load_from_csv>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = ExponentialSmoothing(adjusted_y_train_confirmed, trend = 'additive' ).fit()
y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = ExponentialSmoothing(adjusted_y_train_fatalities, trend = 'additive' ).fit()
y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_2 = df_val.copy()
credit = pd.read_csv(".. /input/credit_card_balance.csv" )<merge>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()
credit['tmp'] = credit['AMT_BALANCE']/credit['AMT_CREDIT_LIMIT_ACTUAL']
tmp = credit.groupby(["SK_ID_CURR","SK_ID_PREV"])['tmp'].max().reset_index()
tmp = tmp.groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_max_ratio_balance_limit_credit'] = tmp_merge['des1']
df['max_max_ratio_balance_limit_credit'] = tmp_merge['des2']
df['mean_max_ratio_balance_limit_credit'] = tmp_merge['des3']
df['sum_max_ratio_balance_limit_credit'] = tmp_merge['des4'] 
<merge>
method_list = ['Poly Bayesian Ridge','Exponential Smoothing','SARIMA']
method_val = [df_val_1,df_val_2,df_val_3]
for i in range(0,3):
df_val = method_val[i]
method_score = [method_list[i]] + [RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values)] + [RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values)]
print(method_score )
credit['tmp'] = credit['AMT_BALANCE']/credit['AMT_CREDIT_LIMIT_ACTUAL']
tmp = credit.groupby(["SK_ID_CURR","SK_ID_PREV"])['tmp'].min().reset_index()
tmp = credit.groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_min_ratio_balance_limit_credit'] = tmp_merge['des1']
df['max_min_ratio_balance_limit_credit'] = tmp_merge['des2']
df['mean_min_ratio_balance_limit_credit'] = tmp_merge['des3']
df['sum_min_ratio_balance_limit_credit'] = tmp_merge['des4'] 
<categorify>
df_val = df_val_3
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission = submission.round({'ConfirmedCases': 0, 'Fatalities': 0})
 submission.to_csv('submission.csv', index=False)
 submission