Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<merge>
import pandas as pd
import numpy as np
from xgboost import XGBRegressor
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
tmp = bureau[(~bureau['DAYS_CREDIT_ENDDATE'].isnull())&(( bureau['CREDIT_TYPE'] == "Consumer credit")) ].sort_values(by=['SK_ID_CURR','DAYS_CREDIT_ENDDATE'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-2 ).reset_index()
tmp = tmp[['SK_ID_CURR','DAYS_ENDDATE_FACT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_endatefact_bureau'] = tmp_merge['des']<merge>
df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv' )
tmp = bureau[(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['DAYS_ENDDATE_FACT'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_endatefact_bureau'] = tmp_merge['des1']
df['max_endatefact_bureau'] = tmp_merge['des2']
df['mean_endatefact_bureau'] = tmp_merge['des3']
df['sum_endatefact_bureau'] = tmp_merge['des4']<merge>
def fill_States(s):
if pd.isna(s['Province_State']):
if pd.isna(s['Country_Region']):
return "NA"
else:
return s['Country_Region']
else:
return s['Province_State']
bureau['tmp'] = bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_ENDDATE_FACT']
tmp = bureau.groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_deltaendate_bureau'] = tmp_merge['des1']
df['max_deltaendate_bureau'] = tmp_merge['des2']
df['mean_deltaendate_bureau'] = tmp_merge['des3']
df['sum_deltaendate_bureau'] = tmp_merge['des4']<merge>
df['Province_State'] = df.apply(fill_States,axis=1 )
bureau['tmp'] =(bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_CREDIT'])
 tmp = bureau[(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_duration_bureau'] = tmp_merge['des1']
df['max_duration_bureau'] = tmp_merge['des2']
df['mean_duration_bureau'] = tmp_merge['des3']
df['sum_duration_bureau'] = tmp_merge['des4']
bureau['tmp'] =(bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_CREDIT'])
 tmp = bureau[(bureau['CREDIT_TYPE'] != "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_duration_bureau_v1'] = tmp_merge['des1']
df['max_duration_bureau_v1'] = tmp_merge['des2']
df['mean_duration_bureau_v1'] = tmp_merge['des3']
df['sum_duration_bureau_v1'] = tmp_merge['des4']<merge>
df_test['Province_State'] = df_test.apply(fill_States,axis=1 )
bureau['tmp'] =(bureau['DAYS_ENDDATE_FACT'] - bureau['DAYS_CREDIT'])
 tmp = bureau[(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_durationfact_bureau'] = tmp_merge['des1']
df['max_durationfact_bureau'] = tmp_merge['des2']
df['mean_durationfact_bureau'] = tmp_merge['des3']
df['sum_durationfact_bureau'] = tmp_merge['des4']<groupby>
df['Date'] = pd.to_datetime(df['Date'])
 df_test['Date'] = pd.to_datetime(df_test['Date'] )
bureau['tmp'] =(bureau['DAYS_ENDDATE_FACT'] - bureau['DAYS_CREDIT_ENDDATE'])/(bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE'])
 tmp = bureau[(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_sooner_bureau'] = tmp_merge['des1']
df['max_sooner_bureau'] = tmp_merge['des2']
df['mean_sooner_bureau'] = tmp_merge['des3']
df['sum_sooner_bureau'] = tmp_merge['des4']<groupby>
df['Date'] = df['Date'].dt.month * 100 + df['Date'].dt.day
df_test['Date'] = df_test['Date'].dt.month * 100 + df_test['Date'].dt.day
bureau['tmp'] =(bureau['DAYS_ENDDATE_FACT'] - bureau['DAYS_CREDIT_ENDDATE'])/(bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE'])
 tmp = bureau[(~bureau['CREDIT_TYPE'].isin(['Credit card','Consumer credit'])) ].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_sooner_bureau_v1'] = tmp_merge['des1']
df['max_sooner_bureau_v1'] = tmp_merge['des2']
df['mean_sooner_bureau_v1'] = tmp_merge['des3']
df['sum_sooner_bureau_v1'] = tmp_merge['des4']<merge>
preds = pd.DataFrame()
bureau['tmp'] =(bureau['AMT_CREDIT_SUM'])/(bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE'])
 tmp = bureau[bureau['CREDIT_TYPE'] == "Credit card"].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_annuity_bureau'] = tmp_merge['des1']
df['max_annuity_bureau'] = tmp_merge['des2']
df['mean_annuity_bureau'] = tmp_merge['des3']
df['sum_annuity_bureau'] = tmp_merge['des4']<groupby>
preds['ForecastId'] = df_test.ForecastId
bureau['tmp'] =(bureau['AMT_CREDIT_SUM'])/(bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE'])
 tmp = bureau[(bureau['CREDIT_ACTIVE'] == "Active")&(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_annuity_bureau_v1'] = tmp_merge['des1']
df['max_annuity_bureau_v1'] = tmp_merge['des2']
df['mean_annuity_bureau_v1'] = tmp_merge['des3']
df['sum_annuity_bureau_v1'] = tmp_merge['des4']
tmp = bureau[(bureau['CREDIT_ACTIVE'] == "Active")&(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_annuity_bureau_v2'] = tmp_merge['des1']
df['max_annuity_bureau_v2'] = tmp_merge['des2']
df['mean_annuity_bureau_v2'] = tmp_merge['des3']
df['sum_annuity_bureau_v2'] = tmp_merge['des4']<merge>
le = LabelEncoder()
le2 = LabelEncoder()
bureau['tmp'] =(bureau['AMT_CREDIT_SUM'])/(bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE'])
 tmp = bureau[~bureau['CREDIT_TYPE'].isin(["Credit card","Consumer credit"])].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_annuity_bureau_v3'] = tmp_merge['des1']
df['max_annuity_bureau_v3'] = tmp_merge['des2']
df['mean_annuity_bureau_v3'] = tmp_merge['des3']
df['sum_annuity_bureau_v3'] = tmp_merge['des4']<categorify>
for country in df['Country_Region'].unique() :
for state in df[df['Country_Region']==country].loc[:,'Province_State'].unique() :
train = df[ df['Province_State'] == state ].copy()
X = train.drop(['Id','ConfirmedCases','Fatalities'],axis=1)
 X['Province_State'] = le.fit_transform(X['Province_State'])
 X['Country_Region'] = le2.fit_transform(X['Country_Region'])
 y1 = train['ConfirmedCases']
y2 = train['Fatalities']
test = df_test[ df_test['Province_State'] == state ].copy()
X_test = test.drop(['ForecastId'],axis=1)
 X_test['Province_State'] = le.transform(X_test['Province_State'])
 X_test['Country_Region'] = le2.transform(X_test['Country_Region'])
 model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X,y1)
 preds.at[test['ForecastId'] - 1,'ConfirmedCases'] = model1.predict(X_test)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X,y2)
 preds.at[test['ForecastId'] - 1,'Fatalities'] = model2.predict(X_test)
bureau['AMT_CREDIT_SUM_DEBT_v1'] = bureau['AMT_CREDIT_SUM_DEBT'].replace(0, np.nan)
 tmp = bureau[(bureau['CREDIT_ACTIVE'] == "Active")&(bureau['CREDIT_TYPE'].isin(["Credit card"])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM_DEBT_v1'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_debt_bureau'] = tmp_merge['des1']
df['max_debt_bureau'] = tmp_merge['des2']
df['mean_debt_bureau'] = tmp_merge['des3']
df['sum_debt_bureau'] = tmp_merge['des4']
tmp = bureau[(bureau['CREDIT_ACTIVE'] == "Active")&(bureau['CREDIT_TYPE'].isin(["Consumer credit"])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM_DEBT_v1'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_debt_bureau_v1'] = tmp_merge['des1']
df['max_debt_bureau_v1'] = tmp_merge['des2']
df['mean_debt_bureau_v1'] = tmp_merge['des3']
df['sum_debt_bureau_v1'] = tmp_merge['des4']<groupby>
preds.to_csv('submission.csv',index=False )