Kaggle competition: COVID19 Global Forecasting (Week 3)
df_test = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/test.csv.zip', sep=',')
 test = df_test.merge(df_features_stores, how='inner', on=['Store','Date','IsHoliday'])
 test.head()<data_type_conversions>
train = pd.read_csv('.. /input/covid19-global-forecasting-week-3/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-3/test.csv')
 submission = pd.read_csv('.. /input/covid19-global-forecasting-week-3/submission.csv')
 clean_train = pd.read_csv('.. /input/clean-data/clean_train.csv')
 clean_test = pd.read_csv('.. /input/clean-data/clean_test.csv' )
train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train['Week'] = train['Date'].dt.isocalendar().week
test['Week'] = test['Date'].dt.isocalendar().week
train['Year'] = train['Date'].dt.isocalendar().year
test['Year'] = test['Date'].dt.isocalendar().year<data_type_conversions>
tr_p = train[train['Province_State'].notnull() ]
tr_p['Country_Province'] = tr_p['Country_Region'] + '_' + tr_p['Province_State']
tr_np = train[train['Province_State'].isnull() ]
tr_np['Country_Province'] = tr_np['Country_Region']
train['Date'] = pd.to_datetime(train['Date'])
 train['Type'] = train['Type'].apply(lambda x: 3 if x == 'A' else(2 if x == 'B' else 1))
train['IsHoliday'] = train['IsHoliday'].apply(lambda x: 1 if x == True else 0)
 cols = train.columns.drop(['Date'])
 train[cols] = train[cols].apply(pd.to_numeric, errors='coerce' )<data_type_conversions>
train1 = pd.concat([tr_np,tr_p])
 train1.drop(['Province_State','Country_Region'], axis=1,inplace = True )
test['Date'] = pd.to_datetime(test['Date'])
 test['Type'] = test['Type'].apply(lambda x: 3 if x == 'A' else(2 if x == 'B' else 1))
test['IsHoliday'] = test['IsHoliday'].apply(lambda x: 1 if x == True else 0)
 cols = test.columns.drop(['Date'])
 test[cols] = test[cols].apply(pd.to_numeric, errors='coerce' )<remove_duplicates>
te_p = test[test['Province_State'].notnull() ]
te_p['Country_Province'] = te_p['Country_Region'] + '_' + te_p['Province_State']
te_np = test[test['Province_State'].isnull() ]
te_np['Country_Province'] = te_np['Country_Region']
holiday_train = train[['Date','Week','Year','IsHoliday']]
holiday_train = holiday_train.loc[holiday_train['IsHoliday']==True].drop_duplicates()
holiday_test = test[['Date','Week','Year','IsHoliday']]
holiday_test = holiday_test.loc[holiday_test['IsHoliday']==True].drop_duplicates()
holidays = pd.concat([holiday_train, holiday_test])
 holidays<categorify>
test1 = pd.concat([te_np,te_p])
 test1.drop(['Province_State','Country_Region'], axis=1,inplace = True )
def holiday_type(x):
if(x['IsHoliday']== 1)&(x['Week']==6):
return 1
elif(x['IsHoliday']== 1)&(x['Week']==36):
return 2
elif(x['IsHoliday']== 1)&(x['Week']==47):
return 3
elif(x['IsHoliday']== 1)&(x['Week']==52):
return 4
else:
return 0<feature_engineering>
train1['Date'] = pd.to_datetime(train1['Date'])
 test1['Date'] = pd.to_datetime(test1['Date'] )
train['IsHoliday'] = train.apply(holiday_type, axis=1)
 train['IsHoliday'].unique()<feature_engineering>
train1['Country_Province'] = train1['Country_Province'].astype('category')
 test1['Country_Province'] = test1['Country_Province'].astype('category' )
test['IsHoliday'] = test.apply(holiday_type, axis=1)
 test['IsHoliday'].unique()<categorify>
cl_tr1 = clean_train[['Country_Region','Province_State','Lat','Long','firstcase','density','medianage','urbanpop','hospibed','lung','avgtemp','avghumidity']]
train = train.replace('None', np.nan)
 train = train.replace('NaN', np.nan)
 train = train.replace('NaT', np.nan)
 train = train.replace('', np.nan)
 train_nulls =(train.isnull().sum(axis = 0)/len(train)) *100
train_nulls<categorify>
cl_tr1.drop_duplicates(subset=None, keep='first', inplace=True )
test = test.replace('None', np.nan)
 test = test.replace('NaN', np.nan)
 test = test.replace('NaT', np.nan)
 test = test.replace('', np.nan)
 test_nulls =(test.isnull().sum(axis = 0)/len(test)) *100
test_nulls<data_type_conversions>
cl_p = cl_tr1[cl_tr1['Province_State'].notnull() ]
cl_p['Country_Province'] = cl_p['Country_Region'] + '_' + cl_p['Province_State']
cl_np = cl_tr1[cl_tr1['Province_State'].isnull() ]
cl_np['Country_Province'] = cl_np['Country_Region']
train = train.fillna(0)
 test = test.fillna(0)
 train.isnull().sum()<groupby>
cl_tr = pd.concat([cl_p,cl_np])
 cl_tr.drop(['Country_Region','Province_State'], axis = 1, inplace = True )
weekly_sales = train.groupby(['Year','Week'] ).agg({'Weekly_Sales': ['mean', 'median']})
 weekly_sales2010 = train.loc[train['Year']==2010].groupby(['Week'] ).agg({'Weekly_Sales': ['mean', 'median']})
 weekly_sales2011 = train.loc[train['Year']==2011].groupby(['Week'] ).agg({'Weekly_Sales': ['mean', 'median']})
 weekly_sales2012 = train.loc[train['Year']==2012].groupby(['Week'] ).agg({'Weekly_Sales': ['mean', 'median']} )<create_dataframe>
train_cl = pd.merge(train1,cl_tr)
 test_cl = pd.merge(test1,cl_tr )
sample_weight = train['IsHoliday'].apply(lambda x: 1 if x==0 else 5)
 sample_weight_frame = pd.DataFrame(sample_weight, index=train.index )<compute_test_metric>
train_cl['Country_Province'] = train_cl['Country_Province'].astype('category')
 test_cl['Country_Province'] = test_cl['Country_Province'].astype('category' )
def WMAE(y_test, y_pred):
y_pred_df = pd.DataFrame(y_pred,index=y_test.index)
 weights_5 = sample_weight_frame.loc[(y_test.index)].loc[sample_weight_frame.IsHoliday==5].index
weights_1 = sample_weight_frame.loc[(y_test.index)].loc[sample_weight_frame.IsHoliday==1].index
sum_5 = np.sum(5*(abs(y_test.loc[weights_5].values-y_pred_df.loc[weights_5].values)))
 sum_1 = np.sum(abs(y_test.loc[weights_1].values-y_pred_df.loc[weights_1].values))
return np.round(( sum_5+sum_1)/(5*len(weights_5)+len(weights_1)) ,2)
 my_score = make_scorer(WMAE,greater_is_better=False )<drop_column>
train_cl['firstcase'] = pd.to_datetime(train_cl['firstcase'])
 test_cl['firstcase'] = pd.to_datetime(test_cl['firstcase'] )
train_all = train.drop(['Date'],axis=1)
 train_all<prepare_x_and_y>
train_cl['jan']="2020-01-01"
train_cl['jan'] = pd.to_datetime(train_cl['jan'])
 test_cl['jan']="2020-01-01"
test_cl['jan'] = pd.to_datetime(test_cl['jan'])
 train_cl['days_since_jan1'] = train_cl['Date']-train_cl['jan']
test_cl['days_since_jan1'] = test_cl['Date']-test_cl['jan']
y_train_all = train_all.loc[:, ['Weekly_Sales']]
x_train_all = train_all.drop(['Weekly_Sales'], axis=1 )<split>
for i in range(len(train_cl)) :
train_cl['days_since_jan1'][i]=train_cl['days_since_jan1'][i].days
for i in range(len(test_cl)) :
test_cl['days_since_jan1'][i]=test_cl['days_since_jan1'][i].days
x_train, x_test, y_train, y_test = train_test_split(x_train_all, y_train_all, test_size=0.2, random_state=0)
 print(x_train.shape)
 print(x_test.shape )<define_search_space>
train_cl['days_since_firstcase'] = train_cl['firstcase']-train_cl['Date']
test_cl['days_since_firstcase'] = test_cl['firstcase']-test_cl['Date']
clf = RandomForestRegressor(random_state=0)
 pca = PCA()
pipe = Pipeline(steps=[('clf', clf)])
 param_grid = [ {
'clf':[RandomForestRegressor() ],
'clf__n_estimators': [50,100,150],
'clf__max_depth': [10,20,30]
},
{
'clf': [ExtraTreesRegressor() ],
'clf__n_estimators': [50,100,150],
'clf__max_depth': [10,20,30]
},
{
'clf': [XGBRegressor() ],
'clf__learning_rate':[0.1,0.05],
'clf__min_samples_split':[5,7,9],
'clf__max_depth':[10,20,30]
}
]
rscv_all_tree = RandomizedSearchCV(pipe, param_grid, cv = 3, scoring = my_score, n_jobs=-1)
 model_all_tree = rscv_all_tree.fit(x_train, y_train )<find_best_params>
for i in range(len(train_cl)) :
train_cl['days_since_firstcase'][i]=train_cl['days_since_firstcase'][i].days
for i in range(len(test_cl)) :
test_cl['days_since_firstcase'][i]=test_cl['days_since_firstcase'][i].days
rscv_all_tree.best_estimator_<predict_on_test>
cols = ['days_since_jan1','days_since_firstcase']
for col in cols:
train_cl[col] = train_cl[col].astype('int64')
 test_cl[col] = test_cl[col].astype('int64' )
y_pred = rscv_all_tree.best_estimator_.predict(x_test)
 print('WMAE:', WMAE(y_test, y_pred))<drop_column>
train_clean_cases = train_cl[['Lat', 'Long','density', 'medianage', 'urbanpop',
'hospibed','lung', 'avgtemp', 'avghumidity','days_since_jan1', 'days_since_firstcase']]
test_clean_cases = test_cl[['Lat', 'Long','density', 'medianage', 'urbanpop',
'hospibed','lung', 'avgtemp', 'avghumidity','days_since_jan1', 'days_since_firstcase']]
train_relevant = train.drop(['Date','Temperature','Fuel_Price','MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5','CPI','Unemployment'],axis=1)
 train_relevant<prepare_x_and_y>
train_clean_fatal = train_cl[['Lat', 'Long','density', 'medianage', 'urbanpop',
'hospibed','lung', 'avgtemp', 'avghumidity','days_since_jan1', 'days_since_firstcase','ConfirmedCases']]
test_clean_fatal = test_cl[['Lat', 'Long','density', 'medianage', 'urbanpop',
'hospibed','lung', 'avgtemp', 'avghumidity','days_since_jan1', 'days_since_firstcase']]
y_relevant = train_relevant.loc[:, ['Weekly_Sales']]
x_relevant = train_relevant.drop(['Weekly_Sales'], axis=1 )<split>
train_y1 = train_cl['ConfirmedCases']
train_y2 = train_cl['Fatalities']
x_train_relevant, x_test_relevant, y_train_relevant, y_test_relevant = train_test_split(x_relevant, y_relevant, test_size=0.2, random_state=0)
 print(x_train_relevant.shape)
 print(x_test_relevant.shape )<set_options>
dt_1=DecisionTreeRegressor(max_depth=30,max_features=8,min_samples_split=2,min_samples_leaf=1)
 dt_2=DecisionTreeRegressor(max_depth=30,max_features=8,min_samples_split=2,min_samples_leaf=1)
 dt_1.fit(train_clean_cases,train_y1)
 dt_2.fit(train_clean_fatal,train_y2 )
clf = RandomForestRegressor(random_state=0)
 pca = PCA()
pipe = Pipeline(steps=[('clf', clf)])
 param_grid = [ {
'clf':[RandomForestRegressor() ],
'clf__n_estimators': [50,100,150],
'clf__max_depth': [10,20,30]
},
{
'clf': [ExtraTreesRegressor() ],
'clf__n_estimators': [50,100,150],
'clf__max_depth': [10,20,30]
},
{
'clf': [XGBRegressor() ],
'clf__learning_rate':[0.1,0.05],
'clf__min_samples_split':[5,7,9],
'clf__max_depth':[10,20,30]
}
]
rscv_relevant_tree = RandomizedSearchCV(pipe, param_grid, cv = 3, scoring = my_score, n_jobs=-1)
 model_relevant_tree = rscv_relevant_tree.fit(x_train_relevant, y_train_relevant )<find_best_params>
dt_train_cases_pred = dt_1.predict(train_clean_cases)
 dt_train_fatal_pred = dt_2.predict(train_clean_fatal )
rscv_relevant_tree.best_estimator_<predict_on_test>
dt_mse_train_cases = mean_squared_error(dt_train_cases_pred, train_y1)
 dt_rmse_train_cases = np.sqrt(dt_mse_train_cases)
 print("DT Regression MSE on train cases: %.4f" %dt_mse_train_cases)
 print('DT Regression RMSE on train cases: %.4f' % dt_rmse_train_cases )
y_pred= rscv_relevant_tree.best_estimator_.predict(x_test_relevant)
 print('WMAE:', WMAE(y_test_relevant, y_pred))<define_search_space>
dt_mse_train_fatalities = mean_squared_error(dt_train_fatal_pred, train_y2)
 dt_rmse_train_fatalities = np.sqrt(dt_mse_train_fatalities)
 print("DT Regression MSE on train fatalities: %.4f" %dt_mse_train_cases)
 print('DT Regression RMSE on train fatalities: %.4f' % dt_rmse_train_cases )
clf = RandomForestRegressor(random_state=0)
 pipe = Pipeline(steps=[('clf', clf)])
 param_grid_rf = [ {
'clf':[RandomForestRegressor() ],
'clf__n_estimators': [140,150,160],
'clf__max_depth': [25,30,35],
'clf__max_features': ['auto',5,6]
}
]
gscv_rf1 = GridSearchCV(pipe, param_grid_rf, cv = 3, scoring = my_score, n_jobs=-1)
 model_rf1 = gscv_rf1.fit(x_train_relevant, y_train_relevant )<find_best_params>
dt_test_cases_pred = dt_1.predict(test_clean_cases)
 dt_test_cases_pred = np.where(dt_test_cases_pred<0,0,np.rint(dt_test_cases_pred))
gscv_rf1.best_estimator_<predict_on_test>
test_clean_fatal['ConfirmedCases']= dt_test_cases_pred
y_pred_rf = gscv_rf1.best_estimator_.predict(x_test_relevant)
 print('WMAE:', WMAE(y_test_relevant, y_pred_rf))<drop_column>
dt_test_fatal_pred = dt_2.predict(test_clean_fatal )
date = test['Date']
test = test.drop(['Date'], axis=1 )<predict_on_test>
submission['ForecastId'] = test_cl['ForecastId']
submission['ConfirmedCases'] = dt_test_cases_pred
submission['Fatalities'] = dt_test_fatal_pred
test_relevant = test.drop(['Temperature','Fuel_Price','MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5','CPI', 'Unemployment'],axis=1)
 test_relevant = test_relevant.sort_values(['Store', 'Dept'], ascending=[True, True])
 y_pred_rf = gscv_rf1.best_estimator_.predict(test_relevant )<sort_values>
submission.to_csv('submission.csv',index=False )