Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<predict_on_test>
sns.set_style('darkgrid')
 %matplotlib inline
warnings.filterwarnings('ignore')
X_train['lgb_pred'] = lgb_model.predict(X_train ).clip(0,20)
 X_train['target'] = y_train
X_train['sq_err'] =(X_train['lgb_pred']-X_train['target'])**2
X_val['lgb_pred'] = lgb_model.predict(X_val ).clip(0,20)
 X_val['target'] = y_val
X_val['sq_err'] =(X_val['lgb_pred']-X_val['target'])**2
X_test['lgb_pred'] = lgb_model.predict(X_test ).clip(0,20 )<feature_engineering>
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv' )
data = X_train.groupby('date_block_num' ).agg({'lgb_pred':'mean','target':'mean','sq_err':'mean'} ).reset_index()
data['new_item_rmse'] = np.sqrt(X_train.query('item_age<=1' ).groupby('date_block_num' ).agg({'sq_err':'mean'} ).sq_err)
 data['old_item_rmse'] = np.sqrt(X_train.query('item_age>1' ).groupby('date_block_num' ).agg({'sq_err':'mean'} ).sq_err)
 data = data.append([
{'date_block_num':33,
'target':X_val.target.mean() ,
'lgb_pred':X_val.lgb_pred.mean() ,
'sq_err':np.sqrt(X_val.sq_err.mean()),
'old_item_rmse':np.sqrt(X_val.query('item_age>1' ).sq_err.mean()),
'new_item_rmse':np.sqrt(X_val.query('item_age<=1' ).sq_err.mean())},
{'date_block_num':34,
'target':0,
'lgb_pred':X_test.lgb_pred.mean() ,
'sq_err':0,
'old_item_rmse':0,
'new_item_rmse':0}
],
ignore_index=True 
)
 data['date'] = [x[:7] for x in pd.date_range(start='2013-03',end='2015-09',freq='MS' ).astype('str')]+['Validation','Test']<merge>
df_train['Date'] = pd.to_datetime(df_train['Date'], format = '%Y-%m-%d')
 df_test['Date'] = pd.to_datetime(df_test['Date'], format = '%Y-%m-%d' )
df = pd.read_pickle('.. /input/feature-engineering-lightgbm-exploring-performance/df_complete.pkl')
(
 df
[df['category_id'].isin(X_test.category_id.unique())] 
.query('item_cnt>0')
.groupby('category_id')
.agg({
'category_age':'max',
'shop_id':['nunique','unique'],
'item_cnt':'sum'
})
.join(categories['category_name'])
.join(
 X_test 
.groupby('category_id')
.agg({'item_id':'nunique'})
.rename(columns={'item_id':'test_set_items'})
)
.sort_values(( 'shop_id', 'nunique')) 
.head(20)
 )<merge>
def add_daily_measures(df):
df.loc[0,'Daily Cases'] = df.loc[0,'ConfirmedCases']
df.loc[0,'Daily Deaths'] = df.loc[0,'Fatalities']
for i in range(1,len(df_world)) :
df.loc[i,'Daily Cases'] = df.loc[i,'ConfirmedCases'] - df.loc[i-1,'ConfirmedCases']
df.loc[i,'Daily Deaths'] = df.loc[i,'Fatalities'] - df.loc[i-1,'Fatalities']
df.loc[0,'Daily Cases'] = 0
df.loc[0,'Daily Deaths'] = 0
return df
(
X_val
[X_val['category_id'].isin(X_test.category_id.unique())] 
.groupby('shop_id')
.agg({
'sq_err':'mean',
'target':'mean',
'lgb_pred':'mean'
})
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred'})
.groupby('shop_id')
.agg({'test_pred':'mean'})
)
.join(shops['shop_name'])
.sort_values('sq_err', ascending=False)
.head(20)
 )<merge>
df_world = df_train.copy()
df_world = df_world.groupby('Date',as_index=False)['ConfirmedCases','Fatalities'].sum()
df_world = add_daily_measures(df_world )
(
X_val
[X_val['category_id'].isin(X_test.category_id.unique())] 
.groupby('category_id')
.agg({
'sq_err':['sum','mean'],
'target':'mean',
'lgb_pred':['sum','mean'],
'item_id':'nunique'
})
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred','item_id':'test_items'})
.groupby('category_id')
.agg({
'test_pred':['sum','mean'],
'test_items':'nunique'
}),
on='category_id' 
)
.join(categories)
.sort_values(( 'sq_err', 'mean'), ascending=False)
.head(20)
 )<merge>
df_map = df_train.copy()
df_map['Date'] = df_map['Date'].astype(str)
 df_map = df_map.groupby(['Date','Country_Region'], as_index=False)['ConfirmedCases','Fatalities'].sum()
CATEGORY = 20 
(
 items[
items['item_id'].isin(X_val.item_id.unique())&
items['item_id'].isin(X_test.item_id.unique())
 ]
[['category_id','category_name','item_id','item_name']] 
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred'})
.groupby('item_id')
.agg({'test_pred':'mean'}),
on='item_id' 
)
.join(
 X_val 
.groupby('item_id')
.agg({
'lgb_pred':'mean',
'target':'mean',
'sq_err':'mean',
'same_name4catage_cnt_all_shops':'first',
'new_items_in_cat_all_shops_lag1to12':'first',
'item_cnt_all_shops_lag1':'first',
'category_cnt_all_shops_lag1':'first',
'item_cnt_sum_alltime_allshops':'first',
'prev_days_on_sale':'first'
})
.rename(columns={
'lgb_pred':'val_pred',
'target':'val_target',
}),
on='item_id' 
)
.query(f'category_id=={CATEGORY}')
.sort_values('sq_err',ascending=False)
.rename(columns={
'same_name4catage_cnt_all_shops':'name4mean',
'new_items_in_cat_all_shops_lag1to12':'new_in_cat_mean',
'item_cnt_all_shops_lag1':'item_cnt_lag1',
'category_cnt_all_shops_lag1':'cat_cnt_lag1',
'category_id':'cat',
'item_cnt_sum_alltime_allshops':'item_cnt_alltime'
})
.head(20)
 )<merge>
def create_features(df):
df['day'] = df['Date'].dt.day
df['month'] = df['Date'].dt.month
df['dayofweek'] = df['Date'].dt.dayofweek
df['dayofyear'] = df['Date'].dt.dayofyear
df['quarter'] = df['Date'].dt.quarter
df['weekofyear'] = df['Date'].dt.weekofyear
return df
CATEGORY = 20 
(
 items[items['item_id'].isin(X_test.item_id.unique())]
[['category_id','category_name','item_id','item_name']] 
.join(
 X_test 
.groupby('item_id')
.agg({
'lgb_pred':'mean',
'same_name4catage_cnt_all_shops':'first',
'new_items_in_cat_all_shops_lag1to12':'first',
'item_cnt_all_shops_lag1':'first',
'category_cnt_all_shops_lag1':'first',
'item_cnt_sum_alltime_allshops':'first',
'prev_days_on_sale':'first'
})
.rename(columns={'lgb_pred':'test_pred'}),
on='item_id' 
)
.query(f'category_id=={CATEGORY}')
.sort_values('test_pred',ascending=False)
.rename(columns={
'same_name4catage_cnt_all_shops':'name4mean',
'new_items_in_cat_all_shops_lag1to12':'new_in_cat_mean',
'item_cnt_all_shops_lag1':'item_cnt_lag1',
'category_cnt_all_shops_lag1':'cat_cnt_lag1',
'item_category_id':'cat',
'item_cnt_sum_alltime_allshops':'item_cnt_alltime'
})
.head(20)
 )<set_options>
def train_dev_split(df, days):
date = df['Date'].max() - dt.timedelta(days=days)
 return df[df['Date'] <= date], df[df['Date'] > date]
%matplotlib inline
sns.set()
print(os.listdir(".. /input"))
warnings.filterwarnings("ignore" )<load_from_csv>
df_train = create_features(df_train )
train=pd.read_csv('.. /input/act_train.csv',parse_dates=['date'])
 test=pd.read_csv('.. /input/act_test.csv',parse_dates=['date'])
 people=pd.read_csv('.. /input/people.csv',parse_dates=['date'] )<merge>
df_train, df_dev = train_dev_split(df_train,0 )
train=pd.merge(train,people,on='people_id')
 test=pd.merge(test,people,on='people_id' )<count_unique_values>
columns = ['day','month','dayofweek','dayofyear','quarter','weekofyear','Province_State', 'Country_Region','ConfirmedCases','Fatalities']
df_train = df_train[columns]
df_dev = df_dev[columns]
train.char_38.nunique()<count_values>
df_train.Province_State.fillna('NaN', inplace=True)
 df_test.Province_State.fillna('NaN', inplace=True)
 df_test = create_features(df_test)
 columns = ['day','month','dayofweek','dayofyear','quarter','weekofyear']
len(set(test['group_1'])-set(train['group_1']))<data_type_conversions>
submission = []
for country in df_train.Country_Region.unique() :
df_train1 = df_train[df_train["Country_Region"]==country]
for state in df_train1.Province_State.unique() :
df_train2 = df_train1[df_train1["Province_State"]==state]
df_train3 = df_train2.drop(['Country_Region','Province_State'], axis=1)
 train = df_train3.values
X_train, y_train = train[:,:-2], train[:,-2:]
model1 = XGBRegressor(random_state=1,n_estimators=1000)
 model1.fit(X_train, y_train[:,0])
 model2 = XGBRegressor(random_state=1,n_estimators=1000)
 model2.fit(X_train, y_train[:,1])
 df_test1 = df_test[(df_test["Country_Region"]==country)&(df_test["Province_State"] == state)]
ForecastId = df_test1.ForecastId.values
df_test2 = df_test1[columns]
y_pred1 = model1.predict(df_test2.values ).astype(int)
 y_pred2 = model2.predict(df_test2.values ).astype(int)
 for i in range(len(y_pred1)) :
d = {'ForecastId':ForecastId[i], 'ConfirmedCases':y_pred1[i], 'Fatalities':y_pred2[i]}
submission.append(d )
for i in train.columns:
if np.dtype(train[i])==np.dtype('object'):
train[i].fillna('type 0',inplace=True )<count_missing_values>
df_submit = pd.DataFrame(submission )
checkMissing(train)
 checkMissing(test )<feature_engineering>
df_submit.to_csv(r'submission.csv', index=False )