Kaggle competition: COVID19 Global Forecasting (Week 4)
def agg_cnt_col(df, merging_cols, new_col,aggregation):
temp = df.groupby(merging_cols ).agg(aggregation ).reset_index()
temp.columns = merging_cols + [new_col]
df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = agg_cnt_col(df, ['date_block_num','item_id'],'item_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'item_cnt_all_shops_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'category_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'category_cnt_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','category_id'],'category_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id'],'category_cnt_all_shops_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','group_id','shop_id'],'group_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','group_id'],'group_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','shop_id'],'shop_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','shop_city'],'city_cnt',{'item_cnt':'mean'} )<merge>
df=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv")
 test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv" )
def new_item_sales(df, merging_cols, new_col):
temp =(
 df 
.query('item_age==0')
.groupby(merging_cols)['item_cnt'] 
.mean() 
.reset_index() 
.rename(columns={'item_cnt': new_col})
)
 df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = new_item_sales(df, ['date_block_num','category_id','shop_id'], 'new_items_in_cat')
 df = new_item_sales(df, ['date_block_num','category_id'], 'new_items_in_cat_all_shops' )<merge>
df["Province_State"].fillna("state", inplace = True)
 df["Country_Region"] = [country_name.replace("'","")for country_name in df["Country_Region"]]
print(df.shape,"
",df.head() )
def agg_price_col(df, merging_cols, new_col):
temp = df.groupby(merging_cols ).agg({'revenue':'sum','item_cnt_unclipped':'sum'} ).reset_index()
temp[new_col] = temp['revenue']/temp['item_cnt_unclipped']
temp = temp[merging_cols + [new_col]]
df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = agg_price_col(df,['date_block_num','item_id'],'item_price')
 df = agg_price_col(df,['date_block_num','category_id'],'category_price')
 df = agg_price_col(df,['date_block_num'],'block_price' )<categorify>
X_y=shuffle(new_data)
 y_cases=X_y['target_infection']
y_fatal=X_y['target_fatal']
X=X_y.drop(['target_infection','target_fatal'],axis=1)
 X_train_cases, X_test_cases, y_train_cases, y_test_cases = train_test_split(X, y_cases, test_size=0.33)
 X_train_fatal, X_test_fatal, y_train_fatal, y_test_fatal = train_test_split(X, y_fatal, test_size=0.33)
 print("Shape of infection train dataset:",(X_train_cases.shape,y_train_cases.shape))
print("Shape of infection test dataset:",(X_test_cases.shape,y_test_cases.shape))
print("Shape of fatal train dataset:",(X_train_fatal.shape,y_train_fatal.shape))
print("Shape of fatal test dataset:",(X_test_fatal.shape,y_test_fatal.shape))
df = downcast(df )<merge>
reg_case=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=2200)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf = RandomizedSearchCV(reg_case, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search=clf.fit(X_train_cases, y_train_cases)
 results = pd.DataFrame.from_dict(clf.cv_results_ )
def lag_feature(df, lag, col, merge_cols):
temp = df[merge_cols + [col]]
temp = temp.groupby(merge_cols ).agg({f'{col}':'first'} ).reset_index()
temp.columns = merge_cols + [f'{col}_lag{lag}']
temp['date_block_num'] += lag
df = pd.merge(df, temp, on=merge_cols, how='left')
 df[f'{col}_lag{lag}'] = df[f'{col}_lag{lag}'].fillna(0 ).astype('float32')
 return df<define_variables>
best_alpha=1000
best_itr=2400
final_reg_case=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_case.fit(X_train_cases,y_train_cases )
lag12_cols = {
'item_cnt':['date_block_num', 'shop_id', 'item_id'],
'item_cnt_all_shops':['date_block_num', 'item_id'],
'category_cnt':['date_block_num', 'shop_id', 'category_id'],
'category_cnt_all_shops':['date_block_num', 'category_id'],
'group_cnt':['date_block_num', 'shop_id', 'group_id'],
'group_cnt_all_shops':['date_block_num', 'group_id'],
'shop_cnt':['date_block_num', 'shop_id'],
'city_cnt':['date_block_num', 'shop_city'],
'new_items_in_cat':['date_block_num', 'shop_id', 'category_id'],
'new_items_in_cat_all_shops':['date_block_num', 'category_id']
}
for col,merge_cols in lag12_cols.items() :
df[f'{col}_lag1to12'] = 0
for i in range(1,13):
df = lag_feature(df, i, col, merge_cols)
 df[f'{col}_lag1to12'] += df[f'{col}_lag{i}']
if i > 2:
del df[f'{col}_lag{i}']
if col == 'item_cnt':
del df[f'{col}_lag1']
del df[f'{col}_lag2']
else:
del df[col]<drop_column>
pred=final_reg_case.predict(X_test_cases)
 print("The RMSE value",(mean_squared_error(y_test_cases,pred)) **0.5 )
lag2_cols = {
'item_cnt_unclipped':['date_block_num', 'shop_id', 'item_id'],
'item_cnt_all_shops_median':['date_block_num', 'item_id'],
'category_cnt_median':['date_block_num', 'shop_id', 'category_id'],
'category_cnt_all_shops_median':['date_block_num', 'category_id']
}
for col in lag2_cols:
df = lag_feature(df, 1, col, merge_cols)
 df = lag_feature(df, 2, col, merge_cols)
 if col!='item_cnt_unclipped':
del df[col]<feature_engineering>
reg_fatal=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=3500)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf = RandomizedSearchCV(reg_fatal, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search=clf.fit(X_train_fatal, y_train_fatal)
 results = pd.DataFrame.from_dict(clf.cv_results_ )
df['item_cnt_diff'] = df['item_cnt_unclipped_lag1']/df['item_cnt_lag1to12']
df['item_cnt_all_shops_diff'] = df['item_cnt_all_shops_lag1']/df['item_cnt_all_shops_lag1to12']
df['category_cnt_diff'] = df['category_cnt_lag1']/df['category_cnt_lag1to12']
df['category_cnt_all_shops_diff'] = df['category_cnt_all_shops_lag1']/df['category_cnt_all_shops_lag1to12']<drop_column>
featured=pd.DataFrame(data)
 X_y_f=shuffle(featured)
 y_cases_f=X_y_f['target_infection']
y_fatal_f=X_y_f['target_fatal']
X_f=X_y_f.drop(['target_infection','target_fatal'],axis=1)
 X_train_cases_f, X_test_cases_f, y_train_cases_f, y_test_cases_f = train_test_split(X_f, y_cases_f, test_size=0.33)
 X_train_fatal_f, X_test_fatal_f, y_train_fatal_f, y_test_fatal_f = train_test_split(X_f, y_fatal_f, test_size=0.33)
 print("Shape of featurized infection train dataset:",(X_train_cases_f.shape,y_train_cases_f.shape))
print("Shape of featurized infection test dataset:",(X_test_cases_f.shape,y_test_cases_f.shape))
print("Shape of featurized fatal train dataset:",(X_train_fatal_f.shape,y_train_fatal_f.shape))
print("Shape of featurized fatal test dataset:",(X_test_fatal_f.shape,y_test_fatal_f.shape))
df = lag_feature(df, 1, 'category_price',['date_block_num', 'category_id'])
 df = lag_feature(df, 1, 'block_price',['date_block_num'])
 del df['category_price'], df['block_price']<feature_engineering>
reg_case_f=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=2200)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf_f= RandomizedSearchCV(reg_case_f, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search_f=clf_f.fit(X_train_cases_f, y_train_cases_f)
 results_f = pd.DataFrame.from_dict(clf_f.cv_results_ )
df.loc[(df['item_age']>0)&(df['item_cnt_lag1to12'].isna()), 'item_cnt_lag1to12'] = 0
df.loc[(df['category_age']>0)&(df['category_cnt_lag1to12'].isna()), 'category_cnt_lag1to12'] = 0
df.loc[(df['group_age']>0)&(df['group_cnt_lag1to12'].isna()), 'group_cnt_lag1to12'] = 0<feature_engineering>
best_alpha=10000
best_itr=4200
final_reg_case_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_case_f.fit(X_train_cases_f,y_train_cases_f )
df['item_cnt_lag1to12'] /= [min(idx)for idx in zip(df['item_age'],df['shop_age'],[12]*len(df)) ]
df['item_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['item_age'],[12]*len(df)) ]
df['category_cnt_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],df['shop_age'],[12]*len(df)) ]
df['category_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],[12]*len(df)) ]
df['group_cnt_lag1to12'] /= [min(idx)for idx in zip(df['group_age'],df['shop_age'],[12]*len(df)) ]
df['group_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['group_age'],[12]*len(df)) ]
df['city_cnt_lag1to12'] /= [min(idx)for idx in zip(df['date_block_num'],[12]*len(df)) ]
df['shop_cnt_lag1to12'] /= [min(idx)for idx in zip(df['shop_age'],[12]*len(df)) ]
df['new_items_in_cat_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],df['shop_age'],[12]*len(df)) ]
df['new_items_in_cat_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],[12]*len(df)) ]<categorify>
pred_f=final_reg_case_f.predict(X_test_cases_f)
 print("RMSE is:",(mean_squared_error(y_test_cases_f,pred_f)) **0.5 )
df = downcast(df )<merge>
reg_fatal_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=2200)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf_f= RandomizedSearchCV(reg_fatal_f, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search_f=clf_f.fit(X_train_fatal_f, y_train_fatal_f)
 results_f = pd.DataFrame.from_dict(clf_f.cv_results_ )
def past_information(df, merging_cols, new_col, aggregation):
temp = []
for i in range(1,35):
block = df.query(f'date_block_num < {i}' ).groupby(merging_cols ).agg(aggregation ).reset_index()
block.columns = merging_cols + [new_col]
block['date_block_num'] = i
block = block[block[new_col]>0]
temp.append(block)
 temp = pd.concat(temp)
 df = pd.merge(df, temp, on=['date_block_num']+merging_cols, how='left')
 return df
df = past_information(df, ['item_id'],'last_item_price',{'item_price':'last'})
 df = past_information(df, ['shop_id','item_id'],'item_cnt_sum_alltime',{'item_cnt':'sum'})
 df = past_information(df, ['item_id'],'item_cnt_sum_alltime_allshops',{'item_cnt':'sum'})
 del df['revenue'], df['item_cnt_unclipped'], df['item_price']<feature_engineering>
best_alpha=100
best_itr=2400
final_reg_fatal_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_fatal_f.fit(X_train_fatal_f,y_train_fatal_f )
df['relative_price_item_block_lag1'] = df['last_item_price']/df['block_price_lag1']<data_type_conversions>
pred_f=final_reg_fatal_f.predict(X_test_fatal_f)
 print("RMSE is:",(mean_squared_error(y_test_fatal_f,pred_f)) **0.5 )
df['item_cnt_per_day_alltime'] =(df['item_cnt_sum_alltime']/df['prev_days_on_sale'] ).fillna(0)
 df['item_cnt_per_day_alltime_allshops'] =(df['item_cnt_sum_alltime_allshops']/df['prev_days_on_sale'] ).fillna(0 )<set_options>
test["Province_State"].fillna("state", inplace = True)
 test["Country_Region"] = [country_name.replace("'","")for country_name in test["Country_Region"]]
gc.collect()
df = downcast(df )<groupby>
test['ConfirmedCases']=list(map(int,predicted_case))
test['Fatalities']=list(map(int,predicted_fatal))
def matching_name_cat_age(df,n,all_shops):
temp_cols = [f'same_name{n}catage_cnt','date_block_num', f'item_name_first{n}','item_age','category_id']
if all_shops:
temp_cols[0] += '_all_shops'
else:
temp_cols += ['shop_id']
temp = []
for i in range(1,35):
block =(
 df 
.query(f'date_block_num < {i}')
.groupby(temp_cols[2:])
.agg({'item_cnt':'mean'})
.reset_index() 
.rename(columns={'item_cnt':temp_cols[0]})
)
 block = block[block[temp_cols[0]]>0]
block['date_block_num'] = i
temp.append(block)
 temp = pd.concat(temp)
 df = pd.merge(df, temp, on=temp_cols[1:], how='left')
 return df
for n in [4,6,11]:
for all_shops in [True,False]:
df = matching_name_cat_age(df,n,all_shops )<data_type_conversions>
submission=test[['ForecastId','ConfirmedCases','Fatalities']]
submission=shuffle(submission)
 submission.to_csv("submission.csv",index=False )