Kaggle competition: COVID19 Global Forecasting (Week 4)
def predict(test, submission):
predictions = test[['id', 'date', 'demand']]
predictions = pd.pivot(predictions, index = 'id', columns = 'date', values = 'demand' ).reset_index()
predictions.columns = ['id'] + ['F' + str(i + 1)for i in range(28)]
evaluation_rows = [row for row in submission['id'] if 'evaluation' in row]
evaluation = submission[submission['id'].isin(evaluation_rows)]
validation = submission[['id']].merge(predictions, on = 'id')
 final = pd.concat([validation, evaluation])
 final.to_csv('submission_new.csv', index = False)
<define_variables>
warnings.filterwarnings('ignore' )
features = [
"item_id", "dept_id", "cat_id", "store_id", "state_id", "event_name_1", "event_type_1", "snap_CA", "snap_TX", \
"snap_WI", "sell_price", \
"shift_t28", \
"rolling_mean_28_7","rolling_mean_28_28", \
"price_change_t1",
"year", "month", "dayofweek", "before_holiday", "after_holiday"
]
print(f'length: {len(features)}')
 print(features )<categorify>
df_train=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv")
 df_train.tail(5 )
data = pd.read_pickle(f'{INPUT_DIR_PATH}Features{1}.pkl')
 for i in range(2,6):
data = pd.concat([data, pd.read_pickle(f'{INPUT_DIR_PATH}Features{i}.pkl')])
 data.shape<load_from_csv>
df_train.isna().sum()
sales_train_val = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv')
 NUM_ITEMS = sales_train_val.shape[0]
submission = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sample_submission.csv')
 DAYS_PRED = submission.shape[1] - 1
product = sales_train_val[['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']].drop_duplicates() 
<data_type_conversions>
df_train['Province_State'].isna().sum()
weight_mat = np.c_[np.ones([NUM_ITEMS,1] ).astype(np.int8),
pd.get_dummies(product.state_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.store_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.cat_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.dept_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.state_id.astype(str)+ product.cat_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.state_id.astype(str)+ product.dept_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.store_id.astype(str)+ product.cat_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.store_id.astype(str)+ product.dept_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.item_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.state_id.astype(str)+ product.item_id.astype(str),drop_first=False ).astype('int8' ).values,
np.identity(NUM_ITEMS ).astype(np.int8)
 ].T
weight_mat_csr = csr_matrix(weight_mat)
 del weight_mat; gc.collect()
def weight_calc(data, product, sales_train_val):
d_name = ['d_' + str(i+1)for i in range(1913)]
sales_train_val = weight_mat_csr * sales_train_val[d_name].values
df_tmp =(( sales_train_val>0)* np.tile(np.arange(1,1914),(weight_mat_csr.shape[0],1)))
 start_no = np.min(np.where(df_tmp==0,9999,df_tmp),axis=1)-1
flag = np.dot(np.diag(1/(start_no+1)) , np.tile(np.arange(1,1914),(weight_mat_csr.shape[0],1)))<1
sales_train_val = np.where(flag,np.nan,sales_train_val)
 weight1 = np.nansum(np.diff(sales_train_val,axis=1)**2,axis=1)/(1913-start_no)
 df_tmp = data[(data['date'] > '2016-03-27')&(data['date'] <= '2016-04-24')]
df_tmp['amount'] = df_tmp['demand'] * df_tmp['sell_price']
df_tmp =df_tmp.groupby(['id'])['amount'].apply(np.sum)
 df_tmp = df_tmp[product.id].values
weight2 = weight_mat_csr * df_tmp
weight2 = weight2/np.sum(weight2)
 del sales_train_val
gc.collect()
return weight1, weight2
weight1, weight2 = weight_calc(data,product,sales_train_val)
 def wrmsse(preds, data):
y_true = data.get_label()
y_true = y_true[-(NUM_ITEMS * DAYS_PRED):]
preds = preds[-(NUM_ITEMS * DAYS_PRED):]
num_col = DAYS_PRED
reshaped_preds = preds.reshape(num_col, NUM_ITEMS ).T
reshaped_true = y_true.reshape(num_col, NUM_ITEMS ).T
train = weight_mat_csr*np.c_[reshaped_preds, reshaped_true]
score = np.sum(
 np.sqrt(
 np.mean(
 np.square(
 train[:,:num_col] - train[:,num_col:])
 ,axis=1)/ weight1)* weight2)
 return 'wrmsse', score, False
def wrmsse_simple(preds, data):
y_true = data.get_label()
y_true = y_true[-(NUM_ITEMS * DAYS_PRED):]
preds = preds[-(NUM_ITEMS * DAYS_PRED):]
num_col = DAYS_PRED
reshaped_preds = preds.reshape(num_col, NUM_ITEMS ).T
reshaped_true = y_true.reshape(num_col, NUM_ITEMS ).T
train = np.c_[reshaped_preds, reshaped_true]
weight2_2 = weight2[:NUM_ITEMS]
weight2_2 = weight2_2/np.sum(weight2_2)
 score = np.sum(
 np.sqrt(
 np.mean(
 np.square(
 train[:,:num_col] - train[:,num_col:])
 ,axis=1)/ weight1[:NUM_ITEMS])*weight2_2)
 return 'wrmsse', score, False<split>
df_train['Province_State']=df_train['Province_State'].fillna('')
 df_train['Province_State'].isna().sum()
x_train, y_train, x_val, y_val, test = train_test_split(data)
 del data<compute_test_metric>
df_train['Country_Region']=df_train['Country_Region']+'_'+df_train['Province_State']
test, model = run_lgb(x_train, y_train, x_val, y_val, test )<load_from_csv>
df_test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv")
 df_test.sample(5 )
<drop_column>
df_test.isna().sum()
del x_train, y_train, x_val, y_val
del weight_mat_csr
gc.collect()<predict_on_test>
df_test['Province_State']=df_test['Province_State'].fillna('')
 df_test['Province_State'].isna().sum()
predict(test, submission )<set_options>
df_test['Country_Region']=df_test['Country_Region']+'_'+df_test['Province_State']
!pip install catboost
!pip install ipywidgets
!jupyter nbextension enable --py widgetsnbextension
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 50)
 pd.set_option('display.width', 1000)
<load_pretrained>
def create_features(df,label=None):

 df = df.copy()
df['Date'] = df.index
df['dayofweek'] = df['Date'].dt.dayofweek
df['quarter'] = df['Date'].dt.quarter
df['month'] = df['Date'].dt.month
df['dayofyear'] = df['Date'].dt.dayofyear
df['dayofmonth'] = df['Date'].dt.day
df['weekofyear'] = df['Date'].dt.weekofyear
X = df[['dayofweek','quarter','month',
'dayofyear','dayofmonth','weekofyear']]
return X
X_train = pd.read_pickle('.. /input/featureengineering/df_data_features.pkl')
 day_ini = X_train['d'].min()
day_train_last = 1913
print('start day',day_ini)
 day_train_first = day_train_last - 1*365
X_train = X_train.loc[(X_train['d'] >= day_train_first), :]
X_train = X_train.loc[(( X_train['Lag28_rmean28_id'].notna())&(X_train['type'] =='train')) |(X_train['type'] =='val'), :]
print('First training day: ', day_train_first)
 print('Last training day: ', day_train_last)
 print(X_train.groupby(['year', 'month'])['d'].count())
 X_train.columns = [c.replace('[','')for c in X_train.columns]
X_train.columns = [c.replace(']','')for c in X_train.columns]
X_train.columns = [c.replace(' ','_')for c in X_train.columns]
X_train.columns = [c.replace(',','')for c in X_train.columns]
X_train.columns = [c.replace("'",'')for c in X_train.columns]
print(X_train.info())
 X_train.head() 
<drop_column>
train = df_train.set_index(['Date'])
 train.index = pd.to_datetime(train.index)
 train_features=pd.DataFrame(create_features(train))
train_features
def repeat_data(df):
rep_month = 0
rep_event = 0
rep_event_test = 0
months = [5]
events_test = ['Pesach End', 'OrthodoxEaster', 'Cinco De Mayo', "Mother's day"]
events2 = df[(df['event_name_2'].notna())]
ts = events2.loc[:,'event_name_2']
events2.loc[:,'event_name_1'] = ts
ts = events2.loc[:,'event_type_1']
events2.loc[:,'event_type_2'] = ts
df = df.append(events2, ignore_index=True)
 df = df.drop(['event_name_2', 'event_type_2'], axis='columns')
 df_month = df.loc[df['month'].isin(months), :]
df_events_test = df.loc[df['event_name_1'].isin(events_test), :]
df_events = df.loc[df['event_name_1'].notna() , :]
for i in range(rep_month):
df = df.append(df_month,ignore_index=True)
 for i in range(rep_event):
df = df.append(df_events,ignore_index=True)
 for i in range(rep_event_test):
df = df.append(df_events_test,ignore_index=True)
 df[['event_name_1', 'event_type_1']] = df[['event_name_1', 'event_type_1']].astype('category')
 return df 
<feature_engineering>
df_train_final = pd.concat([train,train_features], axis=1)
 df_train_final.reset_index(drop=True ,inplace =True)
 df_train_final
def addTrendFeatures(df):
df.loc[df['Lag1_rmean7_id'] > 0, 'Lag1_rmean1_id_div_Lag1_rmean7_id'] = df['Lag1_rmean1_id']/df['Lag1_rmean7_id']
df.loc[(df['Lag1_rmean7_id'] == 0)&(df['Lag1_rmean1_id'] > 0), 'Lag1_rmean1_id_div_Lag1_rmean7_id'] = df['Lag1_rmean1_id']
df.loc[(df['Lag1_rmean7_id'] == 0)&(df['Lag1_rmean1_id'] == 0), 'Lag1_rmean1_id_div_Lag1_rmean7_id'] = -1
df.loc[df['Lag1_rmean28_id'] > 0, 'Lag1_rmean7_id_div_Lag1_rmean28_id'] = df['Lag1_rmean7_id']/df['Lag1_rmean28_id']
df.loc[(df['Lag1_rmean28_id'] == 0)&(df['Lag1_rmean7_id'] > 0), 'Lag1_rmean7_id_div_Lag1_rmean28_id'] = df['Lag1_rmean7_id']
df.loc[(df['Lag1_rmean28_id'] == 0)&(df['Lag1_rmean7_id'] == 0), 'Lag1_rmean7_id_div_Lag1_rmean28_id'] = -1
df['Lag1_rmean1_id_div_Lag1_rmean7_id'] = df['Lag1_rmean1_id_div_Lag1_rmean7_id'].astype('float32')
 df['Lag1_rmean7_id_div_Lag1_rmean28_id'] = df['Lag1_rmean7_id_div_Lag1_rmean28_id'].astype('float32')
 print(df.head())
 return df
def addDateTypeAvg(df, cols):
print(f'Calculating item average over {cols} grouping')
 df[f'{cols}_Avg'] = df.groupby(cols)['saleCount'].transform(lambda x : np.round(x.mean() ,2))
return df 
<define_variables>
le = LabelEncoder()
df_train_final['Country_Region'] = le.fit_transform(df_train_final['Country_Region'] )
best_cols = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id',
'wday', 'month', 'year', 'event_name_1', 'event_type_1',
'snap_CA', 'snap_TX', 'snap_WI', 'sell_price',
'available',
'Lag1_rmean1_id', 'Lag7_rmean1_id', 'Lag28_rmean1_id',
'Lag1_rmean7_id', 'Lag7_rmean7_id', 'Lag28_rmean7_id',
'Lag1_rmean28_id', 'Lag28_rmean28_id',
"id_wday_Avg", "id_month_Avg",
"item_id_wday_Avg", "item_id_month_Avg"]
cols_drop = ['event_name_2', 'event_type_2','available', 'cat_id','state_id', 'dept_id', 'year']
X_train = X_train.drop(cols_drop, axis='columns')
 X_train.info()<define_variables>
df_train_final.drop(columns='Province_State' , inplace=True)
 df_train_final
cols_ignore = ['id', 'd', 'saleCount', 'type',
'event_name_2', 'event_type_2', 'available']
col_feature = [i
for i in X_train.columns
if i not in cols_ignore]
mask_val =(X_train['d'].between(1913 - 365, 1913-365 +28)) |(X_train['d'].between(1913-28, 1913))
mask_train = X_train['type']=='train'
print('Feature columns:
',col_feature)
 cat_cols = X_train[col_feature].select_dtypes(include=['category'] ).columns
for c in X_train.select_dtypes(include=['category'] ).columns:
if c not in ['id','type']:
X_train.loc[:,c] = X_train.loc[:,c].cat.codes
X_train.info()<filter>
test = df_test.set_index(['Date'])
 test.index = pd.to_datetime(test.index)
 test_features=pd.DataFrame(create_features(test))
test_features
X_train[(X_train['id'] == 'FOODS_3_318_CA_1_validation')& mask_train]<categorify>
df_test_final = pd.concat([test,test_features], axis=1)
 df_test_final.reset_index(drop=True ,inplace =True)
 df_test_final
def repNAN(df, cols, val):
print('Setting NAN values to {}'.format(val))
for c in cols:
print(f'Col.{c} contains {df[c].isna().sum() } NAN')
 df.loc[df[c].isna() , c] = val
return df
cols_rep = ['Lag1_rmean1_id', 'Lag7_rmean1_id', 'Lag28_rmean1_id',
'Lag1_rmean7_id', 'Lag7_rmean7_id', 'Lag28_rmean7_id',
'Lag1_rmean28_id', 'Lag28_rmean28_id']
X_train = repNAN(X_train, ['sell_price'], -1)
 X_train = repNAN(X_train, ['saleCount'], 0)
<choose_model_class>
le = LabelEncoder()
df_test_final['Country_Region'] = le.fit_transform(df_test_final['Country_Region'] )
def run_nn() :
n_feat = X_train[col_feature].shape[1]
n_items = X_train['id'].nunique()
print(n_feat, n_items)
 model = tf.keras.Sequential([
tf.keras.layers.InputLayer(input_shape=(n_feat,)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(n_feat*3, activation='sigmoid'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(n_feat*2, activation='sigmoid'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(n_feat*1, activation='sigmoid'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(1, activation=None)])
 model.summary()
model.compile(optimizer="Adam", loss="mse")
 callback = tf.keras.callbacks.EarlyStopping(monitor='mse', mode='auto',min_delta=0.01, patience=10)
 t0 = time.time()
history = model.fit(X_train.loc[mask_train, col_feature],
X_train.loc[mask_train, 'saleCount'],
batch_size=n_items,
epochs=30)
 plt.figure()
plt.plot(history.history['loss'],label='Training')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend()
plt.show()
return model 
<choose_model_class>
df_test_final.drop(columns='Province_State' , inplace=True)
 df_test_final
def run_xgboost(max_depth=6,n_estimators=1000,min_child_weight=1, learning_rate=0.01, gamma=0, subsample=1,colsample_bytree = 1):
model = xgb.XGBRegressor(
 learning_rate=learning_rate,
n_estimators=n_estimators,
max_depth = max_depth,
min_child_weight = min_child_weight,
gamma = gamma,
subsample=subsample,
colsample_bytree = colsample_bytree,
random_state =1,
verbosity=2, 
)
 model.fit(X_train.loc[mask_train, col_feature],
X_train.loc[mask_train, 'saleCount'],
verbose=True)
 return model<train_model>
df_test_final = df_test_final.drop(['ForecastId'],axis=1 )
def run_rf(n_range=1):
score_train = []
score_val = []
train_time = []
for c in X_train.select_dtypes(include=['category'] ).columns:
if c not in ['id','type']:
X_train.loc[:,c] = X_train.loc[:,c].cat.codes
print(f'training with trees {n_range}')
 for n in n_range:
model = RandomForestRegressor(n_estimators = n,
max_depth = None,
random_state=1,
n_jobs = -1,
bootstrap = True,
max_features='sqrt',
min_samples_leaf = 10,
verbose=10)
 t0 = time.time()
print('
',f'Training RF with n={n} initiated at {time.ctime(t0)}')
 model.fit(X_train.loc[~mask_val & mask_train, col_feature], X_train.loc[~mask_val & mask_train, 'saleCount'])
 dt =(time.time() - t0)/60
train_time.append(dt)
 print('Tree depths: ', [estimator.get_depth() for estimator in model.estimators_])
 score_train.append(model.score(X_train.loc[~mask_val & mask_train, col_feature], X_train.loc[~mask_val & mask_train, 'saleCount']))
score_val.append(model.score(X_train.loc[mask_val & mask_train, col_feature], X_train.loc[mask_val & mask_train, 'saleCount']))
for i,(t, v)in enumerate(zip(score_train, score_val)) :
print(f'{n_range[i]} trees(training, validation):({np.round(t,2)}, {np.round(v,2)}), {np.round(train_time[i],3)}')
 if n != n_range[-1]:
del model
gc.collect()
return model 
<train_model>
df_train_final['ConfirmedCases'] = df_train_final['ConfirmedCases'].apply(int)
 df_train_final['Fatalities'] = df_train_final['Fatalities'].apply(int )
def run_cb_val() :
model = CatBoostRegressor(iterations=2000,
learning_rate=0.05,
depth=16,
random_strength = 0,
l2_leaf_reg = 3,
min_data_in_leaf = 1,
boosting_type='Plain',
border_count=256,
task_type='GPU',
random_state=1,
one_hot_max_size = 1)
 cat_cols = X_train[col_feature].select_dtypes(include=['category'] ).columns
print('Converting categorical datatype to object')
 print(X_train[col_feature].columns)
 cat_idx = []
model.fit(X_train.loc[~mask_val & mask_train, col_feature],
X_train.loc[~mask_val & mask_train, 'saleCount'],
eval_set =(X_train.loc[mask_val & mask_train, col_feature],
X_train.loc[mask_val & mask_train, 'saleCount']),
early_stopping_rounds = 10,
cat_features = cat_idx, plot=True)
 return model
def run_cb_train() :
model = CatBoostRegressor(iterations=1000,
learning_rate=0.05,
depth=16,
random_strength = 0,
l2_leaf_reg = 5,
min_data_in_leaf = 1,
boosting_type='Plain',
border_count=512,
task_type='GPU',
logging_level='Verbose',
random_state=1,
one_hot_max_size = 1 
)
 print('Fitting model')
 model.fit(X_train.loc[mask_train, col_feature],
X_train.loc[mask_train, 'saleCount'])
 return model 
<feature_engineering>
y_conf=df_train_final['ConfirmedCases']
y_conf.sample(6)
 y_fat=df_train_final['Fatalities']
def addLaggedRollingAvg(df, grp_cols, val_col, windows, lag):
for w in windows:
for l in lag:
col_name = f'Lag{l}_rmean{w}_{grp_cols}'
df[col_name] = df.groupby(grp_cols)[val_col] \ 
.transform(lambda x : np.round(x.shift(l ).rolling(w, min_periods=1 ).mean() ,2))
return df
X_test = X_train.loc[X_train['d'] >= 1913-(28+1), :]
d_min_val = X_train.loc[X_train['type'] == 'val', 'd'].min()
d_max_val = X_train.loc[X_train['type'] == 'val', 'd'].max()
X_test.loc[X_test['id'] == 'FOODS_3_318_CA_3_validation', :].head(100)
<compute_train_metric>
df_train_final_fat = df_train_final.drop(['Id','Fatalities'],axis=1)
 df_train_final_fat.columns
for d in range(d_min_val,d_max_val+1):
print(f'Day {d}')
 X_test = addLaggedRollingAvg(X_test, grp_cols='id', val_col = 'saleCount', windows=[1], lag = [1,7])
 X_test = addLaggedRollingAvg(X_test, grp_cols='id', val_col = 'saleCount', windows=[7], lag = [1, 7])
 X_test = addLaggedRollingAvg(X_test, grp_cols='id', val_col = 'saleCount', windows=[28], lag = [1])
 y_pred = model.predict(X_test.loc[X_test['d'] == d, col_feature])
 print('Negative sale predictions: ', sum(y_pred < 0))
y_pred[y_pred < 0] = 0
X_test.loc[X_test['d'] == d, 'saleCount'] = y_pred
X_test.loc[(X_test['id'] == 'FOODS_3_318_CA_3_validation')&(X_test['type'] == 'val'), :].head(30 )<count_values>
df_train_final_ConCases = df_train_final.drop(['Id','ConfirmedCases','Fatalities'],axis=1)
 df_train_final_ConCases.columns
X_test[(X_test['d'] == d)&(X_test['saleCount'] < 0)]['d'].count()<feature_engineering>
X_train,X_test,y_train,y_test = train_test_split(df_train_final_ConCases,y_conf,test_size=0.2,random_state=42 )
print('Generating forecast')
 X_forecast = X_test.loc[X_test['type']=='val',['id', 'd', 'saleCount']].copy()
X_forecast.loc[(X_forecast['saleCount'] < 0), 'saleCount'] = 0
X_forecast.loc[:,'saleCount'] = np.round(X_forecast.loc[:,'saleCount'],3)
 days = X_forecast['d'].unique()
n_days = len(days)
 day_min = min(days)
 print(days)
 assert n_days == 28, 'Wrong number of forecastet days'
day_1 = days.min()
X_forecast['d'] = 'F'+ X_forecast['d'].add(- day_min + 1 ).astype('str')
 print('Done' )<load_from_csv>
rf = XGBRegressor(n_estimators = 2400 , random_state = 0 , max_depth = 26)
 rf.fit(X_train,y_train )
df_sub = pd.read_csv('.. /input/m5-forecasting-accuracy/sample_submission.csv')
 fcols = [f for f in df_sub.columns if 'F' in f]
for f in fcols:
X_forecast_f = X_forecast[X_forecast['d'] == f]
df_sub[f] = df_sub.merge(X_forecast_f,
how = 'right',
left_on = 'id',
right_on = 'id')['saleCount']
df_sub.fillna(0)
 df_sub.to_csv('submission.csv', index=False, na_rep=0)
 print('Done' )<import_modules>
pred_conf = rf.predict(X_test)
 predictions = [round(value)for value in pred_conf]
predictions
import os
import gc
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime, timedelta, date
from tqdm.notebook import tqdm
import lightgbm as lgb<define_variables>
rf.score(X_test,y_test )
TEST_RUN = False
MODEL_VERSION = 'final'
pd.set_option('display.max_columns', 50)
 KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
SUBMISSION_PATH = "/kaggle/input/rausnaus-lightgbm-per-category-best-notebook/submission.csv"
PATH_TO_CAT0 = "/kaggle/input/rausnaus-lightgbm-per-category-best-notebook/model_cat0_v10.lgb"
PATH_TO_CAT1 = "/kaggle/input/rausnaus-lightgbm-per-category-best-notebook/model_cat1_v10.lgb"
PATH_TO_CAT2 = "/kaggle/input/rausnaus-lightgbm-per-category-best-notebook/model_cat2_v10.lgb"
PATH_MODELS = [PATH_TO_CAT0, PATH_TO_CAT1, PATH_TO_CAT2]
KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
BACKWARD_LAG = 60
END_DAY = 1913+28
BEGIN_DATE = '2013-08-01'
BEGIN_DAY = str(( datetime.strptime(BEGIN_DATE, '%Y-%m-%d')- datetime.strptime('2011-01-29', '%Y-%m-%d')).days)
 EVAL_SPLIT = '2016-05-22'
TRAIN_SPLIT = str(datetime.strptime(EVAL_SPLIT, '%Y-%m-%d')- timedelta(days=28)) [:10]
print(f"Train split: {datetime.strptime(TRAIN_SPLIT, '%Y-%m-%d')}")
 print(f"Eval split: {datetime.strptime(EVAL_SPLIT, '%Y-%m-%d')}" )<define_variables>
r=np.sqrt(mean_squared_error(y_test,predictions))
r
CALENDAR_DTYPES = {
'date': 'str',
'wm_yr_wk': 'int16',
'weekday': 'category',
'wday': 'int16',
'month': 'int16',
'year': 'int16',
'd': 'object',
'event_name_1': 'category',
'event_type_1': 'category',
'event_name_2': 'category',
'event_type_2': 'category',
'snap_CA': 'int16',
'snap_TX': 'int16',
'snap_WI': 'int16'
}
PARSE_DATES = ['date']
SALES_PRICES_DTYPES = {
'store_id': 'category',
'item_id': 'category',
'wm_yr_wk': 'int16',
'sell_price': 'float32',
}<load_pretrained>
mae = mean_absolute_error(y_test,predictions)
 print('MAE: %f' % mae )
def load_data(train=True):

 print("Loading train and validation data")
 numcols = [f"d_{day}" for day in range(int(BEGIN_DAY),END_DAY+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 sales_train_validation = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_evaluation.csv'),
usecols=catcols+numcols, dtype=dtype)
 for col in catcols:
if col != "id":
sales_train_validation[col] = sales_train_validation[col].cat.codes.astype("int16")
 sales_train_validation[col] -= sales_train_validation[col].min()
if not train:
for day in range(END_DAY+1, END_DAY+28+1):
sales_train_validation[f"d_{day}"] = 0
value_vars = [column for column in sales_train_validation.columns
if(column.startswith('d_')and int(column.replace('d_', '')) >= END_DAY - BACKWARD_LAG)]
else:
value_vars = [col for col in sales_train_validation.columns
if(col.startswith('d_')and(int(col.replace('d_', '')) >= int(BEGIN_DAY)))]
print("Shape:", sales_train_validation.shape)
 print("Memory usage(Mb)before melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 sales_train_validation = pd.melt(
 sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
value_vars=value_vars,
var_name = 'day',
value_name = 'sales')
 print("Completed melting, new shape:", sales_train_validation.shape)
 print("Colums after melting:", sales_train_validation.columns)
 print("Memory usage(Mb)after melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
print("Loading calendar")
 calendar = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'calendar.csv'), dtype=CALENDAR_DTYPES, parse_dates=['date'])
 print("Calendar columns: ", calendar.columns)
 print("Memory usage(Mb)calendar: ", calendar.memory_usage().sum() / 1024**2)
 calendar.rename(columns={'d':'day'}, inplace=True)
 for col, col_dtype in CALENDAR_DTYPES.items() :
if col_dtype == "category":
calendar[col] = calendar[col].cat.codes.astype("int16")
 calendar[col] -= calendar[col].min()
sales_train_validation = sales_train_validation.merge(calendar, on="day", copy=False)
 del calendar; gc.collect()
print("Merged calendar(in place)")
 print("Memory usage(Mb)after merging calendar:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 sell_prices = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sell_prices.csv'), dtype=SALES_PRICES_DTYPES)
 for col, col_dtype in SALES_PRICES_DTYPES.items() :
if col_dtype == "category":
sell_prices[col] = sell_prices[col].cat.codes.astype("int16")
 sell_prices[col] -= sell_prices[col].min()
print("Memory usage(Mb)sell prices:", sell_prices.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'store_id', 'sell_price']
for feature in columns:
if feature == 'sell_price':
sell_prices[feature].fillna(0, inplace=True)
 sales_train_validation = sales_train_validation.merge(sell_prices, on=["store_id","item_id","wm_yr_wk"], copy=False)
 del sell_prices; gc.collect()
print("Merged sales prices(in place)")
 print("Memory usage(Mb)after merging sales:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 return sales_train_validation<categorify>
mse = mean_squared_error(y_test,predictions)
 print('MSE: %f' % mse )
def make_lag_features(strain):

 print('in dataframe:', strain.shape)
 print("headers:", strain.columns)
 lags = [7, 28]
lag_cols = ['lag_{}'.format(lag)for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
strain[lag_col] = strain[['item_id', 'sales']].groupby('item_id')['sales'].shift(lag)
 print('lag sales done')
 windows= [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
window_col = f'item_id_rmean_{lag_col}_{window}'
strain[window_col] = strain[['item_id', lag_col]].groupby('item_id')[lag_col].transform(
 lambda x: x.rolling(window ).mean() 
)
 print(f'Rolling mean sales done for window {window} per item_id')
 def make_date_features(dt):
date_features = {
'week': 'weekofyear',
'quarter': 'quarter',
'mday': 'day',
"wday": "weekday",
"month": "month",
"year": "year"
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt['date'].dt, date_feat_func ).astype('int16')
 print('date features done')
 dt['day'] = dt['day'].apply(lambda x: int(x.replace('d_', '')))
 print('out dataframe:', dt.shape )<load_pretrained>
rmse = sqrt(mse)
 print('RMSE: %f' % rmse )
%%time
sales_train_evaluation = load_data()<feature_engineering>
R2=r2_score(y_test,predictions)
 print('R2: %f' % R2 )
%%time
sales_train_evaluation["sale"] =(( sales_train_evaluation['sell_price'] * 100 % 10)< 6 ).astype('int8')
 make_lag_features(sales_train_evaluation)
 make_date_features(sales_train_evaluation )<categorify>
dfpred = pd.DataFrame(data=predictions, columns=[ "pred"] , index=y_test.index)
 dfpred1=pd.concat([dfpred,y_test],axis=1)
 dfpred1.sample(30 )
before = len(sales_train_evaluation)
 sales_train_evaluation.dropna(inplace = True)
 after = len(sales_train_evaluation)
 print(f"Reduced {(before-after)/before}%" )<split>
X_train2,X_test2,y_train2,y_test2 = train_test_split(df_train_final_fat,y_fat,test_size=0.2,random_state=42 )
print("Splitting data into train, validation, evaluation set")
 train = sales_train_evaluation[(sales_train_evaluation['date'] >= BEGIN_DATE)&(sales_train_evaluation['date'] <= TRAIN_SPLIT)]
validation = sales_train_evaluation[(sales_train_evaluation['date'] > TRAIN_SPLIT)&(sales_train_evaluation['date'] <= EVAL_SPLIT)]
del sales_train_evaluation; gc.collect()<count_values>
rf2 = XGBRegressor(n_estimators = 2000 , random_state = 0 , max_depth = 20)
 rf2.fit(X_train2,y_train2 )
print(train["cat_id"].value_counts())
 print(validation["cat_id"].value_counts() )<define_variables>
pred_fat = rf2.predict(X_test2)
 predictions_fat = [round(value)for value in pred_fat]
predictions_fat
categorical_features = [
'item_id', 'dept_id', 'store_id', 'state_id',
'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2'
]<drop_column>
rf2.score(X_test2,y_test2 )
train_sets = [train[train["cat_id"] == category].drop("cat_id", axis = 1)for category in range(3)]
del train; gc.collect()
val_sets = [validation[validation["cat_id"] == category].drop("cat_id", axis = 1)for category in range(3)]
del validation; gc.collect()<create_dataframe>
r=np.sqrt(mean_squared_error(y_test2,predictions_fat))
r
train_pools = [lgb.Dataset(
 data=train_sets[category][train_columns],
label=train_sets[category]["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 for category in range(3)]
del train_sets; gc.collect()<create_dataframe>
mae2 = mean_absolute_error(y_test2,predictions_fat)
 print('MAE: %f' % mae2 )
val_pools = [lgb.Dataset(
 data=val_sets[category][train_columns],
label=val_sets[category]["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 for category in range(3)]
del val_sets; gc.collect()<init_hyperparams>
mse2 = mean_squared_error(y_test2,predictions_fat)
 print('MSE: %f' % mse2 )
if TEST_RUN:
ITERATIONS=1
else:
ITERATIONS = 1200
params = {
"objective" : "poisson",
"metric" : "rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
'verbosity': 1,
'num_iterations': ITERATIONS,
'num_leaves': 128,
"min_data_in_leaf": 100,
}<train_model>
rmse2 = sqrt(mse2)
 print('RMSE: %f' % rmse2 )
%%time
models = [lgb.train(params,
train_pools[category],
valid_sets = val_pools[category],
verbose_eval=20)
 for category in range(3)]
for cat,model in enumerate(models):
model.save_model(f"model_cat{cat}_{MODEL_VERSION}.lgb" )<set_options>
R2_fat=r2_score(y_test2,predictions_fat)
 print('R2: %f' % R2_fat )
del train_pools, val_pools; gc.collect()<data_type_conversions>
dfpred_fat = pd.DataFrame(data=predictions_fat, columns=[ "pred"] , index=y_test2.index)
 dfpred_fat1=pd.concat([dfpred_fat,y_test2],axis=1)
 dfpred_fat1.sample(30 )
df = load_data(train=False)
 df["sale"] =(( df['sell_price'] * 100 % 10)< 6 ).astype('int8')
 make_date_features(df )<feature_engineering>
accuracy = accuracy_score(y_test2,predictions_fat)
 print("Accuracy: %.2f%%" %(accuracy * 100.0))
def lag_features_for_day(dt, day):
print(type(dt))
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt['date'] == str(day), lag_col] = \
dt.loc[dt['date'] == str(day-timedelta(days=lag)) , 'sales'].values
windows = [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("item_id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'item_id'])
 dt.loc[dt['date'] == str(day),f'item_id_rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
lag = 28
lag_col = 'lag_28'
window = 28
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("store_id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'store_id'])
 dt.loc[dt['date'] == str(day),f'store_id_rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
print(f"Features done for day {str(day)}")
<define_variables>
xgb = XGBRegressor(n_estimators = 2400 , random_state = 0 , max_depth = 26)
 xgb.fit(df_train_final_ConCases,y_conf )
%%time
END_DATE = EVAL_SPLIT
PREDICT_DAYS = 28
for f_day in tqdm(range(1,PREDICT_DAYS+1)) :
pred_date =(datetime.strptime(END_DATE, '%Y-%m-%d')+ timedelta(days=f_day)).date()
print(f"Forecasting day {END_DAY+f_day}, date: {str(pred_date)}")
 pred_begin_date = pred_date - timedelta(days=BACKWARD_LAG+1)
 print(pred_begin_date)
 prediction_data = df[(df['date'] >= str(pred_begin_date)) &(df['date'] <= str(pred_date)) ].copy()
lag_features_for_day(prediction_data, pred_date)
 for i in range(3):
prediction_data_cat = prediction_data.loc[(prediction_data['date'] == str(pred_date)) &(prediction_data["cat_id"] == i), train_columns]
prediction = models[i].predict(prediction_data_cat)
 df.loc[(df['date'] == str(pred_date)) &(df.index.isin(prediction_data_cat.index)) , 'sales'] = prediction
print("Prediction", prediction.size, prediction )<set_options>
pred_conf = xgb.predict(df_test_final)
 predictions_conf = [round(value)for value in pred_conf]
predictions_conf
del prediction_data
gc.collect()<load_from_csv>
predictions_conf = np.around(predictions_conf ,decimals = 0)
 predictions_conf
sales_ = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_evaluation.csv'))
submission = pd.read_csv(os.path.join(SUBMISSION_PATH))<feature_engineering>
conf_test=pd.DataFrame(predictions_conf,index=df_test_final.index,columns=["ConfirmedCases"])
 df_test_final=pd.concat([df_test_final,conf_test],axis=1 )
submission_eval = df.loc[df['date'] > END_DATE, ['id', 'day', 'sales']].copy()
submission_eval.loc[submission_eval['sales'] < 0, 'sales'] = 0
submission_eval.sort_values('id', inplace=True)
 submission_eval['day'] = submission_eval['day'].apply(lambda x: 'F{}'.format(x - END_DAY))
print(submission_eval.columns)
 print(submission_eval.head() , submission_eval.tail() , sep="
" )<data_type_conversions>
df_test_final.loc[10135]
submission.iloc[30490:, 1:] = submission_eval.iloc[:,1:].to_numpy()
print(submission.head() , submission.tail() , sep="
" )<save_to_csv>
xgb2 = XGBRegressor(n_estimators = 2400 , random_state = 0 , max_depth = 26)
 xgb2.fit(df_train_final_fat,y_fat )
submission.to_csv('submission.csv', index=False)
 print('Submission shape', submission.shape )<set_options>
df_test_final=df_test_final[['Country_Region', 'ConfirmedCases', 'dayofweek', 'quarter', 'month',
'dayofyear', 'dayofmonth', 'weekofyear']]
warnings.filterwarnings("ignore" )<define_variables>
pred_fat = xgb2.predict(df_test_final)
 predictions_fat = [round(value)for value in pred_fat]
predictions_fat
seed = 42<train_on_grid>
predictions_fat = np.around(predictions_fat ,decimals = 0)
 predictions_fat
DEVICE = "TPU"
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 print('Number of devices: {}'.format(strategy.num_replicas_in_sync))<define_variables>
df_sub=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/submission.csv")
 df_sub.tail(5 )
dropout_model = 0.36
hidden_dim_first = 128
hidden_dim_second = 256
hidden_dim_third = 128<load_from_csv>
df_sub['ConfirmedCases']=predictions_conf
df_sub['Fatalities']=predictions_fat
<define_variables><EOS> 
df_sub.to_csv("submission.csv" , index = False )