Kaggle competition: COVID19 Global Forecasting (Week 3)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-week-3<train_model>
import plotly.graph_objects as go
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
import time
from datetime import datetime
from pathlib import Path
from sklearn import preprocessing
import keras.backend as K
from keras.models import Sequential
from keras.layers import Dense, LSTM, RNN, Dropout
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras import optimizers
from sklearn.preprocessing import StandardScaler, MinMaxScaler, LabelEncoder
from sklearn.model_selection import train_test_split
hist=model.fit(x=X_train,y=y_train,batch_size=128,epochs=200,verbose=2,validation_split=0.2)
 hist<import_modules>
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/train.csv', parse_dates=['Date'])
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv',parse_dates=['Date'])
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/submission.csv')
 train.tail()
from sklearn.metrics import r2_score 
<compute_test_metric>
mask = train['Date'].max()
world_cum_confirmed = sum(train[train['Date'] == mask].ConfirmedCases)
 world_cum_fatal = sum(train[train['Date'] == mask].Fatalities )
y_pred =model.predict(X_train)
 score = r2_score(y_train,y_pred)
 score<load_from_csv>
print('Number of Countires are: ', len(train['Country_Region'].unique()))
print('Training dataset ends at: ', mask)
 print('Number of cumulative confirmed cases worldwide are: ', world_cum_confirmed)
 print('Number of cumulative fatal cases worldwide are: ', world_cum_fatal )
test = pd.read_csv('.. /input/test/test.csv' )<prepare_x_and_y>
cum_per_country = train[train['Date'] == mask].groupby(['Date','Country_Region'] ).sum().sort_values(['ConfirmedCases'], ascending=False)
 cum_per_country[:10]
y=np.arange(1,501 )<string_transform>
date = train['Date'].unique()
cc_us = train[train['Country_Region'] == 'US'].groupby(['Date'] ).sum().ConfirmedCases
ft_us = train[train['Country_Region'] == 'US'].groupby(['Date'] ).sum().Fatalities
cc_ity = train[train['Country_Region'] == 'Italy'].groupby(['Date'] ).sum().ConfirmedCases
ft_ity = train[train['Country_Region'] == 'Italy'].groupby(['Date'] ).sum().Fatalities
cc_spn = train[train['Country_Region'] == 'Spain'].groupby(['Date'] ).sum().ConfirmedCases
ft_spn = train[train['Country_Region'] == 'Spain'].groupby(['Date'] ).sum().Fatalities
cc_gmn = train[train['Country_Region'] == 'Germany'].groupby(['Date'] ).sum().ConfirmedCases
ft_gmn = train[train['Country_Region'] == 'Germany'].groupby(['Date'] ).sum().Fatalities
cc_frc = train[train['Country_Region'] == 'France'].groupby(['Date'] ).sum().ConfirmedCases
ft_frc = train[train['Country_Region'] == 'France'].groupby(['Date'] ).sum().Fatalities
fig = go.Figure()
fig.add_trace(go.Scatter(x=date, y=cc_us, name='US'))
fig.add_trace(go.Scatter(x=date, y=cc_ity, name='Italy'))
fig.add_trace(go.Scatter(x=date, y=cc_spn, name='Spain'))
fig.add_trace(go.Scatter(x=date, y=cc_gmn, name='Germany'))
fig.add_trace(go.Scatter(x=date, y=cc_frc, name='France'))
fig.update_layout(title="Plot of Cumulative Cases for Top 5 countires(except China)",
xaxis_title="Date",
yaxis_title="Cases")
 fig.update_xaxes(nticks=30)
 fig.show()
image_list=[]
for i in test['Image']:
i=i.split(' ')
 image_list.append(i)
 len(image_list )<prepare_x_and_y>
train.columns = train.columns.str.lower()
test.columns = test.columns.str.lower()
image_list=np.array(image_list,dtype=float)
 images=image_list.reshape(-1,96,96,1)
 X_test =images/255.0 
<predict_on_test>
train.fillna(' ',inplace=True)
 test.fillna(' ', inplace=True)
 train_id = train.pop('id')
 test_id = test.pop('forecastid')
 train['cp'] = train['country_region'] + train['province_state']
test['cp'] = test['country_region'] + test['province_state']
train.drop(['province_state','country_region'], axis=1, inplace=True)
 test.drop(['province_state','country_region'], axis =1, inplace=True )
predicted_value =model.predict(X_test )<create_dataframe>
df = pd.DataFrame()
def create_time_feat(data):
df['date']= data['date']
df['hour']=df['date'].dt.hour
df['weekofyear']=df['date'].dt.weekofyear
df['quarter'] =df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['dayofyear']=df['date'].dt.dayofyear
x=df[['hour','weekofyear','quarter','month','dayofyear']]
return x
cr_tr = create_time_feat(train)
 cr_te = create_time_feat(test )
pv =pd.DataFrame(data=predicted_value)
 img_show(image_list,pv )<save_to_csv>
train_df = pd.concat([train,cr_tr], axis=1)
 test_df = pd.concat([test, cr_te], axis =1)
 test_df.dropna(inplace=True)
pred = model.predict(X_test)
 lookid_data = pd.read_csv('.. /input/IdLookupTable.csv')
 lookid_list = list(lookid_data['FeatureName'])
 imageID = list(lookid_data['ImageId']-1)
 pre_list = list(pred)
 rowid = lookid_data['RowId']
rowid=list(rowid)
 feature = []
for f in list(lookid_data['FeatureName']):
feature.append(lookid_list.index(f))
preded = []
for x,y in zip(imageID,feature):
preded.append(pre_list[x][y])
 rowid = pd.Series(rowid,name = 'RowId')
 loc = pd.Series(preded,name = 'Location')
 submission = pd.concat([rowid,loc],axis = 1)
 submission.to_csv('submision.csv',index = False )<load_from_csv>
le=LabelEncoder()
train_df['cp_le']=le.fit_transform(train_df['cp'])
 test_df['cp_le']=le.transform(test_df['cp'])
 train_df.drop(['cp'], axis=1, inplace=True)
 test_df.drop(['cp'], axis=1, inplace=True )
df=pd.read_csv('submision.csv' )<set_options>
def create_date_feat(data, cf, ft):
for d in data['date'].drop_duplicates() :
for i in data['cp_le'].drop_duplicates() :
org_mask =(data['date']==d)&(data['cp_le']==i)
 for lag in range(1,15):
mask_loc =(data['date']==(d-pd.Timedelta(days=lag)))&(data['cp_le']==i)
 try:
data.loc[org_mask, 'cf_' + str(lag)]=data.loc[mask_loc, cf].values
data.loc[org_mask, 'ft_' + str(lag)]=data.loc[mask_loc, ft].values
except:
data.loc[org_mask, 'cf_' + str(lag)]=0.0
data.loc[org_mask, 'ft_' + str(lag)]=0.0
create_date_feat(train_df,'confirmedcases','fatalities' )
pd.set_option('max_columns', None)
 plt.style.use('fivethirtyeight')
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 print("Libraries imported!" )<train_model>
def rmsle(pred,true):
assert pred.shape[0]==true.shape[0]
return K.sqrt(K.mean(K.square(K.log(pred+1)- K.log(true+1))))
es = EarlyStopping(monitor='val_loss', min_delta = 0, verbose=0, patience=10, mode='auto')
 mc_cf = ModelCheckpoint('model_cf.h5', monitor='val_loss', verbose=0, save_best_only=True)
 mc_ft = ModelCheckpoint('model_ft.h5', monitor='val_loss', verbose=0, save_best_only=True)
 def lstm_model(hidden_nodes, second_dim, third_dim):
model = Sequential([LSTM(hidden_nodes, input_shape=(second_dim, third_dim), activation='relu'),
Dense(64, activation='relu'),
Dense(32, activation='relu'),
Dense(1, activation='relu')])
 model.compile(loss=rmsle, optimizer = 'adam')
 return model
model_cf = lstm_model(10, tr_x_cf.shape[1], tr_x_cf.shape[2])
 model_ft = lstm_model(10, tr_x_ft.shape[1], tr_x_ft.shape[2])
 history_cf = model_cf.fit(tr_x_cf, tr_y_cf, epochs=200, batch_size=512, validation_data=(val_x_cf,val_y_cf), callbacks=[es,mc_cf])
 history_ft = model_ft.fit(tr_x_ft, tr_y_ft, epochs=200, batch_size=512, validation_data=(val_x_ft,val_y_ft), callbacks=[es,mc_ft] )
class BaseModel(object):

 def __init__(self, train_df, test_df, target, features, categoricals=[],
n_splits=3, cv_method="KFold", group=None, task="regression",
parameter_tuning=False, scaler=None, verbose=True):
self.train_df = train_df
self.test_df = test_df
self.target = target
self.features = features
self.n_splits = n_splits
self.categoricals = categoricals
self.cv_method = cv_method
self.group = group
self.task = task
self.parameter_tuning = parameter_tuning
self.scaler = scaler
self.cv = self.get_cv()
self.verbose = verbose
self.params = self.get_params()
self.y_pred, self.score, self.model, self.oof, self.y_val, self.fi_df = self.fit()
def train_model(self, train_set, val_set):
raise NotImplementedError
def get_params(self):
raise NotImplementedError
def convert_dataset(self, x_train, y_train, x_val, y_val):
raise NotImplementedError
def convert_x(self, x):
return x
def calc_metric(self, y_true, y_pred):
if self.task == "classification":
return log_loss(y_true, y_pred)
 elif self.task == "regression":
return np.sqrt(mean_squared_error(y_true, y_pred))
def get_cv(self):
if self.cv_method == "KFold":
cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df)
 elif self.cv_method == "StratifiedKFold":
cv = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target])
 elif self.cv_method == "TimeSeriesSplit":
cv = TimeSeriesSplit(max_train_size=None, n_splits=self.n_splits)
 return cv.split(self.train_df)
 elif self.cv_method == "GroupKFold":
cv = GroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target], self.group)
 elif self.cv_method == "StratifiedGroupKFold":
cv = StratifiedGroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target], self.group)
 def fit(self):
oof_pred = np.zeros(( self.train_df.shape[0],))
y_vals = np.zeros(( self.train_df.shape[0],))
y_pred = np.zeros(( self.test_df.shape[0],))
if self.group is not None:
if self.group in self.features:
self.features.remove(self.group)
 if self.group in self.categoricals:
self.categoricals.remove(self.group)
 fi = np.zeros(( self.n_splits, len(self.features)))
 if self.scaler is not None:
numerical_features = [f for f in self.features if f not in self.categoricals]
self.train_df[numerical_features] = self.train_df[numerical_features].fillna(self.train_df[numerical_features].median())
 self.test_df[numerical_features] = self.test_df[numerical_features].fillna(self.test_df[numerical_features].median())
 self.train_df[self.categoricals] = self.train_df[self.categoricals].fillna(self.train_df[self.categoricals].mode().iloc[0])
 self.test_df[self.categoricals] = self.test_df[self.categoricals].fillna(self.test_df[self.categoricals].mode().iloc[0])
 if self.scaler == "MinMax":
scaler = MinMaxScaler()
elif self.scaler == "Standard":
scaler = StandardScaler()
df = pd.concat([self.train_df[numerical_features], self.test_df[numerical_features]], ignore_index=True)
 scaler.fit(df[numerical_features])
 x_test = self.test_df.copy()
x_test[numerical_features] = scaler.transform(x_test[numerical_features])
 x_test = [np.absolute(x_test[i])for i in self.categoricals] + [x_test[numerical_features]]
else:
x_test = self.test_df[self.features]
for fold,(train_idx, val_idx)in enumerate(self.cv):
x_train, x_val = self.train_df.loc[train_idx, self.features], self.train_df.loc[val_idx, self.features]
y_train, y_val = self.train_df.loc[train_idx, self.target], self.train_df.loc[val_idx, self.target]
if self.scaler is not None:
x_train[numerical_features] = scaler.transform(x_train[numerical_features])
 x_val[numerical_features] = scaler.transform(x_val[numerical_features])
 x_train = [np.absolute(x_train[i])for i in self.categoricals] + [x_train[numerical_features]]
x_val = [np.absolute(x_val[i])for i in self.categoricals] + [x_val[numerical_features]]
train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)
 model, importance = self.train_model(train_set, val_set)
 fi[fold, :] = importance
conv_x_val = self.convert_x(x_val)
 y_vals[val_idx] = y_val
oof_pred[val_idx] = model.predict(conv_x_val ).reshape(oof_pred[val_idx].shape)
 x_test = self.convert_x(x_test)
 y_pred += model.predict(x_test ).reshape(y_pred.shape)/ self.n_splits
print('Partial score of fold {} is: {}'.format(fold, self.calc_metric(y_val, oof_pred[val_idx])))
 fi_df = pd.DataFrame()
for n in np.arange(self.n_splits):
tmp = pd.DataFrame()
tmp["features"] = self.features
tmp["importance"] = fi[n, :]
tmp["fold"] = n
fi_df = pd.concat([fi_df, tmp], ignore_index=True)
 gfi = fi_df[["features", "importance"]].groupby(["features"] ).mean().reset_index()
fi_df = fi_df.merge(gfi, on="features", how="left", suffixes=('', '_mean'))
loss_score = self.calc_metric(self.train_df[self.target], oof_pred)
 if self.verbose:
print('Our oof loss score is: ', loss_score)
 return y_pred, loss_score, model, oof_pred, y_vals, fi_df
def plot_feature_importance(self, rank_range=[1, 50]):
fig, ax = plt.subplots(1, 1, figsize=(10, 20))
sorted_df = self.fi_df.sort_values(by = "importance_mean", ascending=False ).reset_index().iloc[self.n_splits *(rank_range[0]-1): self.n_splits * rank_range[1]]
sns.barplot(data=sorted_df, x ="importance", y ="features", orient='h')
 ax.set_xlabel("feature importance")
 ax.spines['top'].set_visible(False)
 ax.spines['right'].set_visible(False)
 return sorted_df<train_model>
feat = ['confirmedcases','fatalities','cf_1', 'ft_1', 'cf_2', 'ft_2', 'cf_3', 'ft_3',
'cf_4', 'ft_4', 'cf_5', 'ft_5', 'cf_6', 'ft_6', 'cf_7', 'ft_7', 'cf_8', 'ft_8',
'cf_9', 'ft_9', 'cf_10', 'ft_10', 'cf_11', 'ft_11', 'cf_12', 'ft_12', 'cf_13', 'ft_13',
'cf_14', 'ft_14']
c_feat = ['cp_le', 'weekofyear','quarter','month','dayofyear','cf_1', 'cf_2', 'cf_3',
'cf_4', 'cf_5', 'cf_6', 'cf_7', 'cf_8', 'cf_9','cf_10', 'cf_11', 'cf_12',
'cf_13', 'cf_14']
f_feat = ['cp_le', 'weekofyear','quarter','month','dayofyear','ft_1', 'ft_2', 'ft_3',
'ft_4', 'ft_5', 'ft_6', 'ft_7', 'ft_8', 'ft_9','ft_10', 'ft_11', 'ft_12',
'ft_13', 'ft_14']
tot_feat = ['cp_le', 'weekofyear','quarter','month','dayofyear','cf_1', 'ft_1', 'cf_2', 'ft_2', 'cf_3', 'ft_3',
'cf_4', 'ft_4', 'cf_5', 'ft_5', 'cf_6', 'ft_6', 'cf_7', 'ft_7', 'cf_8', 'ft_8',
'cf_9', 'ft_9', 'cf_10', 'ft_10', 'cf_11', 'ft_11', 'cf_12', 'ft_12', 'cf_13', 'ft_13',
'cf_14', 'ft_14']
test_new = test_df.copy().join(pd.DataFrame(columns=feat))
test_mask =(test_df['date'] <= train_df['date'].max())
 train_mask =(train_df['date'] >= test_df['date'].min())
 test_new.loc[test_mask,feat] = train_df.loc[train_mask, feat].values
future_df = pd.date_range(start = train_df['date'].max() +pd.Timedelta(days=1),end=test_df['date'].max() , freq='1D')
 def create_add_trend_pred(data, cf, ft):
for d in future_df:
for i in data['cp_le'].drop_duplicates() :
org_mask =(data['date']==d)&(data['cp_le']==i)
 for lag in range(1,15):
mask_loc =(data['date']==(d-pd.Timedelta(days=lag)))&(data['cp_le']==i)
 try:
data.loc[org_mask, 'cf_' + str(lag)]=data.loc[mask_loc,cf].values
data.loc[org_mask, 'ft_' + str(lag)]=data.loc[mask_loc,ft].values
except:
data.loc[org_mask, 'cf_' + str(lag)]=0.0
data.loc[org_mask, 'ft_' + str(lag)]=0.0
test_x = data.loc[org_mask,tot_feat]
test_x_cf = test_x[c_feat]
test_x_cf = test_x_cf.to_numpy().reshape(1,-1)
 test_x_cf_reshape = test_x_cf.reshape(test_x_cf.shape[0],1,test_x_cf.shape[1])
 test_x_ft = test_x[f_feat]
test_x_ft = test_x_ft.to_numpy().reshape(1,-1)
 test_x_ft_reshape = test_x_ft.reshape(test_x_ft.shape[0],1,test_x_ft.shape[1])
 data.loc[org_mask, cf] = model_cf.predict(test_x_cf_reshape)
 data.loc[org_mask, ft] = model_ft.predict(test_x_ft_reshape)
 create_add_trend_pred(test_new, 'confirmedcases', 'fatalities' )
class LgbModel(BaseModel):

 def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
model = lgb.train(self.params, train_set, num_boost_round = 5000, valid_sets=[train_set, val_set], verbose_eval=verbosity)
 fi = model.feature_importance(importance_type="gain")
 return model, fi
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)
 val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)
 return train_set, val_set
def get_params(self):
params = {
'num_leaves': 127,
'min_data_in_leaf': 50,
'max_depth': -1,
'learning_rate': 0.005,
"boosting_type": "gbdt",
"bagging_seed": 11,
"verbosity": -1,
'random_state': 42,
}
if self.task == "regression":
params["objective"] = "regression"
params["metric"] = "rmse"
elif self.task == "classification":
params["objective"] = "binary"
params["metric"] = "binary_logloss"
if self.parameter_tuning == True:
def objective(trial):
train_x, test_x, train_y, test_y = train_test_split(self.train_df[self.features],
self.train_df[self.target],
test_size=0.3, random_state=42)
 dtrain = lgb.Dataset(train_x, train_y, categorical_feature=self.categoricals)
 dtest = lgb.Dataset(test_x, test_y, categorical_feature=self.categoricals)
 hyperparams = {'num_leaves': trial.suggest_int('num_leaves', 24, 1024),
'boosting_type': 'gbdt',
'objective': params["objective"],
'metric': params["metric"],
'max_depth': trial.suggest_int('max_depth', 4, 16),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
'early_stopping_rounds': 100
}
model = lgb.train(hyperparams, dtrain, valid_sets=dtest, verbose_eval=500)
 pred = model.predict(test_x)
 if self.task == "classification":
return log_loss(test_y, pred)
 elif self.task == "regression":
return np.sqrt(mean_squared_error(test_y, pred))
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials: {}'.format(len(study.trials)))
 print('Best trial:')
 trial = study.best_trial
print(' Value: {}'.format(trial.value))
print(' Params: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))
params = trial.params
params["learning_rate"] = 0.001
plot_optimization_history(study)
 return params<load_from_csv>
sub_pred = pd.DataFrame({'ForecastId': test_id, 'ConfirmedCases':test_new['confirmedcases'],'Fatalities':test_new['fatalities']})
 sub_pred.to_csv('submission.csv', index=False )