Kaggle competition: COVID19 Global Forecasting (Week 1)
X_train,X_val,y_train,y_val = train_test_split(X,y,test_size = 0.1 )<categorify>
test = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv")
 train['Province/State'].fillna('', inplace=True)
 test['Province/State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country/Region','Province/State','Date'])
 test = test.sort_values(['Country/Region','Province/State','Date'] )
y_train = to_categorical(y_train)
 y_val = to_categorical(y_val )<choose_model_class>
train.groupby(['Country/Region'] ).max().sort_values(['ConfirmedCases'],ascending = False ).head(10 )
def DL_Model(filter1_size=64, filter2_size=32 , activation='relu', optimizer='Adam', padding='Same'):
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv2D(filter1_size,(3,3),padding = padding,activation = activation,
input_shape =(28,28,1)))
 model.add(tf.keras.layers.Conv2D(filter1_size,(3,3),padding = padding,activation=activation))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.MaxPool2D(pool_size=(2,2)))
 model.add(tf.keras.layers.Conv2D(filter2_size,(3,3),padding = padding,activation=activation))
model.add(tf.keras.layers.Conv2D(filter2_size,(3,3),padding = padding,activation=activation))
model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.MaxPool2D(pool_size=(2,2)))
 model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(128,activation=activation))
model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Dense(256,activation=activation))
model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Dense(10,activation='softmax'))
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"])
 return model<init_hyperparams>
!pip install pmdarima
filter1_size = [64]
filter2_size = [64]
activation = ['relu','tanh','sigmoid']
padding = ['Same']
optimizer = ['Adam','RMSprop']<choose_model_class>
%matplotlib inline
register_matplotlib_converters()
country_plot = 'Vietnam'
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 def RMSLE_Cal(pred_data):
df = pd.DataFrame(pred_data)
 df.columns = ['Date','Country/Region','Province/State','ForecastId','ConfirmedCases','Fatalities']
df_val = pd.merge(df,train[['Date','Country/Region','Province/State','ConfirmedCases','Fatalities']],on=['Date','Country/Region','Province/State'], how='left')
 df_val = df_val.sort_values('Date')
 val_size = df_val[df_val['ConfirmedCases_y'].notnull() == True].shape[0]
return RMSLE(df_val['ConfirmedCases_y'][0:val_size-1].values, df_val['ConfirmedCases_x'][0:val_size-1].values)
 def plotPrediction(pred_data):
df = pd.DataFrame(pred_data)
 df.columns = ['Date','Country/Region','Province/State','ForecastId','ConfirmedCases','Fatalities']
df = df[df['Country/Region'] == country_plot]
df_val = pd.merge(df,train[['Date','Country/Region','Province/State','ConfirmedCases','Fatalities']],on=['Date','Country/Region','Province/State'], how='left')
 df_val = df_val.sort_values('Date')
 fig, axes = plt.subplots(nrows=1,ncols=2,figsize=(20, 7))
axes[0].plot(df_val['Date'], df_val['ConfirmedCases_y'], label = 'Confirmed Cases')
 axes[0].plot(df_val['Date'], df_val['ConfirmedCases_x'], label = 'Confirmed Cases Forecast')
 axes[1].plot(df_val['Date'], df_val['Fatalities_y'], label = 'Fatalities')
 axes[1].plot(df_val['Date'], df_val['Fatalities_x'], label = 'Fatalities Forecast')
 axes[0].legend()
axes[1].legend()
fig.autofmt_xdate()
return RMSLE(df_val['ConfirmedCases_y'][0:val_size-1].values, df_val['ConfirmedCases_x'][0:val_size-1].values)
model=DL_Model(64,64,'relu','Adam','Same' )<train_model>
df_past = pd.merge(test,train,on = ['Province/State','Date','Country/Region','Lat','Long'], how = 'inner' ).drop(['Id','Lat','Long'],axis=1)
 df_future = pd.DataFrame(pred_data)
 df_future.columns = ['Date','Country/Region','Province/State','ForecastId','ConfirmedCases','Fatalities']
df_future = df_future[(df_future['Date'] > df_past['Date'].max())]
submission = df_past.append(df_future, sort = True)
 submission[['ForecastId','ConfirmedCases','Fatalities']].to_csv('submission.csv',index=False )
epochs = 40
batch_size = 128
model.fit(X_train,y_train,
validation_data=(X_val,y_val),
epochs=epochs,
batch_size=batch_size )<compute_test_metric>
df_train = train.copy()
df_test = test.copy()
train_period = 10
feature_Confirmed = ['ConfirmedCases'] + ['ConfirmedCases_B' + str(i)for i in range(1,train_period)]
feature_Fatalities = ['Fatalities'] + ['Fatalities_B' + str(i)for i in range(1,train_period)]
pred_data = []
model = DecisionTreeRegressor(max_depth=5)
 for country in [country_plot,'Vietnam']:
for province in df_train[(df_train['Country/Region'] == country)]['Province/State'].unique() :
max_train_date = df_train[(df_train['Country/Region'] == country)&(df_train['Province/State'] == province)]['Date'].max()
for forcast in df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)]['ForecastId'].unique() :
testdate = df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)&(df_test['ForecastId'] == forcast)]['Date'].max()
data = df_train[(df_train['Country/Region'] == country)&(df_train['Province/State'] == province)&(df_train['ConfirmedCases'] > 0)&(df_train['Date']<testdate)][['Date','ConfirmedCases','Fatalities']]
for i in range(1,train_period):
data['ConfirmedCases_B' + str(i)] = data['ConfirmedCases'].shift(i)
 data['Fatalities_B' + str(i)] = data['Fatalities'].shift(i)
 data = data[(data.ConfirmedCases_B1 != data.ConfirmedCases)&(data.ConfirmedCases_B2 != data.ConfirmedCases)]
X_train_confirmed = data[feature_Confirmed][train_period-1:-1]
y_train_confirmed = data['ConfirmedCases'][train_period-1:-1]
feature_Confirmed = ['ConfirmedCases'] + ['ConfirmedCases_B' + str(i)for i in range(1,train_period)]
if(len(X_train_confirmed)< 2):
ConfirmedCases_hat = 0
elif(len(X_train_confirmed)< train_period):
ConfirmedCases_hat = X_train_confirmed['ConfirmedCases'].values[-1:][0]
else:
model.fit(X_train_confirmed,y_train_confirmed)
 X_pred_confirmed = data[feature_Confirmed][-1:]
ConfirmedCases_hat = model.predict(X_pred_confirmed)[-1]
data = data[(data.Fatalities_B1 != data.Fatalities)&(data.Fatalities_B2 != data.Fatalities)]
X_train_fatalities = data[feature_Fatalities][train_period-1:-1]
y_train_fatalities = data['Fatalities'][train_period-1:-1]
feature_Fatalities = ['Fatalities'] + ['Fatalities_B' + str(i)for i in range(1,train_period)]
if(len(X_train_fatalities)< 1):
Fatalities_hat = 0
elif(len(X_train_fatalities)< train_period):
Fatalities_hat = X_train_fatalities['Fatalities'].values[-1:][0]
else:
model.fit(X_train_fatalities,y_train_fatalities)
 X_pred_fatalities = data[feature_Fatalities][-1:]
Fatalities_hat = model.predict(X_pred_fatalities)[-1]
pred_data.append([testdate,country,province,forcast,ConfirmedCases_hat,Fatalities_hat])
 if(testdate > max_train_date):
lat = df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)&(df_test['ForecastId'] == forcast)]['Lat'].max()
long = df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)&(df_test['ForecastId'] == forcast)]['Long'].max()
df_train.loc[len(df_train)] = [int(forcast),province,country,lat,long,testdate,float(ConfirmedCases_hat),float(Fatalities_hat)]
model.evaluate(X_val,y_val )<choose_model_class>
def CreateInput(data):
feature = []
for day in [1,100,200,500,1000]:
data['Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country/Region'] == country)&(train['Province/State'] == province)&(train['ConfirmedCases'] > day)]['Date'].count() > 0):
fromday = train[(train['Country/Region'] == country)&(train['Province/State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = pd.Timestamp('2020-12-31')
 for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
model = xgb.XGBRegressor()
pred_data = []
for country in ['Vietnam']:
for province in train[(train['Country/Region'] == country)]['Province/State'].unique() :
df_train = train[(train['Country/Region'] == country)&(train['Province/State'] == province)]
df_test = test[(test['Country/Region'] == country)&(test['Province/State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases']
y_train_fatalities = df_train['Fatalities']
X_pred = CreateInput(df_test)
 if(y_train_confirmed[-5:].mean() == y_train_confirmed.iloc[-1]):
y_hat_confirmed = np.repeat(y_train_confirmed.iloc[-1], len(X_pred))
else:
model.fit(X_train,y_train_confirmed)
 y_hat_confirmed = model.predict(X_pred)
 if(y_train_fatalities[-5:].mean() == y_train_fatalities.iloc[-1]):
y_hat_fatalities = np.repeat(y_train_fatalities.iloc[-1], len(X_pred))
else:
model.fit(X_train,y_train_fatalities)
 y_hat_fatalities = model.predict(X_pred)
 for i in range(0,len(y_hat_confirmed)) :
pred_data.append([df_test['Date'].iloc[i],country,province,df_test['ForecastId'].iloc[i],y_hat_confirmed[i],y_hat_fatalities[i]] )
model=DL_Model(64,64,'relu','Adam','Same' )<choose_model_class>
train_period = 6
feature_Confirmed = ['ConfirmedCases'] + ['ConfirmedCases_B' + str(i)for i in range(1,train_period)]
feature_Fatalities = ['Fatalities'] + ['Fatalities_B' + str(i)for i in range(1,train_period)]
model = tf.keras.Sequential([
tf.keras.layers.Dense(256, activation='relu', input_shape=[len(feature_Confirmed)]),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(1)
 ])
 optimizer = "rmsprop"
model.compile(loss='mse', optimizer=optimizer, metrics=['mse','mae'])
 EPOCHS = 10
BATCH_SIZE = 1
df_train = train.copy()
df_test = test.copy()
pred_data = []
for country in [country_plot,'Vietnam']:
for province in df_train[(df_train['Country/Region'] == country)]['Province/State'].unique() :
max_train_date = df_train[(df_train['Country/Region'] == country)&(df_train['Province/State'] == province)]['Date'].max()
for forcast in df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)]['ForecastId'].unique() :
testdate = df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)&(df_test['ForecastId'] == forcast)]['Date'].max()
data = df_train[(df_train['Country/Region'] == country)&(df_train['Province/State'] == province)&(df_train['ConfirmedCases'] > 0)&(df_train['Date']<testdate)][['Date','ConfirmedCases','Fatalities']]
for i in range(1,train_period):
data['ConfirmedCases_B' + str(i)] = data['ConfirmedCases'].shift(i)
 data['Fatalities_B' + str(i)] = data['Fatalities'].shift(i)
 data = data[(data.ConfirmedCases_B1 != data.ConfirmedCases)&(data.ConfirmedCases_B2 != data.ConfirmedCases)]
X_train_confirmed = data[feature_Confirmed][train_period-1:-1]
y_train_confirmed = data['ConfirmedCases'][train_period-1:-1]
if(len(X_train_confirmed)< 1):
ConfirmedCases_hat = 0
elif(len(X_train_confirmed)< train_period):
ConfirmedCases_hat = X_train_confirmed['ConfirmedCases'].values[-1:][0]
else:
model.fit(X_train_confirmed, y_train_confirmed ,batch_size=BATCH_SIZE, epochs=EPOCHS, verbose = 0)
 X_pred_confirmed = data[feature_Confirmed][-1:]
ConfirmedCases_hat = model.predict(X_pred_confirmed)[-1][0]
data = data[(data.Fatalities_B1 != data.Fatalities)&(data.Fatalities_B2 != data.Fatalities)]
X_train_fatalities = data[feature_Fatalities][train_period-1:-1]
y_train_fatalities = data['Fatalities'][train_period-1:-1]
if(len(X_train_fatalities)< 1):
Fatalities_hat = 0
elif(len(X_train_fatalities)< train_period):
Fatalities_hat = X_train_fatalities['Fatalities'].values[-1:][0]
else:
model.fit(X_train_fatalities, y_train_fatalities ,batch_size=BATCH_SIZE, epochs=EPOCHS, verbose = 0)
 X_pred_fatalities = data[feature_Fatalities][-1:]
Fatalities_hat = model.predict(X_pred_fatalities)[-1][0]
pred_data.append([testdate,country,province,forcast,ConfirmedCases_hat,Fatalities_hat])
 if(testdate > max_train_date):
lat = df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)&(df_test['ForecastId'] == forcast)]['Lat'].max()
long = df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)&(df_test['ForecastId'] == forcast)]['Long'].max()
df_train.loc[len(df_train)] = [int(forcast),province,country,lat,long,testdate,float(ConfirmedCases_hat),float(Fatalities_hat)]
early_stopping_monitor = tf.keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0,
patience=6,
verbose=0,
mode='auto',
baseline=None,
restore_best_weights=True
)<choose_model_class>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data['Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country/Region'] == country)&(train['Province/State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country/Region'] == country)&(train['Province/State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = pd.Timestamp('2020-12-31')
 for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
model = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='elu', input_shape=[len(feature_day)]),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(32, activation='elu'),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(1)
 ])
 optimizer = "rmsprop"
model.compile(loss='mse', optimizer=optimizer, metrics=['mse','mae'])
 EPOCHS = 10
BATCH_SIZE = 1
pred_data = []
for country in ['Vietnam']:
for province in train[(train['Country/Region'] == country)]['Province/State'].unique() :
df_train = train[(train['Country/Region'] == country)&(train['Province/State'] == province)]
df_test = test[(test['Country/Region'] == country)&(test['Province/State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases']
y_train_fatalities = df_train['Fatalities']
X_pred = CreateInput(df_test)
 if(y_train_confirmed[-5:].mean() == y_train_confirmed.iloc[-1]):
y_hat_confirmed = np.repeat(y_train_confirmed.iloc[-1], len(X_pred))
else:
model.fit(X_train,y_train_confirmed,batch_size=BATCH_SIZE, epochs=EPOCHS, verbose = 0)
 y_hat = model.predict(X_pred)
 y_hat_confirmed = [item[0] for item in y_hat]
if(y_train_fatalities[-5:].mean() == y_train_fatalities.iloc[-1]):
y_hat_fatalities = np.repeat(y_train_fatalities.iloc[-1], len(X_pred))
else:
model.fit(X_train,y_train_fatalities,batch_size=BATCH_SIZE, epochs=EPOCHS, verbose = 0)
 y_hat = model.predict(X_pred)
 y_hat_fatalities = [item[0] for item in y_hat]
for i in range(0,len(y_hat_confirmed)) :
pred_data.append([df_test['Date'].iloc[i],country,province,df_test['ForecastId'].iloc[i],y_hat_confirmed[i],y_hat_fatalities[i]] )
learning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<define_variables>
def CreateInput(data):
feature_day = [5,20,50,100,200,500,1000]
feature_name = ['Lat','Long']
for day in feature_day:
data['Number day from ' + str(day)+ ' case'] = 0
for country in data['Country/Region'].unique() :
for province in data[data['Country/Region'] == country]['Province/State'].unique() :
fromday = train[(train['Country/Region'] == country)&(train['Province/State'] == province)&(train['ConfirmedCases'] > day)]['Date'].min()
if(pd.isnull(fromday)== False):
data.loc[(data['Country/Region'] == country)&(data['Province/State'] == province),'Number day from ' + str(day)+ ' case'] =(data[(data['Country/Region'] == country)&(data['Province/State'] == province)]['Date'] - fromday ).dt.days
data.loc[(data['Number day from ' + str(day)+ ' case'] < 0),'Number day from ' + str(day)+ ' case'] = 0
feature_name = feature_name + ['Number day from ' + str(day)+ ' case']
return data[feature_name]
X_train = CreateInput(train.copy())
 y_train = train[['ConfirmedCases','Fatalities']]
model = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='relu', input_shape=[X_train.shape[1]]),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(32, activation='relu'),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(2)
 ])
 optimizer = "rmsprop"
model.compile(loss='mse', optimizer=optimizer, metrics=['mse','mae'])
 EPOCHS = 100
BATCH_SIZE = 24
model.fit(X_train,y_train, batch_size=BATCH_SIZE, epochs=EPOCHS,verbose = 0)
 X_pred = CreateInput(test.copy())
 y_hat = model.predict(X_pred)
epochs = 30
batch_size = 86<init_hyperparams>
df_forcast = pd.concat([test[['Date','ForecastId']],pd.DataFrame(y_hat)],axis = 1)
 df_forcast.columns = ['Date','ForecastId','ConfirmedCases','Fatalities']
df_forcast.loc[df_forcast['Fatalities'] < 0,'Fatalities'] = 0
df_forcast.loc[df_forcast['ConfirmedCases'] < 0,'ConfirmedCases'] = 0
df_past = pd.merge(test,train,on = ['Province/State','Date','Country/Region','Lat','Long'], how = 'inner' ).drop(['Id','Lat','Long'],axis=1)
 df_future = df_forcast[(df_forcast['Date'] > df_past['Date'].max())]
submission = df_past.append(df_future, sort = True)
 submission[['ForecastId','ConfirmedCases','Fatalities']]
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
df_val = pd.merge(test,df_forcast[['ForecastId','ConfirmedCases','Fatalities']], how='inner', on = 'ForecastId')
 df_val = pd.merge(df_val,train,on=['Date','Country/Region','Province/State','Lat','Long'], how='left')
 df_val = df_val.sort_values(by = ['Date','Country/Region'])
 df = df_val[df_val['Country/Region'] == 'Vietnam']
val_size = df_val[df_val['ConfirmedCases_y'].notnull() == True].shape[0]
fig, axes = plt.subplots(nrows=1,ncols=2,figsize=(20, 7))
axes[0].plot(df['Date'], df['ConfirmedCases_x'], label = 'ConfirmedCase forcast ' + df['Date'].max().strftime('%Y-%m-%d'))
axes[0].plot(df['Date'][0:val_size], df['ConfirmedCases_y'][0:val_size], label = 'ConfirmedCase ' + df[df['ConfirmedCases_y'].isnull() == False]['Date'].max().strftime('%Y-%m-%d'))
axes[0].legend()
axes[1].plot(df['Date'], df['Fatalities_x'], label = 'Fatalities forcast ' + df['Date'].max().strftime('%Y-%m-%d'))
axes[1].plot(df['Date'][0:val_size], df['Fatalities_y'][0:val_size], label = 'Fatalities ' + df[df['Fatalities_y'].isnull() == False]['Date'].max().strftime('%Y-%m-%d'))
axes[1].legend()
fig.autofmt_xdate()
history = model.fit(datagen.flow(X_train,y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<compute_test_metric>
RMSLE(df_val['ConfirmedCases_x'][0:val_size].values,df_val['ConfirmedCases_y'][0:val_size].values )