Kaggle competition: COVID19 Global Forecasting (Week 3)
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<load_from_csv>
data= pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/test.csv" )
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<feature_engineering>
data['Province_State']=data['Province_State'].fillna('')
 test['Province_State']=test['Province_State'].fillna('' )
FirstDate = train.groupby('Country_Region' ).min() ['Date'].unique() [0]
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 while train[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate)].shape[0] > 0:
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1)
 train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'ConfirmedCases'] = train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'Last Confirm']
train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Fatalities'] = train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Last Fatalities']
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1 )<feature_engineering>
import matplotlib.pyplot as plt
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
adjusted_X_pred = X_pred[feature_use].values.reshape(-1, 1)
 model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(adjusted_X_train,adjusted_y_train_confirmed)
 y_hat_confirmed = model.predict(adjusted_X_pred)
 model.fit(adjusted_X_train,adjusted_y_train_fatalities)
 y_hat_fatalities = model.predict(adjusted_X_pred)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_1 = df_val.copy()<compute_test_metric>
datetime_str = '01/22/20 00:00:00'
datetime_object = datetime.strptime(datetime_str, '%m/%d/%y %H:%M:%S')
 data['days']=pd.to_datetime(data['Date'] ).sub(datetime_object)/np.timedelta64(1, 'D')
 test['days']=pd.to_datetime(test['Date'] ).sub(datetime_object)/np.timedelta64(1, 'D' )
RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values )<compute_test_metric>
data.loc[(data['Province_State']=='')&(data['Country_Region']=='India'),:].sort_values(by="Date" )
RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values )<groupby>
from statsmodels.tsa.arima_model import ARIMA
country = "India"
df_val = df_val_1
df_val[df_val['Country_Region'] == country].groupby(['Date','Country_Region'] ).sum().reset_index()<feature_engineering>
data['Date']=pd.to_datetime(data['Date'])
 test['Date']=pd.to_datetime(test['Date'] )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = ExponentialSmoothing(adjusted_y_train_confirmed, trend = 'additive' ).fit()
y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = ExponentialSmoothing(adjusted_y_train_fatalities, trend = 'additive' ).fit()
y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_2 = df_val.copy()<feature_engineering>
pd.DataFrame(data.loc[data['Country_Region']=='Afghanistan',['ConfirmedCases']] ).reset_index(drop=True )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()<compute_test_metric>
data.isna().sum(axis=0 )
method_list = ['Poly Bayesian Ridge','Exponential Smoothing','SARIMA']
method_val = [df_val_1,df_val_2,df_val_3]
for i in range(0,3):
df_val = method_val[i]
method_score = [method_list[i]] + [RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values)] + [RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values)]
print(method_score )<save_to_csv>
data['ConfirmedCases'][data['Country_Region']==''][51:]
df_val = df_val_3
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False)
 submission<load_from_csv>
data['ConfirmedCases'][data['Country_Region']=='India'].value_counts()
path = '.. /input/covid19-global-forecasting-week-2/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'submission.csv')
 train['Date'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 test['Date'] = test['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 train['days'] =(train['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
test['days'] =(test['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
train.loc[train['Province_State'].isnull() , 'Province_State'] = 'N/A'
test.loc[test['Province_State'].isnull() , 'Province_State'] = 'N/A'
train['Area'] = train['Country_Region'] + '_' + train['Province_State']
test['Area'] = test['Country_Region'] + '_' + test['Province_State']
print(train['Date'].max())
 print(test['Date'].min())
 print(train['days'].max())
 N_AREAS = train['Area'].nunique()
AREAS = np.sort(train['Area'].unique())
 TRAIN_N = 70
print(train[train['days'] < TRAIN_N]['Date'].max())
 train.head()<prepare_x_and_y>
pd.DataFrame(data.loc[data['Country_Region']=='India',['ConfirmedCases']] )
train_p_c_raw = train.pivot(index='Area', columns='days', values='ConfirmedCases' ).sort_index()
train_p_f_raw = train.pivot(index='Area', columns='days', values='Fatalities' ).sort_index()
train_p_c = np.maximum.accumulate(train_p_c_raw, axis=1)
 train_p_f = np.maximum.accumulate(train_p_f_raw, axis=1)
 f_rate =(train_p_f / train_p_c ).fillna(0)
 X_c = np.log(1+train_p_c.values)[:,:TRAIN_N]
X_f = train_p_f.values[:,:TRAIN_N] 
<compute_train_metric>
datetime_str = '03/22/20 00:00:00'
datetime_object = datetime.strptime(datetime_str, '%m/%d/%y %H:%M:%S' )
def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run_c(params, X, test_size=50):
gr_base = []
gr_base_factor = []
x_min = np.ma.MaskedArray(X, X<1)
 x_min = x_min.argmin(axis=1)
 for i in range(X.shape[0]):
temp = X[i,:]
threshold = np.log(1+params['min cases for growth rate'])
 num_days = params['last N days']
if(temp > threshold ).sum() > num_days:
gr_base.append(np.clip(np.diff(temp[temp > threshold])[-num_days:].mean() , 0, params['growth rate max']))
gr_base_factor.append(np.clip(np.diff(np.diff(temp[temp > threshold])) [-num_days:].mean() , -0.2, params["growth rate factor max"]))
else:
gr_base.append(params['growth rate default'])
 gr_base_factor.append(params['growth rate factor'])
 gr_base = np.array(gr_base)
 gr_base_factor = np.array(gr_base_factor)
 preds = X.copy()
for i in range(test_size):
delta = np.clip(preds[:, -1], np.log(2), None)+ gr_base *(1 + params['growth rate factor']*(1 + params['growth rate factor factor'])**(i)) **(i)
 preds = np.hstack(( preds, delta.reshape(-1,1)))
 return preds
params = {
"min cases for growth rate": 30,
"last N days": 5,
"growth rate default": 0.25,
"growth rate max": 0.3,
"growth rate factor max": -0.01,
"growth rate factor": -0.05,
"growth rate factor factor": 0.005,
}
x = train_p_c
preds_c = run_c(params, np.log(1+x.values)[:,:TRAIN_N])
<feature_engineering>
from datetime import timedelta
for i in range(N_AREAS):
if 'China' in AREAS[i] and preds_c[i, TRAIN_N-1] < np.log(31):
preds_c[i, TRAIN_N:] = preds_c[i, TRAIN_N-1] 
<normalization>
import math
def lin_w(sz):
res = np.linspace(0, 1, sz+1, endpoint=False)[1:]
return np.append(res, np.append([1], res[::-1]))
def run_f(params, X_c, X_f, X_f_r, test_size=50):
X_f_r = np.array(np.ma.mean(np.ma.masked_outside(X_f_r, 0.06, 0.4)[:,:], axis=1))
X_f_r = np.clip(X_f_r, params['fatality_rate_lower'], params['fatality_rate_upper'])
 X_c = np.clip(np.exp(X_c)-1, 0, None)
 preds = X_f.copy()
train_size = X_f.shape[1] - 1
for i in range(test_size):
t_lag = train_size+i-params['length']
t_wsize = 3
delta = np.average(np.diff(X_c, axis=1)[:, t_lag-t_wsize:t_lag+1+t_wsize], axis=1)
 delta = params['absolute growth'] + delta * X_f_r
preds = np.hstack(( preds, preds[:, -1].reshape(-1,1)+ delta.reshape(-1,1)))
 return preds
params = {
"length": 6,
"absolute growth": 0.02,
"fatality_rate_lower": 0.035,
"fatality_rate_upper": 0.40,
}
preds_f = run_f(params, preds_c, X_f, f_rate.values[:,:TRAIN_N])
 preds_f = np.log(1+preds_f)
<compute_test_metric>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, d]), preds_c[:, d]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, d]), preds_f[:, d]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c[:, TRAIN_N:TRAIN_N+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )<define_variables>
def evaluate_arima_model(X,forecast_days, arima_order):
X=[x for x in X]
train_size = int(len(X)* 0.9)
 train, test1 = X[0:train_size], X[train_size:]
history=train
model = ARIMA(history, order=arima_order)
 model_fit = model.fit(disp=0)
 predictions = list()
predictions =model_fit.forecast(steps=len(test1)) [0]
model = ARIMA(X, order=arima_order)
 model_fit = model.fit(disp=0)
 if np.isnan(model_fit.forecast(steps=forecast_days)[0] ).sum() >0:
return float('inf')
 error = rmsle(test1, predictions)
 return error
EU_COUNTRIES = ['Austria', 'Italy', 'Belgium', 'Latvia', 'Bulgaria', 'Lithuania', 'Croatia', 'Luxembourg', 'Cyprus', 'Malta', 'Czechia',
'Netherlands', 'Denmark', 'Poland', 'Estonia', 'Portugal', 'Finland', 'Romania', 'France', 'Slovakia', 'Germany', 'Slovenia',
'Greece', 'Spain', 'Hungary', 'Sweden', 'Ireland']
EUROPE_OTHER = ['Albania', 'Andorra', 'Bosnia and Herzegovina', 'Liechtenstein', 'Monaco', 'Montenegro', 'North Macedonia',
'Norway', 'San Marino', 'Serbia', 'Switzerland', 'Turkey', 'United Kingdom']
AFRICA = ['Algeria', 'Burkina Faso', 'Cameroon', 'Congo(Kinshasa)', "Cote d'Ivoire", 'Egypt', 'Ghana', 'Kenya', 'Madagascar',
'Morocco', 'Nigeria', 'Rwanda', 'Senegal', 'South Africa', 'Togo', 'Tunisia', 'Uganda', 'Zambia']
NORTH_AMERICA = ['US', 'Canada', 'Mexico']
SOUTH_AMERICA = ['Argentina', 'Bolivia', 'Brazil', 'Chile', 'Colombia', 'Ecuador', 'Paraguay', 'Peru', 'Uruguay', 'Venezuela']
MIDDLE_EAST = ['Afghanistan', 'Bahrain', 'Iran', 'Iraq', 'Israel', 'Jordan', 'Kuwait', 'Lebanon', 'Oman', 'Qatar', 'Saudi Arabia', 'United Arab Emirates']
ASIA = ['Bangladesh', 'Brunei', 'Cambodia', 'India', 'Indonesia', 'Japan', 'Kazakhstan', 'Korea, South', 'Kyrgyzstan', 'Malaysia',
'Pakistan', 'Singapore', 'Sri Lanka', 'Taiwan*', 'Thailand', 'Uzbekistan', 'Vietnam']<categorify>
def evaluate_models(dataset,forcast_days, p_values, d_values, q_values):
best_score, best_cfg = float("inf"),(0,0,0)
 for p in p_values:
for d in d_values:
for q in q_values:
order =(p,d,q)
 try:
mse = evaluate_arima_model(dataset,forcast_days, order)
 if mse < best_score:
best_score, best_cfg = mse, order
except:
continue
print('Best ARIMA%s MSE=%.3f' %(best_cfg, best_score))
return best_cfg, best_score
temp = pd.DataFrame(np.clip(np.exp(preds_c)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="ConfirmedCases")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 temp = pd.DataFrame(np.clip(np.exp(preds_f)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="Fatalities")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 test.head()<save_to_csv>
warnings.filterwarnings('ignore' )
test.to_csv("submission.csv", index=False, columns=["ForecastId", "ConfirmedCases", "Fatalities"] )<sort_values>
test['ConfirmedCases']=0
test['Fatalities']=0
for i, rec in test.groupby('Area' ).last().sort_values("ConfirmedCases", ascending=False ).iterrows() :
print(f"{rec['ConfirmedCases']:10.1f} {rec['Fatalities']:10.1f} {rec['Country_Region']}, {rec['Province_State']}")
<compute_test_metric>
sliced_data=data.loc[(data['Province_State']=='')&(data['Country_Region']=='India'),:]
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
def fix_target(frame, key, target, new_target_name="target"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if current_value<previous_value:
current_value=previous_value
target[i] =current_value
target[i] =max(0,target[i])
 frame[new_target_name] = np.array(target)
 def rate(frame, key, target, new_target_name="rate"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[1.0 for k in range(len(target)) ]
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if previous_value!=0.0:
rate[i]=current_value/previous_value
rate[i] =max(1,rate[i])
 frame[new_target_name] = np.array(rate)
 def get_data_by_key(dataframe, key, key_value, fields=None):
mini_frame=dataframe[dataframe[key]==key_value]
if not fields is None:
mini_frame=mini_frame[fields].values
return mini_frame
directory="/kaggle/input/covid19-global-forecasting-week-2/"
model_directory="/kaggle/input/model-dir/model"
train=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 test=pd.read_csv(directory + "test.csv", parse_dates=["Date"], engine="python")
 train["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 test["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 max_train_date=train["Date"].max()
max_test_date=test["Date"].max()
horizon=(max_test_date-max_train_date ).days
print("horizon", int(horizon))
target1="ConfirmedCases"
target2="Fatalities"
key="key"<count_unique_values>
from pandas import read_csv
from pandas import datetime
from matplotlib import pyplot
from statsmodels.tsa.arima_model import ARIMA
from sklearn.metrics import mean_squared_error
from math import sqrt
from time import time
from sklearn.metrics import mean_squared_error
fix_target(train, key, target1, new_target_name=target1)
 rate(train, key, target1, new_target_name="rate_" +target1)
 rate(train, key, target2, new_target_name="rate_" +target2)
 unique_keys=train[key].unique()
print(len(unique_keys))
train<categorify>
country='India'
state=''
sliced_data=data.loc[(data['Province_State']==state)&(data['Country_Region']==country),:]
test_sliced=test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:]
sliced_data=sliced_data.drop_duplicates()
sliced_data=sliced_data.reset_index(drop=True)
 sliced_data=sliced_data.sort_values(by='Date')
 if sliced_data.loc[sliced_data['ConfirmedCases']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['ConfirmedCases']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['ConfirmedCases'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(10),range(7),range(7))
preds=[]
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions=pd.DataFrame()
predictions['Date']=dates
predictions['ConfirmedCases']=preds
test_sliced=test_sliced.merge(sliced_data[['Date','ConfirmedCases']], on='Date',how='left')
 test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
test_sliced=test_sliced.merge(predictions, on='Date',how='left')
 test_sliced['ConfirmedCases_x'][test_sliced['ConfirmedCases_x'].isna() ]=test_sliced['ConfirmedCases_y'][test_sliced['ConfirmedCases_x'].isna() ]
test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_x']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
sliced_data_bck=sliced_data.copy()
if sliced_data.loc[sliced_data['Fatalities']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['Fatalities']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['Fatalities'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model=None
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions_f=pd.DataFrame()
predictions_f['Date']=dates
predictions_f['Fatalities']=preds
test_sliced=test_sliced.merge(sliced_data_bck[['Date','Fatalities']], on='Date',how='left')
 test_sliced['Fatalities']=test_sliced['Fatalities_y']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test_sliced=test_sliced.merge(predictions_f, on='Date',how='left')
 test_sliced['Fatalities_x'][test_sliced['Fatalities_x'].isna() ]=test_sliced['Fatalities_y'][test_sliced['Fatalities_x'].isna() ]
test_sliced['Fatalities']=test_sliced['Fatalities_x']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test=test.merge(test_sliced,on='ForecastId',how='left')
 test['ConfirmedCases_x'][test['ConfirmedCases_y'].notna() ]=test['ConfirmedCases_y'][test['ConfirmedCases_y'].notna() ]
test['Fatalities_x'][test['Fatalities_y'].notna() ]=test['Fatalities_y'][test['Fatalities_y'].notna() ]
new_cols=[]
for col in test.columns:
if col[-2:]=='_y':
del test[col]
elif col[-2:]=='_x':
new_cols.append(col[:-2])
 else:
new_cols.append(col)
 test.columns=new_cols
test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:].head()
plt.plot('Date', 'ConfirmedCases', data=sliced_data, color='blue', linewidth=2)
 plt.plot('Date','ConfirmedCases',data=test_sliced,color='orange',linewidth=2)
 plt.plot('Date', 'Fatalities', data=sliced_data, color='purple', linewidth=2)
 plt.plot('Date','Fatalities',data=test_sliced,color='red',linewidth=2)
 plt.show()
def get_lags(rate_array, current_index, size=20):
lag_confirmed_rate=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
lag_confirmed_rate[j]=rate_array[current_index-j]
else :
break
return lag_confirmed_rate
def days_ago_thresold_hit(full_array, indx, thresold):
days_ago_confirmed_count_10=-1
if full_array[indx]>thresold:
for j in range(indx,-1,-1):
entered=False
if full_array[j]<=thresold:
days_ago_confirmed_count_10=abs(j-indx)
 entered=True
break
if entered==False:
days_ago_confirmed_count_10=100
return days_ago_confirmed_count_10
def ewma_vectorized(data, alpha):
sums=sum([(alpha**(k+1)) *data[k] for k in range(len(data)) ])
 counts=sum([(alpha**(k+1)) for k in range(len(data)) ])
 return sums/counts
def generate_ma_std_window(rate_array, current_index, size=20, window=3):
ma_rate_confirmed=[-1 for k in range(size)]
std_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ma_rate_confirmed[j]=np.mean(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 std_rate_confirmed[j]=np.std(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 else :
break
return ma_rate_confirmed, std_rate_confirmed
def generate_ewma_window(rate_array, current_index, size=20, window=3, alpha=0.05):
ewma_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ewma_rate_confirmed[j]=ewma_vectorized(rate_array[max(0,current_index-j-window+1):current_index-j+1, ], alpha)
 else :
break
return ewma_rate_confirmed
def get_target(rate_col, indx, horizon=33, average=3, use_hard_rule=False):
target_values=[-1 for k in range(horizon)]
cou=0
for j in range(indx+1, indx+1+horizon):
if j<len(rate_col):
if average==1:
target_values[cou]=rate_col[j]
else :
if use_hard_rule and j +average <=len(rate_col):
target_values[cou]=np.mean(rate_col[j:j +average])
 else :
target_values[cou]=np.mean(rate_col[j:min(len(rate_col),j +average)])
 cou+=1
else :
break
return target_values
def dereive_features(frame, confirmed, fatalities, rate_confirmed, rate_fatalities,
horizon ,size=20, windows=[3,7], days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10]):
targets=[]
names=["lag_confirmed_rate" + str(k+1)for k in range(size)]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["lag_fatalities_rate" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
names+=["confirmed_plus" + str(k+1)for k in range(horizon)]
names+=["fatalities_plus" + str(k+1)for k in range(horizon)]
features=[]
for i in range(len(confirmed)) :
row_features=[]
lag_confirmed_rate=get_lags(rate_confirmed, i, size=size)
 row_features+=lag_confirmed_rate
for day in days_back_confimed:
days_ago_confirmed_count_10=days_ago_thresold_hit(confirmed, i, day)
 row_features+=[days_ago_confirmed_count_10]
for window in windows:
ma3_rate_confirmed,std3_rate_confirmed= generate_ma_std_window(rate_confirmed, i, size=size, window=window)
 row_features+= ma3_rate_confirmed
row_features+= std3_rate_confirmed
ewma3_rate_confirmed=generate_ewma_window(rate_confirmed, i, size=size, window=window, alpha=0.05)
 row_features+= ewma3_rate_confirmed
lag_fatalities_rate=get_lags(rate_fatalities, i, size=size)
 row_features+=lag_fatalities_rate
for day in days_back_fatalities:
days_ago_fatalitiescount_2=days_ago_thresold_hit(fatalities, i, day)
 row_features+=[days_ago_fatalitiescount_2]
for window in windows:
ma3_rate_fatalities,std3_rate_fatalities= generate_ma_std_window(rate_fatalities, i, size=size, window=window)
 row_features+= ma3_rate_fatalities
row_features+= std3_rate_fatalities
ewma3_rate_fatalities=generate_ewma_window(rate_fatalities, i, size=size, window=window, alpha=0.05)
 row_features+= ewma3_rate_fatalities
confirmed_level=0

 confirmed_level= confirmed[i]
row_features+=[confirmed_level]
fatalities_is_level=0

 fatalities_is_level= fatalities[i]
row_features+=[fatalities_is_level]
confirmed_plus=get_target(rate_confirmed, i, horizon=horizon)
 row_features+= confirmed_plus
fatalities_plus=get_target(rate_fatalities, i, horizon=horizon)
 row_features+= fatalities_plus
features.append(row_features)
 new_frame=pd.DataFrame(data=features, columns=names ).reset_index(drop=True)
 frame=frame.reset_index(drop=True)
 frame=pd.concat([frame, new_frame], axis=1)
 return frame
def feature_engineering_for_single_key(frame, group, key, horizon=33, size=20, windows=[3,7],
days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10]):
mini_frame=get_data_by_key(frame, group, key, fields=None)
 mini_frame_with_features=dereive_features(mini_frame, mini_frame["ConfirmedCases"].values,
mini_frame["Fatalities"].values, mini_frame["rate_ConfirmedCases"].values,
mini_frame["rate_Fatalities"].values, horizon ,size=size, windows=windows,
days_back_confimed=days_back_confimed, days_back_fatalities=days_back_fatalities)
 return mini_frame_with_features<predict_on_test>
test.loc[(test['Province_State']==state)&(test['Country_Region']==country),['Country_Region','Date','ConfirmedCases','Fatalities']]
def predict(xtest,input_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 model= joblib.load(input_name)
 preds=model.predict(xtest)
 baggedpred+=preds
return baggedpred 
<define_variables>
test['ConfirmedCases']=0
test['Fatalities']=0
names=["lag_confirmed_rate" + str(k+1)for k in range(size)]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["lag_fatalities_rate" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"] 
<normalization>
for country in countries:
for state in countries[country]:
print('Country : '+country,'State : '+state)
 sliced_data=data.loc[(data['Province_State']==state)&(data['Country_Region']==country),:]
test_sliced=test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:]
sliced_data=sliced_data.drop_duplicates()
sliced_data=sliced_data.reset_index(drop=True)
 sliced_data=sliced_data.sort_values(by='Date')
 if sliced_data.loc[sliced_data['ConfirmedCases']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['ConfirmedCases']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['ConfirmedCases'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions=pd.DataFrame()
predictions['Date']=dates
predictions['ConfirmedCases']=preds
test_sliced=test_sliced.merge(sliced_data[['Date','ConfirmedCases']], on='Date',how='left')
 test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
test_sliced=test_sliced.merge(predictions, on='Date',how='left')
 test_sliced['ConfirmedCases_x'][test_sliced['ConfirmedCases_x'].isna() ]=test_sliced['ConfirmedCases_y'][test_sliced['ConfirmedCases_x'].isna() ]
test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_x']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
sliced_data_bck=sliced_data.copy()
if sliced_data.loc[sliced_data['Fatalities']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['Fatalities']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['Fatalities'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model=None
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions_f=pd.DataFrame()
predictions_f['Date']=dates
predictions_f['Fatalities']=preds
test_sliced=test_sliced.merge(sliced_data_bck[['Date','Fatalities']], on='Date',how='left')
 test_sliced['Fatalities']=test_sliced['Fatalities_y']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test_sliced=test_sliced.merge(predictions_f, on='Date',how='left')
 test_sliced['Fatalities_x'][test_sliced['Fatalities_x'].isna() ]=test_sliced['Fatalities_y'][test_sliced['Fatalities_x'].isna() ]
test_sliced['Fatalities']=test_sliced['Fatalities_x']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test=test.merge(test_sliced,on='ForecastId',how='left')
 test['ConfirmedCases_x'][test['ConfirmedCases_y'].notna() ]=test['ConfirmedCases_y'][test['ConfirmedCases_y'].notna() ]
test['Fatalities_x'][test['Fatalities_y'].notna() ]=test['Fatalities_y'][test['Fatalities_y'].notna() ]
new_cols=[]
for col in test.columns:
if col[-2:]=='_y':
del test[col]
elif col[-2:]=='_x':
new_cols.append(col[:-2])
 else:
new_cols.append(col)
 test.columns=new_cols
print(test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:].head())
 plt.plot('Date', 'ConfirmedCases', data=sliced_data, color='blue', linewidth=2)
 plt.plot('Date','ConfirmedCases',data=test_sliced,color='orange',linewidth=2)
 plt.plot('Date', 'Fatalities', data=sliced_data, color='purple', linewidth=2)
 plt.plot('Date','Fatalities',data=test_sliced,color='red',linewidth=2)
 plt.show()
<categorify><EOS> 
output=pd.DataFrame()
output['ForecastId']=test['ForecastId'].astype(int)
 output['ConfirmedCases']=test['ConfirmedCases'].astype(int)
 output['Fatalities']=test['Fatalities'].astype(int)
 output.to_csv('submission.csv',index=False )