Kaggle competition: COVID19 Global Forecasting (Week 3)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-week-3<feature_engineering>
path = '.. /input/covid19-global-forecasting-week-3/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'submission.csv')
 train['Date'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 test['Date'] = test['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 train['days'] =(train['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
test['days'] =(test['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
train.loc[train['Province_State'].isnull() , 'Province_State'] = 'N/A'
test.loc[test['Province_State'].isnull() , 'Province_State'] = 'N/A'
train['Area'] = train['Country_Region'] + '_' + train['Province_State']
test['Area'] = test['Country_Region'] + '_' + test['Province_State']
print(train['Date'].max())
 print(test['Date'].min())
 print(train['days'].max())
 N_AREAS = train['Area'].nunique()
AREAS = np.sort(train['Area'].unique())
 TRAIN_N = 77
print(train[train['days'] < TRAIN_N]['Date'].max())
 print(train[train['days'] >= TRAIN_N]['Date'].min())
 print(train[train['days'] >= TRAIN_N]['Date'].max())
 train.head()
test_orig = test.copy()
train.loc[train.wg2<0,'wg2'] = 0
test.loc[test.wg2<0,'wg2'] = 0
train.loc[train.wg2.isnull() ,'wg2'] = 0
test.loc[test.wg2.isnull() ,'wg2'] = 0
train.loc[np.isinf(train.wg2),'wg2'] = 0
test.loc[ np.isinf(test.wg2),'wg2'] = 0
train.loc[train.wg3<0,'wg3'] = 0
test.loc[test.wg3<0,'wg3'] = 0
train.loc[train.wg3.isnull() ,'wg3'] = 0
test.loc[test.wg3.isnull() ,'wg3'] = 0
train.loc[np.isinf(train.wg3),'wg3'] = 0
test.loc[ np.isinf(test.wg3),'wg3'] = 0
print('done', datetime.now() )<compute_train_metric>
train_p_c_raw = train.pivot(index='Area', columns='days', values='ConfirmedCases' ).sort_index()
train_p_f_raw = train.pivot(index='Area', columns='days', values='Fatalities' ).sort_index()
train_p_c = np.maximum.accumulate(train_p_c_raw, axis=1)
 train_p_f = np.maximum.accumulate(train_p_f_raw, axis=1)
 f_rate =(train_p_f / train_p_c ).fillna(0)
 X_c = np.log(1+train_p_c.values)[:,:TRAIN_N]
X_f = train_p_f.values[:,:TRAIN_N]
print(mean_squared_log_error(train.ConfirmedCases, train.wg))
print(mean_squared_log_error(train.ConfirmedCases, train.wg2))
print(mean_squared_log_error(train.ConfirmedCases, train.wg3)) 
<compute_test_metric>
def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run_c(params, X, test_size=50):
gr_base = []
gr_base_factor = []
x_min = np.ma.MaskedArray(X, X<1)
 x_min = x_min.argmin(axis=1)
 for i in range(X.shape[0]):
temp = X[i,:]
threshold = np.log(1+params['min cases for growth rate'])
 num_days = params['last N days']
if(temp > threshold ).sum() > num_days:
d = np.diff(temp[temp > threshold])[-num_days:]
w = np.arange(len(d)) +1
w = w**5
w = w / np.sum(w)
 gr_base.append(np.clip(np.average(d, weights=w), 0, params['growth rate max']))
d2 = np.diff(d)
 w = np.arange(len(d2)) +1
w = w**10
w = w / np.sum(w)
 gr_base_factor.append(np.clip(np.average(d2, weights=w), -0.5, params["growth rate factor max"]))
else:
gr_base.append(params['growth rate default'])
 gr_base_factor.append(params['growth rate factor'])
 gr_base = np.array(gr_base)
 gr_base_factor = np.array(gr_base_factor)
 preds = X.copy()
for i in range(test_size):
delta = np.clip(preds[:, -1], np.log(2), None)+ gr_base *(1 + params['growth rate factor']*(1 + params['growth rate factor factor'])**(i)) **(np.log1p(i))
preds = np.hstack(( preds, delta.reshape(-1,1)))
 return preds
params = {
"min cases for growth rate": 0,
"last N days": 15,
"growth rate default": 0.10,
"growth rate max": 0.2,
"growth rate factor max": -0.1,
"growth rate factor": -0.3,
"growth rate factor factor": 0.01,
}
x = train_p_c
preds_c = run_c(params, np.log(1+x.values)[:,:TRAIN_N])
for country in train.Country_Region.unique() :
bool1 = train.Country_Region == country
for state in train[bool1].Province_State.unique() :
bool2 = bool1 &(train.Province_State == state)
 data = train[ bool2 ].copy().reset_index()
wg1 = np.round(mean_squared_log_error(data.ConfirmedCases, data.wg),2)
 wg2 = np.round(mean_squared_log_error(data.ConfirmedCases, data.wg2),2)
 wg3 = np.round(mean_squared_log_error(data.ConfirmedCases, data.wg3),2)
 boolt =(test.Country_Region==country)&(test.Province_State== state)
 datat = test[boolt].copy().reset_index()
if wg1<wg2 and wg1<wg3:
data['best'] = data.wg
datat['best'] = datat.wg
elif wg2<wg1 and wg2 < wg3:
data['best'] = data.wg2
datat['best'] = datat.wg2
else:
data['best'] = data.wg3
datat['best'] = datat.wg3
train.loc[bool2, 'wg'] = data['best'].values
test.loc[boolt, 'wg'] = datat['best'].values
print('done', datetime.now() )<compute_test_metric>
for i in range(N_AREAS):
if 'China' in AREAS[i] and preds_c[i, TRAIN_N-1] < np.log(31):
preds_c[i, TRAIN_N:] = preds_c[i, TRAIN_N-1]
print(mean_squared_log_error(train.ConfirmedCases, train.wg))
train.drop('wg2', inplace=True, axis=1)
 train.drop('wg3', inplace=True, axis=1)
 test.drop('wg2', inplace=True, axis=1)
 test.drop('wg3', inplace=True, axis=1)
 print('done', datetime.now() )<feature_engineering>
def lin_w(sz):
res = np.linspace(0, 1, sz+1, endpoint=False)[1:]
return np.append(res, np.append([1], res[::-1]))
def run_f(params, X_c, X_f, X_f_r, test_size=50):
X_f_r = np.array(np.ma.mean(np.ma.masked_outside(X_f_r, 0.03, 0.5)[:,:], axis=1))
X_f_r = np.clip(X_f_r, params['fatality_rate_lower'], params['fatality_rate_upper'])
 X_c = np.clip(np.exp(X_c)-1, 0, None)
 preds = X_f.copy()
train_size = X_f.shape[1] - 1
for i in range(test_size):
t_lag = train_size+i-params['length']
t_wsize = 5
d = np.diff(X_c, axis=1)[:, t_lag-t_wsize:t_lag+1+t_wsize]
delta = np.average(d, axis=1)
 delta = params['absolute growth'] + delta * X_f_r
preds = np.hstack(( preds, preds[:, -1].reshape(-1,1)+ delta.reshape(-1,1)))
 return preds
params = {
"length": 7,
"absolute growth": 0.02,
"fatality_rate_lower": 0.02,
"fatality_rate_upper": 0.3,
}
preds_f_1 = run_f(params, preds_c, X_f, f_rate.values[:,:TRAIN_N])
 preds_f_1 = np.log(1+preds_f_1 )
train_data = train.copy()
train_df = train_data
train_df['area'] = [str(i)+str(' - ')+str(j)for i,j in zip(train_data['Country_Region'], train_data['Province_State'])]
train_df['Date'] = pd.to_datetime(train_df['Date'])
 full_data = train_df
today = full_data['Date'].max() +timedelta(days=1)
 def get_country_data(train_df, area, metric):
country_data = train_df[train_df['area']==area]
country_data = country_data.drop(['Id','Province_State', 'Country_Region', 'Lat','Long'], axis=1)
 country_data = pd.pivot_table(country_data, values=['ConfirmedCases','Fatalities'], index=['Date'], aggfunc=np.sum)
 country_data = country_data[country_data[metric]!=0]
return country_data
area_info = pd.DataFrame(columns=['area', 'cases_start_date', 'deaths_start_date', 'init_ConfirmedCases', 'init_Fatalities'])
 for i in range(len(train_df['area'].unique())) :
area = train_df['area'].unique() [i]
area_cases_data = get_country_data(train_df, area, 'ConfirmedCases')
 area_deaths_data = get_country_data(train_df, area, 'Fatalities')
 cases_start_date = area_cases_data.index.min()
deaths_start_date = area_deaths_data.index.min()
if len(area_cases_data)> 0:
confirmed_cases = max(area_cases_data['ConfirmedCases'])
 else:
confirmed_cases = 0
if len(area_deaths_data)> 0:
fatalities = max(area_deaths_data['Fatalities'])
 else:
fatalities = 0
area_info.loc[i] = [area, cases_start_date, deaths_start_date, confirmed_cases, fatalities]
area_info = area_info.fillna(pd.to_datetime(today))
area_info['init_cases_day_no'] = pd.to_datetime(today)-area_info['cases_start_date']
area_info['init_cases_day_no'] = area_info['init_cases_day_no'].dt.days.fillna(0 ).astype(int)
 area_info['init_deaths_day_no'] = pd.to_datetime(today)-area_info['deaths_start_date']
area_info['init_deaths_day_no'] = area_info['init_deaths_day_no'].dt.days.fillna(0 ).astype(int)
 area_info.head()
def log_curve(x, k, x_0, ymax):
return ymax /(1 + np.exp(-k*(x-x_0)))
 def log_fit(train_df, area, metric):
area_data = get_country_data(train_df, area, metric)
 x_data = range(len(area_data.index))
y_data = area_data[metric]
if len(y_data)< 5:
estimated_k = -1
estimated_x_0 = -1
ymax = -1
elif max(y_data)== 0:
estimated_k = -1
estimated_x_0 = -1
ymax = -1
else:
try:
popt, pcov = curve_fit(log_curve, x_data, y_data, bounds=([0,0,0],np.inf), p0=[0.3,100,10000], maxfev=1000000)
 estimated_k, estimated_x_0, ymax = popt
except RuntimeError:
print(area)
 print("Error - curve_fit failed")
 estimated_k = -1
estimated_x_0 = -1
ymax = -1
estimated_parameters = pd.DataFrame(np.array([[area, estimated_k, estimated_x_0, ymax]]), columns=['area', 'k', 'x_0', 'ymax'])
 return estimated_parameters
def get_parameters(metric):
parameters = pd.DataFrame(columns=['area', 'k', 'x_0', 'ymax'], dtype=np.float)
 for area in train_df['area'].unique() :
estimated_parameters = log_fit(train_df, area, metric)
 parameters = parameters.append(estimated_parameters)
 parameters['k'] = pd.to_numeric(parameters['k'], downcast="float")
 parameters['x_0'] = pd.to_numeric(parameters['x_0'], downcast="float")
 parameters['ymax'] = pd.to_numeric(parameters['ymax'], downcast="float")
 parameters = parameters.replace({'k': {-1: parameters[parameters['ymax']>0].median() [0]},
'x_0': {-1: parameters[parameters['ymax']>0].median() [1]},
'ymax': {-1: parameters[parameters['ymax']>0].median() [2]}})
 return parameters
cases_parameters = get_parameters('ConfirmedCases')
 cases_parameters.head(20)
 deaths_parameters = get_parameters('Fatalities')
 deaths_parameters.head(20)
 fit_df = area_info.merge(cases_parameters, on='area', how='left')
 fit_df = fit_df.rename(columns={"k": "cases_k", "x_0": "cases_x_0", "ymax": "cases_ymax"})
 fit_df = fit_df.merge(deaths_parameters, on='area', how='left')
 fit_df = fit_df.rename(columns={"k": "deaths_k", "x_0": "deaths_x_0", "ymax": "deaths_ymax"})
 fit_df['init_ConfirmedCases_fit'] = log_curve(fit_df['init_cases_day_no'], fit_df['cases_k'], fit_df['cases_x_0'], fit_df['cases_ymax'])
 fit_df['init_Fatalities_fit'] = log_curve(fit_df['init_deaths_day_no'], fit_df['deaths_k'], fit_df['deaths_x_0'], fit_df['deaths_ymax'])
 fit_df['ConfirmedCases_error'] = fit_df['init_ConfirmedCases']-fit_df['init_ConfirmedCases_fit']
fit_df['Fatalities_error'] = fit_df['init_Fatalities']-fit_df['init_Fatalities_fit']
fit_df.head()
test_data = test.copy()
test_df = test_data
test_df['area'] = [str(i)+str(' - ')+str(j)for i,j in zip(test_data['Country_Region'], test_data['Province_State'])]
test_df = test_df.merge(fit_df, on='area', how='left')
 test_df['Date'] = pd.to_datetime(test_df['Date'])
 test_df['cases_start_date'] = pd.to_datetime(test_df['cases_start_date'])
 test_df['deaths_start_date'] = pd.to_datetime(test_df['deaths_start_date'])
 test_df['cases_day_no'] = test_df['Date']-test_df['cases_start_date']
test_df['cases_day_no'] = test_df['cases_day_no'].dt.days.fillna(0 ).astype(int)
 test_df['deaths_day_no'] = test_df['Date']-test_df['deaths_start_date']
test_df['deaths_day_no'] = test_df['deaths_day_no'].dt.days.fillna(0 ).astype(int)
 test_df['ConfirmedCases_fit'] = log_curve(test_df['cases_day_no'], test_df['cases_k'], test_df['cases_x_0'], test_df['cases_ymax'])
 test_df['Fatalities_fit'] = log_curve(test_df['deaths_day_no'], test_df['deaths_k'], test_df['deaths_x_0'], test_df['deaths_ymax'])
 test_df['ConfirmedCases_pred'] = round(test_df['ConfirmedCases_fit']+test_df['ConfirmedCases_error'])
 test_df['Fatalities_pred'] = round(test_df['Fatalities_fit']+test_df['Fatalities_error'])
 test_df.head()
train_df = train.copy()
train_df['area'] = [str(i)+str(' - ')+str(j)for i,j in zip(train_df['Country_Region'], train_df['Province_State'])]
train_df = train_df.merge(fit_df, on='area', how='left')
 train_df['Date'] = pd.to_datetime(train_df['Date'])
 train_df['cases_start_date'] = pd.to_datetime(train_df['cases_start_date'])
 train_df['deaths_start_date'] = pd.to_datetime(train_df['deaths_start_date'])
 train_df['cases_day_no'] = train_df['Date']-train_df['cases_start_date']
train_df['cases_day_no'] = train_df['cases_day_no'].dt.days.fillna(0 ).astype(int)
 train_df['deaths_day_no'] = train_df['Date']-train_df['deaths_start_date']
train_df['deaths_day_no'] = train_df['deaths_day_no'].dt.days.fillna(0 ).astype(int)
 train_df['ConfirmedCases_fit'] = log_curve(train_df['cases_day_no'], train_df['cases_k'], train_df['cases_x_0'], train_df['cases_ymax'])
 train_df['Fatalities_fit'] = log_curve(train_df['deaths_day_no'], train_df['deaths_k'], train_df['deaths_x_0'], train_df['deaths_ymax'])
 train_df['ConfirmedCases_pred'] = round(train_df['ConfirmedCases_fit']+train_df['ConfirmedCases_error'])
 train_df['Fatalities_pred'] = round(train_df['Fatalities_fit']+train_df['Fatalities_error'])
 train_df.head()
print('done', datetime.now() )<train_model>
class ZDatasetF(Dataset):
def __init__(self, X_c, X_f=None, hist_len=10):
self.X_c = X_c
self.X_f = X_f
self.hist_len = hist_len
self.is_test = X_f is None
def __len__(self):
return self.X_c.shape[1]
def __getitem__(self, idx):
if self.is_test:
return {'x_c':self.X_c[:, idx-self.hist_len:idx]}
else:
return {'x_c':self.X_c[:, idx-self.hist_len:idx],
'x_f':self.X_f[:, idx-1],
'y':np.log(1+self.X_f[:, idx])}
class PrLayer2(nn.Module):
def __init__(self, in_features1, in_features2):
super(PrLayer2, self ).__init__()
self.weight0 = Parameter(torch.Tensor(1, 1, in_features2))
self.weight1 = Parameter(torch.Tensor(1, in_features1, in_features2))
self.reset_parameters()
def reset_parameters(self):
init.kaiming_uniform_(self.weight0, a=math.sqrt(5))
init.kaiming_uniform_(self.weight1, a=math.sqrt(5))
def forward(self, input):
return input * torch.sigmoid(self.weight0 + self.weight1)
 class ZModelF(nn.Module):
def __init__(self, hist_len):
super(ZModelF, self ).__init__()
self.l_conv = PrLayer2(len(X_c),hist_len-1)
 def forward(self, x_c, x_f):
x = x_c[:,:,1:] - x_c[:,:,:-1]
res = torch.sum(self.l_conv(x), 2)
 return {'preds': torch.log(1 + x_f + res)}
class DummySampler(torch.utils.data.sampler.Sampler):
def __init__(self, idx):
self.idx = idx
def __iter__(self):
return iter(self.idx)
 def __len__(self):
return len(self.idx)
 def _smooth_l1_loss(target):
t = torch.abs(target)
 t = torch.where(t < 1, 0.5 * t ** 2, t - 0.5)
 return torch.mean(t)
 n_epochs = 5000
lr = 0.18
bag_size = 4
device = 'cpu'
hist_len = 14
loss_func = torch.nn.MSELoss()
reg_loss_func = _smooth_l1_loss
reg_factor = 0.035
train_dataset = ZDatasetF(np.exp(X_c)-1, X_f, hist_len=hist_len)
 test_dataset = ZDatasetF(np.exp(preds_c)-1, hist_len=hist_len)
 trn_idx = np.arange(hist_len+1, len(train_dataset))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(trn_idx)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=len(trn_idx), sampler=train_sampler, num_workers=0, pin_memory=True)
 test_idx = np.arange(TRAIN_N, len(test_dataset))
test_sampler = DummySampler(test_idx)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, sampler=test_sampler, num_workers=0, pin_memory=True)
 gradient_accumulation = 1
preds_f = 0
for m_i in range(bag_size):
model_f = ZModelF(hist_len=hist_len ).to(device)
 optimizer_f = torch.optim.Adam(model_f.parameters() , lr=lr)
 model_f.train()
start_time = time.time()
for epoch in range(n_epochs):
s = time.time()
avg_train_loss = 0
optimizer_f.zero_grad()
for idx, data in enumerate(train_loader):
X1 = data['x_c'].to(device ).float()
X2 = data['x_f'].to(device ).float()
y = data['y'].to(device ).float()
preds = model_f(X1, X2)['preds'].float()
cond = X2 > np.log(10)
 preds = preds[cond]
y = y[cond]
loss = loss_func(preds, y)
 loss += reg_factor * reg_loss_func(model_f.l_conv.weight1)
 avg_train_loss += loss / len(train_loader)
 loss.backward()
if(idx+1)% gradient_accumulation == 0 or idx == len(train_loader)- 1:
optimizer_f.step()
optimizer_f.zero_grad()
if False:
model_f.eval()
preds_f_delta = train_p_f.values[:,:TRAIN_N]
for idx, data in enumerate(test_loader):
X1 = data['x_c'].to(device ).float()
temp = model_f(X1, torch.Tensor(preds_f_delta[:,-1] ).unsqueeze(0)) ['preds']
temp = np.exp(temp.detach().cpu().numpy().reshape(-1,1)) - 1
preds_f_delta = np.hstack(( preds_f_delta, temp))
preds_f_delta = np.log(1 + preds_f_delta)
 val_len = train_p_c.values.shape[1] - TRAIN_N
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , \
preds_f_delta[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{epoch:2} train_loss {avg_train_loss:<8.4f} val_loss {m2:8.5f} {time.time() -s:<2.2f}")
 model_f.train()
model_f.eval()
preds_f_delta = train_p_f.values[:,:TRAIN_N]
for idx, data in enumerate(test_loader):
X1 = data['x_c'].to(device ).float()
temp = model_f(X1, torch.Tensor(preds_f_delta[:,-1] ).unsqueeze(0)) ['preds']
temp = np.exp(temp.detach().cpu().numpy().reshape(-1,1)) - 1
preds_f_delta = np.hstack(( preds_f_delta, temp))
preds_f += preds_f_delta / bag_size
preds_f_2 = np.log(1 + preds_f)
 print("Done")
train['y_hat_log' ] = test_df.ConfirmedCases_fit
test['y_hat_log' ] = test_df.ConfirmedCases_pred<drop_column>
preds_f = np.average([preds_f_1, preds_f_2], axis=0, weights=[1,2] )
dropcols = ['Date', 'date', 'ConfirmedCases', 'Id', 'ForecastId', 'Fatalities']
dropcols = dropcols + ['Country_Region','Province_State']
dropcols = dropcols + ['eg', 'egr', 'mae_eg', 'wga', 'wgr', 'wgc','wghcnb', 'mae_wg', 'SARIMAX', 'ARIMA',
'cc_es',]
print('
Modeling...')
 features = [f for f in train.columns if f not in dropcols +
['shift4w', 'shift6w', 'dist'
, 'dayofyear','year','quarter','hour','month','dayofmonth','dayofweek','weekofyear', 'Lat', 'Long']]
print(features)
 X_train = train[features].copy()
X_test = test[features].copy()
X_train.reset_index(drop=True, inplace=True)
 X_test.reset_index(drop=True, inplace=True)
 y_train = train["Fatalities"]
y_train_cc = train["ConfirmedCases"]
print('done', datetime.now() )<choose_model_class>
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, d]), preds_c[:, d]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, d]), preds_f[:, d]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c[:, TRAIN_N:TRAIN_N+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )
isTraining = False
params_xgb = {}
params_xgb['n_estimators'] = 1100
params_xgb['max_depth'] = 10
params_xgb['seed'] = 2020
params_xgb['colsample_bylevel'] = 1
params_xgb['colsample_bytree'] = 1
params_xgb['learning_rate'] = 0.3
params_xgb['reg_alpha'] = 0
params_xgb['reg_lambda'] = 1
params_xgb['subsample'] = 1
if isTraining:
X_TRAIN = X_train[features].values
kf = KFold(n_splits = 5, shuffle = True, random_state=2020)
 acc = []
for tr_idx, val_idx in kf.split(X_TRAIN, y_train_cc):
X_tr, X_vl = X_TRAIN[tr_idx], X_TRAIN[val_idx, :]
y_tr, y_vl = y_train_cc[tr_idx], y_train_cc[val_idx]
print(X_tr.shape)
 model_xgb_cc = xgb.XGBRegressor(**params_xgb)
 model_xgb_cc.fit(X_tr, y_tr, verbose=True)
 y_hat = model_xgb_cc.predict(X_vl)
 print('xgb mae :', mean_absolute_error(y_vl, y_hat))
acc.append(mean_absolute_error(y_vl, y_hat))
print('done', np.mean(acc))
print('done', datetime.now() )<prepare_x_and_y>
EU_COUNTRIES = ['Austria', 'Italy', 'Belgium', 'Latvia', 'Bulgaria', 'Lithuania', 'Croatia', 'Luxembourg', 'Cyprus', 'Malta', 'Czechia',
'Netherlands', 'Denmark', 'Poland', 'Estonia', 'Portugal', 'Finland', 'Romania', 'France', 'Slovakia', 'Germany', 'Slovenia',
'Greece', 'Spain', 'Hungary', 'Sweden', 'Ireland']
EUROPE_OTHER = ['Albania', 'Andorra', 'Bosnia and Herzegovina', 'Liechtenstein', 'Monaco', 'Montenegro', 'North Macedonia',
'Norway', 'San Marino', 'Serbia', 'Switzerland', 'Turkey', 'United Kingdom']
AFRICA = ['Algeria', 'Burkina Faso', 'Cameroon', 'Congo(Kinshasa)', "Cote d'Ivoire", 'Egypt', 'Ghana', 'Kenya', 'Madagascar',
'Morocco', 'Nigeria', 'Rwanda', 'Senegal', 'South Africa', 'Togo', 'Tunisia', 'Uganda', 'Zambia']
NORTH_AMERICA = ['US', 'Canada', 'Mexico']
SOUTH_AMERICA = ['Argentina', 'Bolivia', 'Brazil', 'Chile', 'Colombia', 'Ecuador', 'Paraguay', 'Peru', 'Uruguay', 'Venezuela']
MIDDLE_EAST = ['Afghanistan', 'Bahrain', 'Iran', 'Iraq', 'Israel', 'Jordan', 'Kuwait', 'Lebanon', 'Oman', 'Qatar', 'Saudi Arabia', 'United Arab Emirates']
ASIA = ['Bangladesh', 'Brunei', 'Cambodia', 'India', 'Indonesia', 'Japan', 'Kazakhstan', 'Korea, South', 'Kyrgyzstan', 'Malaysia',
'Pakistan', 'Singapore', 'Sri Lanka', 'Taiwan*', 'Thailand', 'Uzbekistan', 'Vietnam']
dropcols = ['Date', 'date', 'ConfirmedCases', 'Id', 'ForecastId', 'Fatalities']
dropcols = dropcols + ['Country_Region','Province_State']
dropcols = dropcols + ['eg', 'egr', 'mae_eg', 'wg', 'wga', 'wgr', 'wg_xgb', 'wgc','wghcnb', 'mae_wg', 'SARIMAX', 'ARIMA',
'cc_es',]
print('
Modeling...')
 features = [f for f in train.columns if f not in dropcols +
['shift4w', 'shift6w', 'dist'
, 'dayofyear','year','quarter','hour','month','dayofmonth','dayofweek','weekofyear']]
print(features)
 X_train = train[features].copy()
X_test = test[features].copy()
X_train.reset_index(drop=True, inplace=True)
 X_test.reset_index(drop=True, inplace=True)
 X_train.head()
print('done', datetime.now() )<init_hyperparams>
temp = pd.DataFrame(np.clip(np.exp(preds_c)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="ConfirmedCases")
 test = test_orig.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 temp = pd.DataFrame(np.clip(np.exp(preds_f)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="Fatalities")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 test.head()
params_xgb = {}
params_xgb['n_estimators'] = 1100
params_xgb['max_depth'] = 10
params_xgb['seed'] = 2020
params_xgb['colsample_bylevel'] = 1
params_xgb['colsample_bytree'] = 1
params_xgb['learning_rate'] = 0.3
params_xgb['reg_alpha'] = 0
params_xgb['reg_lambda'] = 1
params_xgb['subsample'] = 1
isTraining = True
X_train.reset_index(drop=True, inplace=True)
 if isTraining:
print('Evaluating model...')
 booll = X_train.day_from_jan_first < 86
X_tr, X_vl = X_train[booll][features ], X_train[~booll][features]
y_tr, y_vl = train[booll]['ConfirmedCases'], train[~booll]['ConfirmedCases']
model_xgb_cc = xgb.XGBRegressor(**params_xgb)
 model_xgb_cc.fit(X_tr, y_tr, verbose=True)
 y_hat = model_xgb_cc.predict(X_vl)
 y_hat[y_hat<0] = 0
print('xgb mae :', mean_absolute_error(y_vl, y_hat), mean_squared_log_error(y_vl, y_hat), X_tr.shape, X_vl.shape)
 print('done', datetime.now() )<train_model>
test.to_csv("submission.csv", index=False, columns=["ForecastId", "ConfirmedCases", "Fatalities"] )
y_train = train["Fatalities"]
y_train_cc = train["ConfirmedCases"]
params_xgb = {}
params_xgb['n_estimators'] = 1100
params_xgb['max_depth'] = 10
params_xgb['seed'] = 2020
params_xgb['colsample_bylevel'] = 1
params_xgb['colsample_bytree'] = 1
params_xgb['learning_rate'] = 0.3
params_xgb['reg_alpha'] = 0
params_xgb['reg_lambda'] = 1
params_xgb['subsample'] = 1
model_xgb_cc = xgb.XGBRegressor(**params_xgb ).fit(X_train[features], y_train_cc, verbose=True)
 y_hat_xgb_c = model_xgb_cc.predict(X_test[features])
 print('done', datetime.now() )<train_model>
test.days.nunique()
params_xgb = {}
params_xgb['n_estimators'] = 1100
params_xgb['max_depth'] = 10
params_xgb['seed'] = 2020
params_xgb['colsample_bylevel'] = 1
params_xgb['colsample_bytree'] = 1
params_xgb['learning_rate'] = 0.300000012
params_xgb['reg_alpha'] = 0
params_xgb['reg_lambda'] = 1
params_xgb['subsample'] = 1
model_xgb_f = xgb.XGBRegressor(**params_xgb ).fit(X_train[features], y_train, verbose=True)
 y_hat_xgb_f = model_xgb_f.predict(X_test[features])
 print(np.mean(y_hat_xgb_f))
print('done', datetime.now() )<feature_engineering>
for i, rec in test.groupby('Area' ).last().sort_values("ConfirmedCases", ascending=False ).iterrows() :
print(f"{rec['ConfirmedCases']:10.1f} {rec['Fatalities']:10.1f} {rec['Country_Region']}, {rec['Province_State']}")