Kaggle competition: COVID19 Global Forecasting (Week 3)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-week-3<count_values>
path = '.. /input/covid19-global-forecasting-week-3/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'submission.csv')
 train['Date'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 test['Date'] = test['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 train['days'] =(train['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
test['days'] =(test['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
train.loc[train['Province_State'].isnull() , 'Province_State'] = 'N/A'
test.loc[test['Province_State'].isnull() , 'Province_State'] = 'N/A'
train['Area'] = train['Country_Region'] + '_' + train['Province_State']
test['Area'] = test['Country_Region'] + '_' + test['Province_State']
print('train Date max',train['Date'].max())
 print('test Date min',test['Date'].min())
 print('train days max', train['days'].max())
 N_AREAS = train['Area'].nunique()
AREAS = np.sort(train['Area'].unique())
 START_PUBLIC = test['days'].min()
print('public LB start day', START_PUBLIC)
 print(' ')
 TRAIN_N = 77
print(train[train['days'] < TRAIN_N]['Date'].max())
 print(train[train['days'] >= TRAIN_N]['Date'].min())
 print(train[train['days'] >= TRAIN_N]['Date'].max())
 train.head()
test_orig = test.copy()
train_data['WheelType'].value_counts()<count_values>
train_p_c_raw = train.pivot(index='Area', columns='days', values='ConfirmedCases' ).sort_index()
train_p_f_raw = train.pivot(index='Area', columns='days', values='Fatalities' ).sort_index()
train_p_c = np.maximum.accumulate(train_p_c_raw, axis=1)
 train_p_f = np.maximum.accumulate(train_p_f_raw, axis=1)
 f_rate =(train_p_f / train_p_c ).fillna(0)
 X_c = np.log(1+train_p_c.values)[:,:TRAIN_N]
X_f = train_p_f.values[:,:TRAIN_N]
train_data['Nationality'].value_counts()<count_values>
def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run_c(params, X, test_size=50):
gr_base = []
gr_base_factor = []
x_min = np.ma.MaskedArray(X, X<1)
 x_min = x_min.argmin(axis=1)
 for i in range(X.shape[0]):
temp = X[i,:]
threshold = np.log(1+params['min cases for growth rate'])
 num_days = params['last N days']
if(temp > threshold ).sum() > num_days:
d = np.diff(temp[temp > threshold])[-num_days:]
w = np.arange(len(d)) +1
w = w**5
w = w / np.sum(w)
 gr_base.append(np.clip(np.average(d, weights=w), 0, params['growth rate max']))
d2 = np.diff(d)
 w = np.arange(len(d2)) +1
w = w**10
w = w / np.sum(w)
 gr_base_factor.append(np.clip(np.average(d2, weights=w), -0.5, params["growth rate factor max"]))
else:
gr_base.append(params['growth rate default'])
 gr_base_factor.append(params['growth rate factor'])
 gr_base = np.array(gr_base)
 gr_base_factor = np.array(gr_base_factor)
 preds = X.copy()
for i in range(test_size):
delta = np.clip(preds[:, -1], np.log(2), None)+ gr_base *(1 + params['growth rate factor']*(1 + params['growth rate factor factor'])**(i)) **(np.log1p(i))
preds = np.hstack(( preds, delta.reshape(-1,1)))
 return preds
params = {
"min cases for growth rate": 0,
"last N days": 15,
"growth rate default": 0.2,
"growth rate max": 0.3,
"growth rate factor max": -0.1,
"growth rate factor": -0.3,
"growth rate factor factor": 0.01,
}
x = train_p_c
preds_c = run_c(params, np.log(1+x.values)[:,:TRAIN_N])
test_data['Nationality'].value_counts()<data_type_conversions>
for i in range(N_AREAS):
if 'China' in AREAS[i] and preds_c[i, TRAIN_N-1] < np.log(31):
preds_c[i, TRAIN_N:] = preds_c[i, TRAIN_N-1]
train_data['Nationality'].fillna(value = 'Nationality_Unknown', inplace = True)
 test_data['Nationality'].fillna(value = 'Nationality_Unknown', inplace = True )<count_values>
def lin_w(sz):
res = np.linspace(0, 1, sz+1, endpoint=False)[1:]
return np.append(res, np.append([1], res[::-1]))
def run_f(params, X_c, X_f, X_f_r, test_size=50):
X_f_r = np.array(np.ma.mean(np.ma.masked_outside(X_f_r, 0.03, 0.5)[:,:], axis=1))
X_f_r = np.clip(X_f_r, params['fatality_rate_lower'], params['fatality_rate_upper'])
 X_c = np.clip(np.exp(X_c)-1, 0, None)
 preds = X_f.copy()
train_size = X_f.shape[1] - 1
for i in range(test_size):
t_lag = train_size+i-params['length']
t_wsize = 5
d = np.diff(X_c, axis=1)[:, t_lag-t_wsize:t_lag+1+t_wsize]
delta = np.average(d, axis=1)
 delta = params['absolute growth'] + delta * X_f_r
preds = np.hstack(( preds, preds[:, -1].reshape(-1,1)+ delta.reshape(-1,1)))
 return preds
params = {
"length": 7,
"absolute growth": 0.02,
"fatality_rate_lower": 0.02,
"fatality_rate_upper": 0.3,
}
preds_f_1 = run_f(params, preds_c, X_f, f_rate.values[:,:TRAIN_N])
 preds_f_1 = np.log(1+preds_f_1 )
train_data['Size'].value_counts()<count_values>
class ZDatasetF(Dataset):
def __init__(self, X_c, X_f=None, hist_len=10):
self.X_c = X_c
self.X_f = X_f
self.hist_len = hist_len
self.is_test = X_f is None
def __len__(self):
return self.X_c.shape[1]
def __getitem__(self, idx):
if self.is_test:
return {'x_c':self.X_c[:, idx-self.hist_len:idx]}
else:
return {'x_c':self.X_c[:, idx-self.hist_len:idx],
'x_f':self.X_f[:, idx-1],
'y':np.log(1+self.X_f[:, idx])}
class PrLayer2(nn.Module):
def __init__(self, in_features1, in_features2):
super(PrLayer2, self ).__init__()
self.weight0 = Parameter(torch.Tensor(1, 1, in_features2))
self.weight1 = Parameter(torch.Tensor(1, in_features1, in_features2))
self.reset_parameters()
def reset_parameters(self):
init.kaiming_uniform_(self.weight0, a=math.sqrt(5))
init.kaiming_uniform_(self.weight1, a=math.sqrt(5))
def forward(self, input):
return input * torch.sigmoid(self.weight0 + self.weight1)
 class ZModelF(nn.Module):
def __init__(self, hist_len):
super(ZModelF, self ).__init__()
self.l_conv = PrLayer2(len(X_c),hist_len-1)
 def forward(self, x_c, x_f):
x = x_c[:,:,1:] - x_c[:,:,:-1]
res = torch.sum(self.l_conv(x), 2)
 return {'preds': torch.log(1 + x_f + res)}
class DummySampler(torch.utils.data.sampler.Sampler):
def __init__(self, idx):
self.idx = idx
def __iter__(self):
return iter(self.idx)
 def __len__(self):
return len(self.idx)
 def _smooth_l1_loss(target):
t = torch.abs(target)
 t = torch.where(t < 1, 0.5 * t ** 2, t - 0.5)
 return torch.mean(t)
 n_epochs = 5000
lr = 0.18
bag_size = 4
device = 'cpu'
hist_len = 14
loss_func = torch.nn.MSELoss()
reg_loss_func = _smooth_l1_loss
reg_factor = 0.035
train_dataset = ZDatasetF(np.exp(X_c)-1, X_f, hist_len=hist_len)
 test_dataset = ZDatasetF(np.exp(preds_c)-1, hist_len=hist_len)
 trn_idx = np.arange(hist_len+1, len(train_dataset))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(trn_idx)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=len(trn_idx), sampler=train_sampler, num_workers=0, pin_memory=True)
 test_idx = np.arange(TRAIN_N, len(test_dataset))
test_sampler = DummySampler(test_idx)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, sampler=test_sampler, num_workers=0, pin_memory=True)
 gradient_accumulation = 1
preds_f = 0
for m_i in range(bag_size):
model_f = ZModelF(hist_len=hist_len ).to(device)
 optimizer_f = torch.optim.Adam(model_f.parameters() , lr=lr)
 model_f.train()
start_time = time.time()
for epoch in range(n_epochs):
s = time.time()
avg_train_loss = 0
optimizer_f.zero_grad()
for idx, data in enumerate(train_loader):
X1 = data['x_c'].to(device ).float()
X2 = data['x_f'].to(device ).float()
y = data['y'].to(device ).float()
preds = model_f(X1, X2)['preds'].float()
cond = X2 > np.log(10)
 preds = preds[cond]
y = y[cond]
loss = loss_func(preds, y)
 loss += reg_factor * reg_loss_func(model_f.l_conv.weight1)
 avg_train_loss += loss / len(train_loader)
 loss.backward()
if(idx+1)% gradient_accumulation == 0 or idx == len(train_loader)- 1:
optimizer_f.step()
optimizer_f.zero_grad()
if epoch % 1000 == 0:
model_f.eval()
preds_f_delta = train_p_f.values[:,:TRAIN_N]
for idx, data in enumerate(test_loader):
X1 = data['x_c'].to(device ).float()
temp = model_f(X1, torch.Tensor(preds_f_delta[:,-1] ).unsqueeze(0)) ['preds']
temp = np.exp(temp.detach().cpu().numpy().reshape(-1,1)) - 1
preds_f_delta = np.hstack(( preds_f_delta, temp))
preds_f_delta = np.log(1 + preds_f_delta)
 model_f.train()
model_f.eval()
preds_f_delta = train_p_f.values[:,:TRAIN_N]
for idx, data in enumerate(test_loader):
X1 = data['x_c'].to(device ).float()
temp = model_f(X1, torch.Tensor(preds_f_delta[:,-1] ).unsqueeze(0)) ['preds']
temp = np.exp(temp.detach().cpu().numpy().reshape(-1,1)) - 1
preds_f_delta = np.hstack(( preds_f_delta, temp))
preds_f += preds_f_delta / bag_size
preds_f_2 = np.log(1 + preds_f)
 print("Done")
test_data['Size'].value_counts()<data_type_conversions>
preds_f = np.mean([preds_f_1, preds_f_2], axis=0 )
train_data['Size'].fillna(value = 'Size_Unknown', inplace = True)
 test_data['Size'].fillna(value = 'Size_Unknown', inplace = True )<count_values>
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, d]), preds_c[:, d]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, d]), preds_f[:, d]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c[:, TRAIN_N:TRAIN_N+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )
train_data['TopThreeAmericanName'].value_counts()<count_values>
def get_cpmp_sub(save_oof=False, save_public_test=False):
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/train.csv')
 train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test
day_min = train['day'].min()
train['day'] -= day_min
test['day'] -= day_min
train = train[train.day < TRAIN_N]
min_test_val_day = test.day.min()
max_test_val_day = train.day.max()
max_test_day = test.day.max()
num_days = max_test_day + 1
min_test_val_day, max_test_val_day, num_days
train['ForecastId'] = -1
test['Id'] = -1
test['ConfirmedCases'] = 0
test['Fatalities'] = 0
debug = False
data = pd.concat([train,
test[test.day > max_test_val_day][train.columns]
] ).reset_index(drop=True)
 if debug:
data = data[data['geo'] >= 'France_'].reset_index(drop=True)
 gc.collect()
dates = data[data['geo'] == 'France_'].Date.values
if 0:
gr = data.groupby('geo')
 data['ConfirmedCases'] = gr.ConfirmedCases.transform('cummax')
 data['Fatalities'] = gr.Fatalities.transform('cummax')
 geo_data = data.pivot(index='geo', columns='day', values='ForecastId')
 num_geo = geo_data.shape[0]
geo_data
geo_id = {}
for i,g in enumerate(geo_data.index):
geo_id[g] = i
ConfirmedCases = data.pivot(index='geo', columns='day', values='ConfirmedCases')
 Fatalities = data.pivot(index='geo', columns='day', values='Fatalities')
 if debug:
cases = ConfirmedCases.values
deaths = Fatalities.values
else:
cases = np.log1p(ConfirmedCases.values)
 deaths = np.log1p(Fatalities.values)
 def get_dataset(start_pred, num_train, lag_period):
days = np.arange(start_pred - num_train + 1, start_pred + 1)
 lag_cases = np.vstack([cases[:, d - lag_period : d] for d in days])
 lag_deaths = np.vstack([deaths[:, d - lag_period : d] for d in days])
 target_cases = np.vstack([cases[:, d : d + 1] for d in days])
 target_deaths = np.vstack([deaths[:, d : d + 1] for d in days])
 geo_ids = np.vstack([geo_ids_base for d in days])
 country_ids = np.vstack([country_ids_base for d in days])
 return lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days
def update_valid_dataset(data, pred_death, pred_case):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
day = days[-1] + 1
new_lag_cases = np.hstack([lag_cases[:, 1:], pred_case])
 new_lag_deaths = np.hstack([lag_deaths[:, 1:], pred_death])
 new_target_cases = cases[:, day:day+1]
new_target_deaths = deaths[:, day:day+1]
new_geo_ids = geo_ids
new_country_ids = country_ids
new_days = 1 + days
return new_lag_cases, new_lag_deaths, new_target_cases, new_target_deaths, new_geo_ids, new_country_ids, new_days
def fit_eval(lr_death, lr_case, data, start_lag_death, end_lag_death, num_lag_case, fit, score):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], country_ids])
 X_death = np.hstack([lag_deaths[:, -num_lag_case:], country_ids])
 X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], lag_deaths[:, -num_lag_case:], country_ids])
 y_death = target_deaths
y_death_prev = lag_deaths[:, -1:]
if fit:
if 0:
keep =(y_death > 0 ).ravel()
X_death = X_death[keep]
y_death = y_death[keep]
y_death_prev = y_death_prev[keep]
lr_death.fit(X_death, y_death)
 y_pred_death = lr_death.predict(X_death)
 y_pred_death = np.maximum(y_pred_death, y_death_prev)
 X_case = np.hstack([lag_cases[:, -num_lag_case:], geo_ids])
 X_case = lag_cases[:, -num_lag_case:]
y_case = target_cases
y_case_prev = lag_cases[:, -1:]
if fit:
lr_case.fit(X_case, y_case)
 y_pred_case = lr_case.predict(X_case)
 y_pred_case = np.maximum(y_pred_case, y_case_prev)
 if score:
death_score = val_score(y_death, y_pred_death)
 case_score = val_score(y_case, y_pred_case)
 else:
death_score = 0
case_score = 0
return death_score, case_score, y_pred_death, y_pred_case
def train_model(train, valid, start_lag_death, end_lag_death, num_lag_case, num_val, score=True):
alpha = 3
lr_death = Ridge(alpha=alpha, fit_intercept=False)
 lr_case = Ridge(alpha=alpha, fit_intercept=True)
(train_death_score, train_case_score, train_pred_death, train_pred_case, 
)= fit_eval(lr_death, lr_case, train, start_lag_death, end_lag_death, num_lag_case, fit=True, score=score)
 death_scores = []
case_scores = []
death_pred = []
case_pred = []
for i in range(num_val): 
(valid_death_score, valid_case_score, valid_pred_death, valid_pred_case, 
)= fit_eval(lr_death, lr_case, valid, start_lag_death, end_lag_death, num_lag_case, fit=False, score=score)
 death_scores.append(valid_death_score)
 case_scores.append(valid_case_score)
 death_pred.append(valid_pred_death)
 case_pred.append(valid_pred_case)
 if 0:
print('val death: %0.3f' % valid_death_score,
'val case: %0.3f' % valid_case_score,
'val : %0.3f' % np.mean([valid_death_score, valid_case_score]),
flush=True)
 valid = update_valid_dataset(valid, valid_pred_death, valid_pred_case)
 if score:
death_scores = np.sqrt(np.mean([s**2 for s in death_scores]))
case_scores = np.sqrt(np.mean([s**2 for s in case_scores]))
if 0:
print('train death: %0.3f' % train_death_score,
'train case: %0.3f' % train_case_score,
'val death: %0.3f' % death_scores,
'val case: %0.3f' % case_scores,
'val : %0.3f' %(( death_scores + case_scores)/ 2),
flush=True)
 else:
print('%0.4f' % case_scores,
', %0.4f' % death_scores,
'= %0.4f' %(( death_scores + case_scores)/ 2),
flush=True)
 death_pred = np.hstack(death_pred)
 case_pred = np.hstack(case_pred)
 return death_scores, case_scores, death_pred, case_pred
countries = [g.split('_')[0] for g in geo_data.index]
countries = pd.factorize(countries)[0]
country_ids_base = countries.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 country_ids_base = 0.2 * ohe.fit_transform(country_ids_base)
 country_ids_base.shape
geo_ids_base = np.arange(num_geo ).reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 geo_ids_base = 0.1 * ohe.fit_transform(geo_ids_base)
 geo_ids_base.shape
def val_score(true, pred):
pred = np.log1p(np.round(np.expm1(pred)- 0.2))
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def val_score(true, pred):
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
start_lag_death, end_lag_death = 14, 6,
num_train = 5
num_lag_case = 14
lag_period = max(start_lag_death, num_lag_case)
 def get_oof(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[start_val], start_val, num_val)
 train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = train[['Date', 'Id', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[sub.day >= start_val]
sub = sub[['Id', 'ConfirmedCases', 'Fatalities']].copy()
return sub
if save_oof:
for start_val_delta, date in zip(range(3, -8, -3),
['2020-03-22', '2020-03-19', '2020-03-16', '2020-03-13']):
print(date, end=' ')
 oof = get_oof(start_val_delta)
 oof.to_csv('.. /submissions/cpmp-%s.csv' % date, index=None)
 def get_sub(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = 14
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub.fillna(0)
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
return sub
return sub
known_test = train[['geo', 'day', 'ConfirmedCases', 'Fatalities']
].merge(test[['geo', 'day', 'ForecastId']], how='left', on=['geo', 'day'])
 known_test = known_test[['ForecastId', 'ConfirmedCases', 'Fatalities']][known_test.ForecastId.notnull() ].copy()
known_test
unknow_test = test[test.day > max_test_val_day]
unknow_test
def get_final_sub() :
start_val = max_test_val_day + 1
last_train = start_val - 1
num_val = max_test_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = num_val + 3
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
(_, _, val_death_preds, val_case_preds 
)= train_model(train_data, valid_data, start_lag_death, end_lag_death, num_lag_case, num_val, score=False)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
print(unknow_test.shape, pred_deaths.shape, pred_cases.shape)
 sub = unknow_test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
sub = pd.concat([known_test, sub])
 return sub
if save_public_test:
sub = get_sub()
else:
sub = get_final_sub()
return sub
test_data['TopThreeAmericanName'].value_counts()<data_type_conversions>
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
m1s = []
m2s = []
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, d]), preds_c_cpmp[:, d-START_PUBLIC]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, d]), preds_f_cpmp[:, d-START_PUBLIC]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 m1s += [m1]
m2s += [m2]
print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c_cpmp[:, TRAIN_N-START_PUBLIC:TRAIN_N-START_PUBLIC+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f_cpmp[:, TRAIN_N-START_PUBLIC:TRAIN_N-START_PUBLIC+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
train_data['TopThreeAmericanName'].fillna(value = 'TopThreeAmericanName_Unknown', inplace = True)
 test_data['TopThreeAmericanName'].fillna(value = 'TopThreeAmericanName_Unknown', inplace = True )<count_values>
COMP = '.. /input/covid19-global-forecasting-week-3'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '' ).fillna('N/A')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '' ).fillna('N/A')
 train['Location'] = train['Country_Region'].fillna('')+ '-' + train['Province_State'].fillna('N/A')
 test['Location'] = test['Country_Region'].fillna('')+ '-' + test['Province_State'].fillna('N/A')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 country_codes = pd.read_csv('.. /input/covid19-metadata/country_codes.csv', keep_default_na=False)
 train = train.merge(country_codes, on='Country_Region', how='left')
 test = test.merge(country_codes, on='Country_Region', how='left')
 train['continent'] = train['continent'].fillna('')
 test['continent'] = test['continent'].fillna('')
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in tqdm(df.groupby('Location')) :
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1
def get_beluga_sub() :
train, test, submission = get_comp_data(COMP)
 train.shape, test.shape, submission.shape
TRAIN_START = train.Date.min()
TEST_START = test.Date.min()
TRAIN_END = train.Date.max()
TEST_END = test.Date.max()
TRAIN_START, TRAIN_END, TEST_START, TEST_END
train_clean = process_each_location(train)
 train_clean['Geo
DECAY = 0.93
DECAY ** 7, DECAY ** 14, DECAY ** 21, DECAY ** 28
confirmed_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
confirmed_deltas['DELTA'] = GLOBAL_DELTA
confirmed_deltas.loc[confirmed_deltas.continent=='Africa', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.continent=='Oceania', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Korea South', 'DELTA'] = 0.011
confirmed_deltas.loc[confirmed_deltas.Country_Region=='US', 'DELTA'] = 0.15
confirmed_deltas.loc[confirmed_deltas.Country_Region=='China', 'DELTA'] = 0.01
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Japan', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Norway', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iceland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Austria', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Italy', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Spain', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Portugal', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Israel', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iran', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Germany', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Russia', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Brazil', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Turkey', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Philippines', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Location=='France-N/A', 'DELTA'] = 0.1
confirmed_deltas.loc[confirmed_deltas.Location=='United Kingdom-N/A', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Location=='Diamond Princess-N/A', 'DELTA'] = 0.00
confirmed_deltas.loc[confirmed_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Location=='San Marino-N/A', 'DELTA'] = 0.03
confirmed_deltas.shape, confirmed_deltas.DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].shape, confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA]
confirmed_deltas.describe()
daily_log_confirmed = train_clean.pivot('Location', 'Date', 'LogConfirmed' ).reset_index()
daily_log_confirmed = daily_log_confirmed.sort_values(TRAIN_END, ascending=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in confirmed_deltas.Location.values:
confirmed_delta = confirmed_deltas.loc[confirmed_deltas.Location == loc, 'DELTA'].values[0]
daily_log_confirmed.loc[daily_log_confirmed.Location == loc, new_day] = daily_log_confirmed.loc[daily_log_confirmed.Location == loc, last_day] + \
confirmed_delta * DECAY ** i
confirmed_prediciton = pd.melt(daily_log_confirmed, id_vars='Location')
 confirmed_prediciton['ConfirmedCases'] = to_exp(confirmed_prediciton['value'])
 confirmed_prediciton_cases = confirmed_prediciton.copy()
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_death_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_death_deltas = latest.merge(daily_death_deltas, on='Geo
death_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
death_deltas['DELTA'] = GLOBAL_DELTA
death_deltas.loc[death_deltas.Country_Region=='China', 'DELTA'] = 0.005
death_deltas.loc[death_deltas.continent=='Oceania', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Country_Region=='Korea South', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Japan', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='US', 'DELTA'] = 0.17
death_deltas.loc[death_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Norway', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Iceland', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Country_Region=='Austria', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Italy', 'DELTA'] = 0.07
death_deltas.loc[death_deltas.Country_Region=='Spain', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Portugal', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Israel', 'DELTA'] = 0.16
death_deltas.loc[death_deltas.Country_Region=='Iran', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='Germany', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Russia', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Brazil', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Turkey', 'DELTA'] = 0.22
death_deltas.loc[death_deltas.Country_Region=='Philippines', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Location=='France-N/A', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='United Kingdom-N/A', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='Diamond Princess-N/A', 'DELTA'] = 0.00
death_deltas.loc[death_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Location=='San Marino-N/A', 'DELTA'] = 0.05
death_deltas.shape
death_deltas.DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].shape
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA]
death_deltas.describe()
daily_log_deaths = train_clean.pivot('Location', 'Date', 'LogFatalities' ).reset_index()
daily_log_deaths = daily_log_deaths.sort_values(TRAIN_END, ascending=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in death_deltas.Location:
death_delta = death_deltas.loc[death_deltas.Location == loc, 'DELTA'].values[0]
daily_log_deaths.loc[daily_log_deaths.Location == loc, new_day] = daily_log_deaths.loc[daily_log_deaths.Location == loc, last_day] + \
death_delta * DECAY ** i
confirmed_prediciton = pd.melt(daily_log_deaths, id_vars='Location')
 confirmed_prediciton['Fatalities'] = to_exp(confirmed_prediciton['value'])
 confirmed_prediciton_fatalities = confirmed_prediciton.copy()
return confirmed_prediciton_cases, confirmed_prediciton_fatalities
preds_beluga_cases, preds_beluga_fatalities = get_beluga_sub()
train_data['PRIMEUNIT'].value_counts()<count_values>
locs = p_f_beluga.index.values
test_data['PRIMEUNIT'].value_counts()<data_type_conversions>
warnings.filterwarnings('ignore' )
train_data['PRIMEUNIT'].fillna(value = 'PRIMEUNIT_Unknown', inplace = True)
 test_data['PRIMEUNIT'].fillna(value = 'PRIMEUNIT_Unknown', inplace = True )<count_values>
FIRST_TEST = test_orig['Date'].apply(lambda x: x.dayofyear ).min()
train_data['AUCGUART'].value_counts()<count_values>
test_data['AUCGUART'].value_counts()<data_type_conversions>
def do_aggregation(df, col, mean_range):
df_new = copy.deepcopy(df)
 col_new = '{}_({}-{})'.format(col, mean_range[0], mean_range[1])
 df_new[col_new] = 0
tmp = df_new[col].rolling(mean_range[1]-mean_range[0]+1 ).mean()
df_new[col_new][mean_range[0]:] = tmp[:-(mean_range[0])]
df_new[col_new][pd.isna(df_new[col_new])] = 0
return df_new[[col_new]].reset_index(drop=True)
 def do_aggregations(df):
df = pd.concat([df, do_aggregation(df, 'cases/day', [1,1] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [1,7] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [8,14] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [15,21] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,1] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,7] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [8,14] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [15,21] ).reset_index(drop=True)], axis=1)
 for threshold in [1, 10, 100]:
days_under_threshold =(df['ConfirmedCases']<threshold ).sum()
tmp = df['day'].values - 22 - days_under_threshold
tmp[tmp<=0] = 0
df['days_since_{}cases'.format(threshold)] = tmp
for threshold in [1, 10, 100]:
days_under_threshold =(df['Fatalities']<threshold ).sum()
tmp = df['day'].values - 22 - days_under_threshold
tmp[tmp<=0] = 0
df['days_since_{}fatal'.format(threshold)] = tmp
if df['place_id'][0]=='China/Hubei':
df['days_since_1cases'] += 35
df['days_since_10cases'] += 35-13
df['days_since_100cases'] += 4
df['days_since_1fatal'] += 13
return df
train_data['AUCGUART'].fillna(value = 'AUCGUART_Unknown', inplace = True)
 test_data['AUCGUART'].fillna(value = 'AUCGUART_Unknown', inplace = True )<drop_column>
def feature_engineering_oscii() :
def fix_area(x):
try:
x_new = x['Country_Region'] + "/" + x['Province_State']
except:
x_new = x['Country_Region']
return x_new
def fix_area2(x):
try:
x_new = x['Country/Region'] + "/" + x['Province/State']
except:
x_new = x['Country/Region']
return x_new
def encode_label(df, col, freq_limit=0):
df[col][pd.isna(df[col])] = 'nan'
tmp = df[col].value_counts()
cols = tmp.index.values
freq = tmp.values
num_cols =(freq>=freq_limit ).sum()
print("col: {}, num_cat: {}, num_reduced: {}".format(col, len(cols), num_cols))
col_new = '{}_le'.format(col)
 df_new = pd.DataFrame(np.ones(len(df), np.int16)*(num_cols-1), columns=[col_new])
 for i, item in enumerate(cols[:num_cols]):
df_new[col_new][df[col]==item] = i
return df_new
def get_df_le(df, col_index, col_cat):
df_new = df[[col_index]]
for col in col_cat:
df_tmp = encode_label(df, col)
 df_new = pd.concat([df_new, df_tmp], axis=1)
 return df_new
def to_float(x):
x_new = 0
try:
x_new = float(x.replace(",", ""))
except:
x_new = np.nan
return x_new
df_train = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv")
 df_test = pd.read_csv(".. /input/covid19-global-forecasting-week-3/test.csv")
 df_traintest = pd.concat([df_train, df_test])
 print('process_date')
 df_traintest['Date'] = pd.to_datetime(df_traintest['Date'])
 df_traintest['day'] = df_traintest['Date'].apply(lambda x: x.dayofyear ).astype(np.int16)
 day_min = df_traintest['day'].min()
df_traintest['days'] = df_traintest['day'] - day_min
df_traintest.loc[df_traintest['Province_State'].isnull() , 'Province_State'] = 'N/A'
df_traintest['place_id'] = df_traintest['Country_Region'] + '_' + df_traintest['Province_State']
print('add lat and long')
 df_latlong = pd.read_csv(".. /input/smokingstats/df_Latlong.csv")
 df_latlong.loc[df_latlong['Province/State'].isnull() , 'Province_State'] = 'N/A'
df_latlong['place_id'] = df_latlong['Country/Region'] + '_' + df_latlong['Province/State']
df_latlong = df_latlong[df_latlong['place_id'].duplicated() ==False]
df_traintest = pd.merge(df_traintest, df_latlong[['place_id', 'Lat', 'Long']], on='place_id', how='left')
 places = np.sort(df_traintest['place_id'].unique())
 print('calc cases, fatalities per day')
 df_traintest2 = copy.deepcopy(df_traintest)
 df_traintest2['cases/day'] = 0
df_traintest2['fatal/day'] = 0
tmp_list = np.zeros(len(df_traintest2))
for place in places:
tmp = df_traintest2['ConfirmedCases'][df_traintest2['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest2['cases/day'][df_traintest2['place_id']==place] = tmp
tmp = df_traintest2['Fatalities'][df_traintest2['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest2['fatal/day'][df_traintest2['place_id']==place] = tmp
print('do agregation')
 df_traintest3 = []
for place in places[:]:
df_tmp = df_traintest2[df_traintest2['place_id']==place].reset_index(drop=True)
 df_tmp = do_aggregations(df_tmp)
 df_traintest3.append(df_tmp)
 df_traintest3 = pd.concat(df_traintest3 ).reset_index(drop=True)
 print('add smoking')
 df_smoking = pd.read_csv(".. /input/smokingstats/share-of-adults-who-smoke.csv")
 df_smoking_recent = df_smoking.sort_values('Year', ascending=False ).reset_index(drop=True)
 df_smoking_recent = df_smoking_recent[df_smoking_recent['Entity'].duplicated() ==False]
df_smoking_recent['Country_Region'] = df_smoking_recent['Entity']
df_smoking_recent['SmokingRate'] = df_smoking_recent['Smoking prevalence, total(ages 15+ )(% of adults)']
df_traintest4 = pd.merge(df_traintest3, df_smoking_recent[['Country_Region', 'SmokingRate']], on='Country_Region', how='left')
 SmokingRate = df_smoking_recent['SmokingRate'][df_smoking_recent['Entity']=='World'].values[0]
df_traintest4['SmokingRate'][pd.isna(df_traintest4['SmokingRate'])] = SmokingRate
print('add data from World Economic Outlook Database')
 df_weo = pd.read_csv(".. /input/smokingstats/WEO.csv")
 subs = df_weo['Subject Descriptor'].unique() [:-1]
df_weo_agg = df_weo[['Country']][df_weo['Country'].duplicated() ==False].reset_index(drop=True)
 for sub in subs[:]:
df_tmp = df_weo[['Country', '2019']][df_weo['Subject Descriptor']==sub].reset_index(drop=True)
 df_tmp = df_tmp[df_tmp['Country'].duplicated() ==False].reset_index(drop=True)
 df_tmp.columns = ['Country', sub]
df_weo_agg = df_weo_agg.merge(df_tmp, on='Country', how='left')
 df_weo_agg.columns = ["".join(c if c.isalnum() else "_" for c in str(x)) for x in df_weo_agg.columns]
df_weo_agg.columns
df_weo_agg['Country_Region'] = df_weo_agg['Country']
df_traintest5 = pd.merge(df_traintest4, df_weo_agg, on='Country_Region', how='left')
 print('add Life expectancy')
 df_life = pd.read_csv(".. /input/smokingstats/Life expectancy at birth.csv")
 tmp = df_life.iloc[:,1].values.tolist()
df_life = df_life[['Country', '2018']]
def func(x):
x_new = 0
try:
x_new = float(x.replace(",", ""))
except:
x_new = np.nan
return x_new
df_life['2018'] = df_life['2018'].apply(lambda x: func(x))
df_life = df_life[['Country', '2018']]
df_life.columns = ['Country_Region', 'LifeExpectancy']
df_traintest6 = pd.merge(df_traintest5, df_life, on='Country_Region', how='left')
 print("add additional info from countryinfo dataset")
 df_country = pd.read_csv(".. /input/countryinfo/covid19countryinfo.csv")
 df_country['Country_Region'] = df_country['country']
df_country = df_country[df_country['country'].duplicated() ==False]
df_traintest7 = pd.merge(df_traintest6,
df_country.drop(['tests', 'testpop', 'country'], axis=1),
on=['Country_Region',], how='left')
 df_traintest7['id'] = np.arange(len(df_traintest7))
df_le = get_df_le(df_traintest7, 'id', ['Country_Region', 'Province_State'])
 df_traintest8 = pd.merge(df_traintest7, df_le, on='id', how='left')
 print('covert object type to float')
 cols = [
'Gross_domestic_product__constant_prices',
'Gross_domestic_product__current_prices',
'Gross_domestic_product__deflator',
'Gross_domestic_product_per_capita__constant_prices',
'Gross_domestic_product_per_capita__current_prices',
'Output_gap_in_percent_of_potential_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__valuation_of_country_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__per_capita_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__share_of_world_total',
'Implied_PPP_conversion_rate', 'Total_investment',
'Gross_national_savings', 'Inflation__average_consumer_prices',
'Inflation__end_of_period_consumer_prices',
'Six_month_London_interbank_offered_rate__LIBOR_',
'Volume_of_imports_of_goods_and_services',
'Volume_of_Imports_of_goods',
'Volume_of_exports_of_goods_and_services',
'Volume_of_exports_of_goods', 'Unemployment_rate', 'Employment', 'Population',
'General_government_revenue', 'General_government_total_expenditure',
'General_government_net_lending_borrowing', 'General_government_structural_balance',
'General_government_primary_net_lending_borrowing', 'General_government_net_debt',
'General_government_gross_debt', 'Gross_domestic_product_corresponding_to_fiscal_year__current_prices',
'Current_account_balance', 'pop'
]
df_traintest8['cases/day'] = df_traintest8['cases/day'].astype(np.float)
 df_traintest8['fatal/day'] = df_traintest8['fatal/day'].astype(np.float)
 for col in cols:
df_traintest8[col] = df_traintest8[col].apply(lambda x: to_float(x))
return df_traintest8
train_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1)
 test_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1 )<count_values>
def calc_score(y_true, y_pred):
y_true[y_true<0] = 0
score = mean_squared_error(np.log(y_true.clip(0, 1e10)+1), np.log(y_pred[:]+1)) **0.5
return score
train_data['PurchDate'].value_counts()<drop_column>
SEED = 42
params = {'num_leaves': 8,
'min_data_in_leaf': 5,
'objective': 'regression',
'max_depth': 8,
'learning_rate': 0.02,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8201,
'bagging_seed': SEED,
'reg_alpha': 1,
'reg_lambda': 4.9847051755586085,
'random_state': SEED,
'metric': 'mse',
'verbosity': 100,
'min_gain_to_split': 0.02,
'min_child_weight': 5,
'num_threads': 6,
}
train_data.drop('PurchDate', axis = 1, inplace = True)
 test_data.drop('PurchDate', axis = 1, inplace = True )<drop_column>
col_target = 'fatal/day'
col_var = [
'Lat', 'Long',
'cases/day_(1-1)',
'cases/day_(1-7)',
'fatal/day_(1-7)',
'fatal/day_(8-14)',
'fatal/day_(15-21)',
'SmokingRate',
'density',
]
col_cat = []
df_train = df_traintest[(pd.isna(df_traintest['ForecastId'])) &(df_traintest['days']<TRAIN_N)]
df_valid = df_traintest[(pd.isna(df_traintest['ForecastId'])) &(df_traintest['days']<TRAIN_N)]
X_train = df_train[col_var]
X_valid = df_valid[col_var]
y_train = np.log(df_train[col_target].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 num_round = 340
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 best_itr = model.best_iteration
train_data.drop(['RefId', 'IsBadBuy'], axis = 1 ).dtypes != 'object'<drop_column>
col_target2 = 'cases/day'
col_var2 = [
'Lat', 'Long',
'days_since_10cases',
'cases/day_(1-1)',
'cases/day_(1-7)',
'cases/day_(8-14)',
'cases/day_(15-21)',
]
not_categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object']<feature_engineering>
df_train = df_traintest[(pd.isna(df_traintest['ForecastId'])) &(df_traintest['days']<TRAIN_N)]
df_valid = df_traintest[(pd.isna(df_traintest['ForecastId'])) &(df_traintest['days']<TRAIN_N)]
X_train = df_train[col_var2]
X_valid = df_valid[col_var2]
y_train = np.log(df_train[col_target2].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target2].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 best_itr2 = model2.best_iteration
for i in not_categorical:
maximum=np.max(train_data[i])
 train_data[i]=train_data[i]/maximum
maximum_test=np.max(test_data[i])
 test_data[i]=test_data[i]/maximum_test<drop_column>
places = AREAS.copy()
categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes=='object']<filter>
df_tmp = df_traintest[(( df_traintest['days']<TRAIN_N)&(pd.isna(df_traintest['ForecastId'])))|(( TRAIN_N<=df_traintest['days'])&(pd.isna(df_traintest['ForecastId'])==False)) ].reset_index(drop=True)
 df_tmp = df_tmp.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_traintest9 = []
for i, place in tqdm(enumerate(places[:])) :
df_tmp2 = df_tmp[df_tmp['place_id']==place].reset_index(drop=True)
 df_tmp2 = do_aggregations(df_tmp2)
 df_traintest9.append(df_tmp2)
 df_traintest9 = pd.concat(df_traintest9 ).reset_index(drop=True)
train_data[categorical[0]]<categorify>
df_preds = []
for i, place in tqdm(enumerate(places[:])) :
df_interest = copy.deepcopy(df_traintest9[df_traintest9['place_id']==place].reset_index(drop=True))
df_interest['cases/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
df_interest['fatal/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
len_known =(df_interest['days']<TRAIN_N ).sum()
len_unknown = 30
for j in range(len_unknown):
X_valid = df_interest[col_var].iloc[j+len_known]
X_valid2 = df_interest[col_var2].iloc[j+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 pred_c =(np.exp(pred_c)-1 ).clip(0, 1e10)
 pred_f =(np.exp(pred_f)-1 ).clip(0, 1e10)
 df_interest['fatal/day'][j+len_known] = pred_f
df_interest['cases/day'][j+len_known] = pred_c
df_interest['Fatalities'][j+len_known] = df_interest['Fatalities'][j+len_known-1] + pred_f
df_interest['ConfirmedCases'][j+len_known] = df_interest['ConfirmedCases'][j+len_known-1] + pred_c
df_interest = df_interest.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_interest = do_aggregations(df_interest)
 if(i+1)%10==0:
print("{:3d}/{} {}, len known: {}, len unknown: {}".format(i+1, len(places), place, len_known, len_unknown), df_interest.shape)
 df_interest['fatal_pred'] = np.cumsum(df_interest['fatal/day'].values)
 df_interest['cases_pred'] = np.cumsum(df_interest['cases/day'].values)
 df_preds.append(df_interest)
 df_preds = pd.concat(df_preds )
pd.get_dummies(train_data[categorical[0]] )<categorify>
preds_f_oscii = np.log1p(p_f_oscii.values[:].copy())
 preds_c_oscii = np.log1p(p_c_oscii.values[:].copy())
 preds_f_oscii.shape, preds_c_oscii.shape
for i in categorical:
dummies=pd.get_dummies(train_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
train_data=pd.concat([train_data,dummies],axis=1)
 train_data.drop(i,inplace=True,axis=1)
 dummies=pd.get_dummies(test_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
test_data=pd.concat([test_data,dummies],axis=1)
 test_data.drop(i,inplace=True,axis=1 )<feature_engineering>
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
m1s = []
m2s = []
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, d]), preds_c_oscii[:, d-START_PUBLIC]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, d]), preds_f_oscii[:, d-START_PUBLIC]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 m1s += [m1]
m2s += [m2]
print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c_oscii[:, TRAIN_N-START_PUBLIC:TRAIN_N-START_PUBLIC+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f_oscii[:, TRAIN_N-START_PUBLIC:TRAIN_N-START_PUBLIC+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )
for i in train_data.drop('IsBadBuy',axis=1 ).columns:
if i not in test_data.columns:
test_data[i]=np.zeros(len(test_data))<feature_engineering>
preds_c_blend = np.log1p(np.average([np.expm1(preds_c_oscii[:,64:107]),np.expm1(preds_c_cpmp[:]),np.expm1(p_c_beluga[:,64:107]),np.expm1(preds_c[:,64:107])],axis=0, weights=[8,1,1,8]))
preds_f_blend = np.log1p(np.average([np.expm1(preds_f_oscii[:,64:107]),np.expm1(preds_f_cpmp[:]),np.expm1(p_f_beluga[:,64:107]),np.expm1(preds_f[:,64:107])],axis=0, weights=[8,1,1,8]))
for i in test_data.columns:
if i not in train_data.columns:
train_data[i]=np.zeros(len(train_data))<drop_column>
val_len = 13
m1s = []
m2s = []
for i in range(val_len):
d = i + 64
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, d]), preds_c_blend[:, i]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, d]), preds_f_blend[:, i]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 m1s += [m1]
m2s += [m2]
print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, 64:64+val_len] ).flatten() , preds_c_blend[:, :val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, 64:64+val_len] ).flatten() , preds_f_blend[:, :val_len].flatten()))
print(f"PUBLIC LB {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )
test_data = test_data[train_data.drop("IsBadBuy",axis=1 ).columns]<prepare_x_and_y>
EU_COUNTRIES = ['Austria', 'Italy', 'Belgium', 'Latvia', 'Bulgaria', 'Lithuania', 'Croatia', 'Luxembourg', 'Cyprus', 'Malta', 'Czechia',
'Netherlands', 'Denmark', 'Poland', 'Estonia', 'Portugal', 'Finland', 'Romania', 'France', 'Slovakia', 'Germany', 'Slovenia',
'Greece', 'Spain', 'Hungary', 'Sweden', 'Ireland']
EUROPE_OTHER = ['Albania', 'Andorra', 'Bosnia and Herzegovina', 'Liechtenstein', 'Monaco', 'Montenegro', 'North Macedonia',
'Norway', 'San Marino', 'Serbia', 'Switzerland', 'Turkey', 'United Kingdom']
AFRICA = ['Algeria', 'Burkina Faso', 'Cameroon', 'Congo(Kinshasa)', "Cote d'Ivoire", 'Egypt', 'Ghana', 'Kenya', 'Madagascar',
'Morocco', 'Nigeria', 'Rwanda', 'Senegal', 'South Africa', 'Togo', 'Tunisia', 'Uganda', 'Zambia']
NORTH_AMERICA = ['US', 'Canada', 'Mexico']
SOUTH_AMERICA = ['Argentina', 'Bolivia', 'Brazil', 'Chile', 'Colombia', 'Ecuador', 'Paraguay', 'Peru', 'Uruguay', 'Venezuela']
MIDDLE_EAST = ['Afghanistan', 'Bahrain', 'Iran', 'Iraq', 'Israel', 'Jordan', 'Kuwait', 'Lebanon', 'Oman', 'Qatar', 'Saudi Arabia', 'United Arab Emirates']
ASIA = ['Bangladesh', 'Brunei', 'Cambodia', 'India', 'Indonesia', 'Japan', 'Kazakhstan', 'Korea, South', 'Kyrgyzstan', 'Malaysia',
'Pakistan', 'Singapore', 'Sri Lanka', 'Taiwan*', 'Thailand', 'Uzbekistan', 'Vietnam']
X=train_data.drop(['RefId','IsBadBuy'],axis=1)
 y=train_data['IsBadBuy']<split>
non_china_mask = np.array(['China' not in a for a in AREAS] ).astype(bool)
 non_china_mask.shape
X_train,X_test,y_train,y_test=train_test_split(X,y,random_state=42 )<import_modules>
preds_c2 = preds_c.copy()
preds_f2 = preds_f.copy()
preds_c2[non_china_mask,64:107] = preds_c_blend[non_china_mask]
preds_f2[non_china_mask,64:107] = preds_f_blend[non_china_mask]
from sklearn.neighbors import KNeighborsClassifier<import_modules>
temp = pd.DataFrame(np.clip(np.exp(preds_c2)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="ConfirmedCases")
 test = test_orig.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 temp = pd.DataFrame(np.clip(np.exp(preds_f2)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="Fatalities")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 test.head()
from sklearn.neighbors import KNeighborsClassifier<import_modules>
test.to_csv("submission.csv", index=False, columns=["ForecastId", "ConfirmedCases", "Fatalities"] )
from sklearn.neighbors import KNeighborsClassifier<train_model>
test.days.nunique()
KNN = KNeighborsClassifier(n_neighbors = 11, n_jobs = -1)
 KNN.fit(X_train,y_train )<compute_test_metric>
for i, rec in test.groupby('Area' ).last().sort_values("ConfirmedCases", ascending=False ).iterrows() :
print(f"{rec['ConfirmedCases']:10.1f} {rec['Fatalities']:10.1f} {rec['Country_Region']}, {rec['Province_State']}")