Kaggle competition: Homework for Students
death_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
death_deltas['DELTA'] = GLOBAL_DELTA
death_deltas.loc[death_deltas.Country_Region=='China', 'DELTA'] = 0.005
death_deltas.loc[death_deltas.continent=='Oceania', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Country_Region=='Korea South', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Japan', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='US', 'DELTA'] = 0.17
death_deltas.loc[death_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Norway', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Iceland', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Country_Region=='Austria', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Italy', 'DELTA'] = 0.07
death_deltas.loc[death_deltas.Country_Region=='Spain', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Portugal', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Israel', 'DELTA'] = 0.16
death_deltas.loc[death_deltas.Country_Region=='Iran', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='Germany', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Russia', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Brazil', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Turkey', 'DELTA'] = 0.22
death_deltas.loc[death_deltas.Country_Region=='Philippines', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Location=='France-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='United Kingdom-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
death_deltas.loc[death_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Location=='San Marino-', 'DELTA'] = 0.05
death_deltas.shape
death_deltas.DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].shape
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA]
death_deltas.describe()<save_to_csv>
df_train = pd.read_csv('.. /input/homework-for-students4plus/train.csv', index_col=0)
 df_test = pd.read_csv('.. /input/homework-for-students4plus/test.csv', index_col=0)
daily_log_deaths = train_clean.pivot('Location', 'Date', 'LogFatalities' ).reset_index()
daily_log_deaths = daily_log_deaths.sort_values(TRAIN_END, ascending=False)
 daily_log_deaths.to_csv('daily_log_deaths.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in death_deltas.Location:
death_delta = death_deltas.loc[death_deltas.Location == loc, 'DELTA'].values[0]
daily_log_deaths.loc[daily_log_deaths.Location == loc, new_day] = daily_log_deaths.loc[daily_log_deaths.Location == loc, last_day] + \
death_delta * DECAY ** i<feature_engineering>
df_train['FLG'] = 'train'
print('df_train' + str(df_train.shape))
df_test['FLG'] = 'test'
print('df_test' + str(df_test.shape))
df_temp = pd.concat([df_train,df_test])
 df_temp['issue_d_year'] = df_temp.issue_d.str[-4:].astype(int)
 df_temp = df_temp[df_temp.issue_d_year > 2010]
print('df_temp' + str(df_temp.shape))
confirmed = []
fatalities = []
for id, d, loc in tqdm(test[['ForecastId', 'Date', 'Location']].values):
c = to_exp(daily_log_confirmed.loc[daily_log_confirmed.Location == loc, d].values[0])
 f = to_exp(daily_log_deaths.loc[daily_log_deaths.Location == loc, d].values[0])
 confirmed.append(c)
 fatalities.append(f )<prepare_output>
df_temp['earliest_cr_line_year'] = df_temp.earliest_cr_line.str[-4:].astype(int)
 df_temp['year_difference'] = df_temp['issue_d_year'] - df_temp['earliest_cr_line_year']
my_submission = test.copy()
my_submission['ConfirmedCases'] = confirmed
my_submission['Fatalities'] = fatalities
my_submission.shape
my_submission.head() 
<save_to_csv>
col = 'sub_grade'
df_temp[col] = df_temp[col].str.replace('A1','0.5' ).str.replace('A2','2' ).str.replace('A3','3' ).str.replace('A4','4' ).str.replace('A5','5')\ 
.str.replace('B1','6' ).str.replace('B2','7' ).str.replace('B3','8' ).str.replace('B4','9' ).str.replace('B5','10')\ 
.str.replace('C1','11' ).str.replace('C2','12' ).str.replace('C3','13' ).str.replace('C4','14' ).str.replace('C5','15')\ 
.str.replace('D1','16' ).str.replace('D2','17' ).str.replace('D3','18' ).str.replace('D4','19' ).str.replace('D5','20')\ 
.str.replace('E1','21' ).str.replace('E2','22' ).str.replace('E3','23' ).str.replace('E4','24' ).str.replace('E5','25')\ 
.str.replace('F1','26' ).str.replace('F2','27' ).str.replace('F3','28' ).str.replace('F4','29' ).str.replace('F5','30')\ 
.str.replace('G1','31' ).str.replace('G2','32' ).str.replace('G3','33' ).str.replace('G4','34' ).str.replace('G5','35')
 df_temp[col] = pd.to_numeric(df_temp[col], errors='coerce')
 print(df_temp[col].unique() )
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 print(DECAY)
 my_submission.head()
my_submission.tail()
my_submission.shape<train_model>
print(df_temp.isnull().sum() )
end = dt.datetime.now()
print('Finished', end,(end - start ).seconds, 's' )<load_from_csv>
col = ['emp_title', 'emp_length','annual_inc', 'title','dti', 'delinq_2yrs', 'earliest_cr_line',
'inq_last_6mths', 'mths_since_last_delinq', 'mths_since_last_record','open_acc',
'pub_rec', 'revol_util', 'total_acc','collections_12_mths_ex_med','mths_since_last_major_derog',
'acc_now_delinq','tot_coll_amt', 'tot_cur_bal']
col2 = ['emp_title_flg', 'emp_length_flg','annual_inc_flg', 'title_flg','dti_flg', 'delinq_2yrs_flg', 'earliest_cr_line_flg',
'inq_last_6mths_flg', 'mths_since_last_delinq_flg', 'mths_since_last_record_flg','open_acc_flg',
'pub_rec_flg', 'revol_util_flg', 'total_acc_flg','collections_12_mths_ex_med_flg','mths_since_last_major_derog_flg',
'acc_now_delinq_flg','tot_coll_amt_flg', 'tot_cur_bal_flg']
df_temp[col2] = df_temp[col].copy()
for i in col2:
exec("df_temp." + str(i)+ "[df_temp." + str(i)+ ".notnull() ] = 1")
 exec("df_temp." + str(i)+ ".fillna(0, inplace=True)")
 print(df_temp.info() )
path = '.. /input/covid19-global-forecasting-week-3/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'submission.csv')
 train['Date'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 test['Date'] = test['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 train['days'] =(train['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
test['days'] =(test['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
train.loc[train['Province_State'].isnull() , 'Province_State'] = 'N/A'
test.loc[test['Province_State'].isnull() , 'Province_State'] = 'N/A'
train['Area'] = train['Country_Region'] + '_' + train['Province_State']
test['Area'] = test['Country_Region'] + '_' + test['Province_State']
print('train Date max',train['Date'].max())
 print('test Date min',test['Date'].min())
 print('train days max', train['days'].max())
 N_AREAS = train['Area'].nunique()
AREAS = np.sort(train['Area'].unique())
 START_PUBLIC = test['days'].min()
print('public LB start day', START_PUBLIC)
 print(' ')
 TRAIN_N = 77
print(train[train['days'] < TRAIN_N]['Date'].max())
 print(train[train['days'] >= TRAIN_N]['Date'].min())
 print(train[train['days'] >= TRAIN_N]['Date'].max())
 train.head()
test_orig = test.copy()<prepare_x_and_y>
col = 'emp_length'
df_temp[col] = df_temp[col].replace('[^0-9]+', '', regex=True)
 df_temp[col].fillna('0', inplace=True)
 df_temp[col] = pd.to_numeric(df_temp[col], errors='coerce')
 print(df_temp.info())
 print(df_temp[col].unique() )
train_p_c_raw = train.pivot(index='Area', columns='days', values='ConfirmedCases' ).sort_index()
train_p_f_raw = train.pivot(index='Area', columns='days', values='Fatalities' ).sort_index()
train_p_c = np.maximum.accumulate(train_p_c_raw, axis=1)
 train_p_f = np.maximum.accumulate(train_p_f_raw, axis=1)
 f_rate =(train_p_f / train_p_c ).fillna(0)
 X_c = np.log(1+train_p_c.values)[:,:TRAIN_N]
X_f = train_p_f.values[:,:TRAIN_N] 
<compute_train_metric>
df_temp['emp_length_max_flg'] = 1
df_temp['emp_length_max_flg'] = df_temp['emp_length_max_flg'].where(df_temp['emp_length']==10,0)
 df_temp['loan_amnt_max_flg'] = 1
df_temp['loan_amnt_max_flg'] = df_temp['loan_amnt_max_flg'].where(df_temp['loan_amnt']==35000,0)
 df_temp['revol_bal_0_flg'] = 1
df_temp['revol_bal_0_flg'] = df_temp['loan_amnt_max_flg'].where(df_temp['loan_amnt']==0,0)
 df_temp['tot_coll_amt_0_flg'] = 1
df_temp['tot_coll_amt_0_flg'] = df_temp['loan_amnt_max_flg'].where(df_temp['loan_amnt']==0,0 )
def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run_c(params, X, test_size=50):
gr_base = []
gr_base_factor = []
x_min = np.ma.MaskedArray(X, X<1)
 x_min = x_min.argmin(axis=1)
 for i in range(X.shape[0]):
temp = X[i,:]
threshold = np.log(1+params['min cases for growth rate'])
 num_days = params['last N days']
if(temp > threshold ).sum() > num_days:
d = np.diff(temp[temp > threshold])[-num_days:]
w = np.arange(len(d)) +1
w = w**5
w = w / np.sum(w)
 gr_base.append(np.clip(np.average(d, weights=w), 0, params['growth rate max']))
d2 = np.diff(d)
 w = np.arange(len(d2)) +1
w = w**10
w = w / np.sum(w)
 gr_base_factor.append(np.clip(np.average(d2, weights=w), -0.5, params["growth rate factor max"]))
else:
gr_base.append(params['growth rate default'])
 gr_base_factor.append(params['growth rate factor'])
 gr_base = np.array(gr_base)
 gr_base_factor = np.array(gr_base_factor)
 preds = X.copy()
for i in range(test_size):
delta = np.clip(preds[:, -1], np.log(2), None)+ gr_base *(1 + params['growth rate factor']*(1 + params['growth rate factor factor'])**(i)) **(np.log1p(i))
preds = np.hstack(( preds, delta.reshape(-1,1)))
 return preds
params = {
"min cases for growth rate": 0,
"last N days": 15,
"growth rate default": 0.2,
"growth rate max": 0.3,
"growth rate factor max": -0.1,
"growth rate factor": -0.3,
"growth rate factor factor": 0.01,
}
x = train_p_c
preds_c = run_c(params, np.log(1+x.values)[:,:TRAIN_N])
<feature_engineering>
df_temp['emp_title'] = df_temp['emp_title'].str.upper()
df_temp['emp_title_flg'] = 1
df_temp['emp_title_flg'] = df_temp['emp_title_flg'].where(df_temp['emp_title'].str.contains('MANAGER'),0)
 df_temp['emp_title_flg2'] = 1
df_temp['emp_title_flg2'] = df_temp['emp_title_flg2'].where(df_temp['emp_title'].str.contains('TEACHER'),0)
for i in range(N_AREAS):
if 'China' in AREAS[i] and preds_c[i, TRAIN_N-1] < np.log(31):
preds_c[i, TRAIN_N:] = preds_c[i, TRAIN_N-1] 
<normalization>
df_temp['zip_code_1'] = df_temp.zip_code.str[:1]
num_cols = ['loan_amnt','annual_inc','revol_bal','tot_cur_bal']
p01 = df_temp[num_cols].quantile(0.01)
 p99 = df_temp[num_cols].quantile(0.99)
 df_temp[num_cols] = df_temp[num_cols].clip(p01,p99,axis=1)
 labels=[10,9,8,7,6,5,4,3,2,1]
df_temp['loan_amnt_cut'] = pd.cut(df_temp.loan_amnt, 10, labels=labels ).astype(int)
 labels=[1,2,3,4,5,6,7,8,9,10]
df_temp['annual_inc_cut'] = pd.cut(df_temp.annual_inc, 10, labels=labels ).astype(int)
 df_temp['loan_risk'] = df_temp['loan_amnt_cut'] * df_temp['annual_inc_cut']
df_temp = df_temp.drop(columns=['loan_amnt_cut'])
 df_temp = df_temp.drop(columns=['annual_inc_cut'])
 df_temp['annual_inc_installment_ratio'] = df_temp['annual_inc'] / 12 / df_temp['installment']
df_temp['subgrade_loanamnt_mean'] = df_temp.groupby('sub_grade')['loan_amnt'].transform(lambda x: x.mean())
 df_temp['subgrade_loanamnt_std'] = df_temp.groupby('sub_grade')['loan_amnt'].transform(lambda x: x.std())
 df_temp['subgrade_loanamnt_var'] = df_temp.groupby('sub_grade')['loan_amnt'].transform(lambda x: x.var())
 df_temp['subgrade_loanamnt_difference'] = df_temp['subgrade_loanamnt_mean'] - df_temp['loan_amnt']
df_temp['sub_grade_annual_inc_mean'] = df_temp.groupby('sub_grade')['annual_inc'].transform(lambda x: x.mean())
 df_temp['sub_grade_annual_inc_std'] = df_temp.groupby('sub_grade')['annual_inc'].transform(lambda x: x.std())
 df_temp['sub_grade_annual_inc_var'] = df_temp.groupby('sub_grade')['annual_inc'].transform(lambda x: x.var())
 df_temp['sub_grade_annual_inc_difference'] = df_temp['loan_amnt'] / df_temp['sub_grade_annual_inc_mean']
df_temp['sub_grade_annual_inc_difference2'] = df_temp['sub_grade_annual_inc_mean'] / df_temp['loan_amnt']
df_temp['loan_amnt_sub_grade_ratio'] = df_temp['loan_amnt'] / df_temp['sub_grade']
df_temp['annual_inc_loanamnt_ratio'] = df_temp['annual_inc'] / df_temp['loan_amnt']
df_temp['loan_amnt_sub_grade_ratio'] = df_temp['loan_amnt'] / df_temp[col]
df_temp['loan_amnt_open_acc_ratio'] = df_temp['loan_amnt'] / df_temp['open_acc']
df_temp['annual_inc_sub_grade_ratio'] = df_temp['annual_inc'] / df_temp[col]
df_temp['annual_inc_open_acc_ratio'] = df_temp['annual_inc'] / df_temp['open_acc']
df_temp['revol_bal_sub_grade_ratio'] = df_temp['revol_bal'] / df_temp[col]
df_temp['revol_bal_open_acc_ratio'] = df_temp['revol_bal'] / df_temp['open_acc']
df_temp['tot_cur_bal_sub_grade_ratio'] = df_temp['tot_cur_bal'] / df_temp[col]
df_temp['tot_cur_bal_open_acc_ratio'] = df_temp['tot_cur_bal'] / df_temp['open_acc']
print(df_temp.shape)
 print(df_temp.info() )
def lin_w(sz):
res = np.linspace(0, 1, sz+1, endpoint=False)[1:]
return np.append(res, np.append([1], res[::-1]))
def run_f(params, X_c, X_f, X_f_r, test_size=50):
X_f_r = np.array(np.ma.mean(np.ma.masked_outside(X_f_r, 0.03, 0.5)[:,:], axis=1))
X_f_r = np.clip(X_f_r, params['fatality_rate_lower'], params['fatality_rate_upper'])
 X_c = np.clip(np.exp(X_c)-1, 0, None)
 preds = X_f.copy()
train_size = X_f.shape[1] - 1
for i in range(test_size):
t_lag = train_size+i-params['length']
t_wsize = 5
d = np.diff(X_c, axis=1)[:, t_lag-t_wsize:t_lag+1+t_wsize]
delta = np.average(d, axis=1)
 delta = params['absolute growth'] + delta * X_f_r
preds = np.hstack(( preds, preds[:, -1].reshape(-1,1)+ delta.reshape(-1,1)))
 return preds
params = {
"length": 7,
"absolute growth": 0.02,
"fatality_rate_lower": 0.02,
"fatality_rate_upper": 0.3,
}
preds_f_1 = run_f(params, preds_c, X_f, f_rate.values[:,:TRAIN_N])
 preds_f_1 = np.log(1+preds_f_1 )<set_options>
df_temp = pd.get_dummies(df_temp, columns=['purpose','grade','home_ownership','zip_code_1'])
 df_temp.shape
class ZDatasetF(Dataset):
def __init__(self, X_c, X_f=None, hist_len=10):
self.X_c = X_c
self.X_f = X_f
self.hist_len = hist_len
self.is_test = X_f is None
def __len__(self):
return self.X_c.shape[1]
def __getitem__(self, idx):
if self.is_test:
return {'x_c':self.X_c[:, idx-self.hist_len:idx]}
else:
return {'x_c':self.X_c[:, idx-self.hist_len:idx],
'x_f':self.X_f[:, idx-1],
'y':np.log(1+self.X_f[:, idx])}
class PrLayer2(nn.Module):
def __init__(self, in_features1, in_features2):
super(PrLayer2, self ).__init__()
self.weight0 = Parameter(torch.Tensor(1, 1, in_features2))
self.weight1 = Parameter(torch.Tensor(1, in_features1, in_features2))
self.reset_parameters()
def reset_parameters(self):
init.kaiming_uniform_(self.weight0, a=math.sqrt(5))
init.kaiming_uniform_(self.weight1, a=math.sqrt(5))
def forward(self, input):
return input * torch.sigmoid(self.weight0 + self.weight1)
 class ZModelF(nn.Module):
def __init__(self, hist_len):
super(ZModelF, self ).__init__()
self.l_conv = PrLayer2(len(X_c),hist_len-1)
 def forward(self, x_c, x_f):
x = x_c[:,:,1:] - x_c[:,:,:-1]
res = torch.sum(self.l_conv(x), 2)
 return {'preds': torch.log(1 + x_f + res)}
class DummySampler(torch.utils.data.sampler.Sampler):
def __init__(self, idx):
self.idx = idx
def __iter__(self):
return iter(self.idx)
 def __len__(self):
return len(self.idx)
 def _smooth_l1_loss(target):
t = torch.abs(target)
 t = torch.where(t < 1, 0.5 * t ** 2, t - 0.5)
 return torch.mean(t)
 n_epochs = 5000
lr = 0.18
bag_size = 4
device = 'cpu'
hist_len = 14
loss_func = torch.nn.MSELoss()
reg_loss_func = _smooth_l1_loss
reg_factor = 0.035
train_dataset = ZDatasetF(np.exp(X_c)-1, X_f, hist_len=hist_len)
 test_dataset = ZDatasetF(np.exp(preds_c)-1, hist_len=hist_len)
 trn_idx = np.arange(hist_len+1, len(train_dataset))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(trn_idx)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=len(trn_idx), sampler=train_sampler, num_workers=0, pin_memory=True)
 test_idx = np.arange(TRAIN_N, len(test_dataset))
test_sampler = DummySampler(test_idx)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, sampler=test_sampler, num_workers=0, pin_memory=True)
 gradient_accumulation = 1
preds_f = 0
for m_i in range(bag_size):
model_f = ZModelF(hist_len=hist_len ).to(device)
 optimizer_f = torch.optim.Adam(model_f.parameters() , lr=lr)
 model_f.train()
start_time = time.time()
for epoch in range(n_epochs):
s = time.time()
avg_train_loss = 0
optimizer_f.zero_grad()
for idx, data in enumerate(train_loader):
X1 = data['x_c'].to(device ).float()
X2 = data['x_f'].to(device ).float()
y = data['y'].to(device ).float()
preds = model_f(X1, X2)['preds'].float()
cond = X2 > np.log(10)
 preds = preds[cond]
y = y[cond]
loss = loss_func(preds, y)
 loss += reg_factor * reg_loss_func(model_f.l_conv.weight1)
 avg_train_loss += loss / len(train_loader)
 loss.backward()
if(idx+1)% gradient_accumulation == 0 or idx == len(train_loader)- 1:
optimizer_f.step()
optimizer_f.zero_grad()
if epoch % 1000 == 0:
model_f.eval()
preds_f_delta = train_p_f.values[:,:TRAIN_N]
for idx, data in enumerate(test_loader):
X1 = data['x_c'].to(device ).float()
temp = model_f(X1, torch.Tensor(preds_f_delta[:,-1] ).unsqueeze(0)) ['preds']
temp = np.exp(temp.detach().cpu().numpy().reshape(-1,1)) - 1
preds_f_delta = np.hstack(( preds_f_delta, temp))
preds_f_delta = np.log(1 + preds_f_delta)
 model_f.train()
model_f.eval()
preds_f_delta = train_p_f.values[:,:TRAIN_N]
for idx, data in enumerate(test_loader):
X1 = data['x_c'].to(device ).float()
temp = model_f(X1, torch.Tensor(preds_f_delta[:,-1] ).unsqueeze(0)) ['preds']
temp = np.exp(temp.detach().cpu().numpy().reshape(-1,1)) - 1
preds_f_delta = np.hstack(( preds_f_delta, temp))
preds_f += preds_f_delta / bag_size
preds_f_2 = np.log(1 + preds_f)
 print("Done")
<compute_test_metric>
X_train = df_temp[df_temp["FLG"] == "train"]
X_train = X_train.drop(columns=["FLG"])
 print('X_train' + str(X_train.shape))
X_test = df_temp[df_temp["FLG"] == "test"]
X_test = X_test.drop(columns=["FLG"])
 print('X_test' + str(X_test.shape))
preds_f = np.mean([preds_f_1, preds_f_2], axis=0 )<compute_test_metric>
X_train['loan_amnt'] = X_train['loan_amnt'].apply(np.log1p)
 X_test['loan_amnt'] = X_test['loan_amnt'].apply(np.log1p)
 X_train['emp_length'] = X_train['emp_length'].apply(np.log1p)
 X_test['emp_length'] = X_test['emp_length'].apply(np.log1p)
 X_train['annual_inc'] = X_train['annual_inc'].apply(np.log1p)
 X_test['annual_inc'] = X_test['annual_inc'].apply(np.log1p)
 X_train['open_acc'] = X_train['open_acc'].apply(np.log1p)
 X_test['open_acc'] = X_test['open_acc'].apply(np.log1p)
 X_train['revol_bal'] = X_train['revol_bal'].apply(np.log1p)
 X_test['revol_bal'] = X_test['revol_bal'].apply(np.log1p)
 X_train['total_acc'] = X_train['total_acc'].apply(np.log1p)
 X_test['total_acc'] = X_test['total_acc'].apply(np.log1p)
 X_train['tot_cur_bal'] = X_train['tot_cur_bal'].apply(np.log1p)
 X_test['tot_cur_bal'] = X_test['tot_cur_bal'].apply(np.log1p)
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, d]), preds_c[:, d]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, d]), preds_f[:, d]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c[:, TRAIN_N:TRAIN_N+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )<load_from_csv>
nums = ['loan_amnt', 'installment', 'annual_inc', 'dti', 'delinq_2yrs', 'inq_last_6mths',
'mths_since_last_delinq', 'mths_since_last_record', 'open_acc', 'pub_rec', 'revol_bal',
'revol_util', 'total_acc', 'collections_12_mths_ex_med', 'mths_since_last_major_derog',
'acc_now_delinq', 'tot_coll_amt', 'tot_cur_bal']
scaler = StandardScaler()
X_train[nums] = X_train[nums].astype(float)
 X_test[nums] = X_test[nums].astype(float)
 scaler.fit(X_train[nums])
 X_train[nums] = scaler.transform(X_train[nums])
 X_test[nums] = scaler.transform(X_test[nums] )
def get_cpmp_sub(save_oof=False, save_public_test=False):
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/train.csv')
 train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test
day_min = train['day'].min()
train['day'] -= day_min
test['day'] -= day_min
train = train[train.day < TRAIN_N]
min_test_val_day = test.day.min()
max_test_val_day = train.day.max()
max_test_day = test.day.max()
num_days = max_test_day + 1
min_test_val_day, max_test_val_day, num_days
train['ForecastId'] = -1
test['Id'] = -1
test['ConfirmedCases'] = 0
test['Fatalities'] = 0
debug = False
data = pd.concat([train,
test[test.day > max_test_val_day][train.columns]
] ).reset_index(drop=True)
 if debug:
data = data[data['geo'] >= 'France_'].reset_index(drop=True)
 gc.collect()
dates = data[data['geo'] == 'France_'].Date.values
if 0:
gr = data.groupby('geo')
 data['ConfirmedCases'] = gr.ConfirmedCases.transform('cummax')
 data['Fatalities'] = gr.Fatalities.transform('cummax')
 geo_data = data.pivot(index='geo', columns='day', values='ForecastId')
 num_geo = geo_data.shape[0]
geo_data
geo_id = {}
for i,g in enumerate(geo_data.index):
geo_id[g] = i
ConfirmedCases = data.pivot(index='geo', columns='day', values='ConfirmedCases')
 Fatalities = data.pivot(index='geo', columns='day', values='Fatalities')
 if debug:
cases = ConfirmedCases.values
deaths = Fatalities.values
else:
cases = np.log1p(ConfirmedCases.values)
 deaths = np.log1p(Fatalities.values)
 def get_dataset(start_pred, num_train, lag_period):
days = np.arange(start_pred - num_train + 1, start_pred + 1)
 lag_cases = np.vstack([cases[:, d - lag_period : d] for d in days])
 lag_deaths = np.vstack([deaths[:, d - lag_period : d] for d in days])
 target_cases = np.vstack([cases[:, d : d + 1] for d in days])
 target_deaths = np.vstack([deaths[:, d : d + 1] for d in days])
 geo_ids = np.vstack([geo_ids_base for d in days])
 country_ids = np.vstack([country_ids_base for d in days])
 return lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days
def update_valid_dataset(data, pred_death, pred_case):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
day = days[-1] + 1
new_lag_cases = np.hstack([lag_cases[:, 1:], pred_case])
 new_lag_deaths = np.hstack([lag_deaths[:, 1:], pred_death])
 new_target_cases = cases[:, day:day+1]
new_target_deaths = deaths[:, day:day+1]
new_geo_ids = geo_ids
new_country_ids = country_ids
new_days = 1 + days
return new_lag_cases, new_lag_deaths, new_target_cases, new_target_deaths, new_geo_ids, new_country_ids, new_days
def fit_eval(lr_death, lr_case, data, start_lag_death, end_lag_death, num_lag_case, fit, score):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], country_ids])
 X_death = np.hstack([lag_deaths[:, -num_lag_case:], country_ids])
 X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], lag_deaths[:, -num_lag_case:], country_ids])
 y_death = target_deaths
y_death_prev = lag_deaths[:, -1:]
if fit:
if 0:
keep =(y_death > 0 ).ravel()
X_death = X_death[keep]
y_death = y_death[keep]
y_death_prev = y_death_prev[keep]
lr_death.fit(X_death, y_death)
 y_pred_death = lr_death.predict(X_death)
 y_pred_death = np.maximum(y_pred_death, y_death_prev)
 X_case = np.hstack([lag_cases[:, -num_lag_case:], geo_ids])
 X_case = lag_cases[:, -num_lag_case:]
y_case = target_cases
y_case_prev = lag_cases[:, -1:]
if fit:
lr_case.fit(X_case, y_case)
 y_pred_case = lr_case.predict(X_case)
 y_pred_case = np.maximum(y_pred_case, y_case_prev)
 if score:
death_score = val_score(y_death, y_pred_death)
 case_score = val_score(y_case, y_pred_case)
 else:
death_score = 0
case_score = 0
return death_score, case_score, y_pred_death, y_pred_case
def train_model(train, valid, start_lag_death, end_lag_death, num_lag_case, num_val, score=True):
alpha = 3
lr_death = Ridge(alpha=alpha, fit_intercept=False)
 lr_case = Ridge(alpha=alpha, fit_intercept=True)
(train_death_score, train_case_score, train_pred_death, train_pred_case, 
)= fit_eval(lr_death, lr_case, train, start_lag_death, end_lag_death, num_lag_case, fit=True, score=score)
 death_scores = []
case_scores = []
death_pred = []
case_pred = []
for i in range(num_val): 
(valid_death_score, valid_case_score, valid_pred_death, valid_pred_case, 
)= fit_eval(lr_death, lr_case, valid, start_lag_death, end_lag_death, num_lag_case, fit=False, score=score)
 death_scores.append(valid_death_score)
 case_scores.append(valid_case_score)
 death_pred.append(valid_pred_death)
 case_pred.append(valid_pred_case)
 if 0:
print('val death: %0.3f' % valid_death_score,
'val case: %0.3f' % valid_case_score,
'val : %0.3f' % np.mean([valid_death_score, valid_case_score]),
flush=True)
 valid = update_valid_dataset(valid, valid_pred_death, valid_pred_case)
 if score:
death_scores = np.sqrt(np.mean([s**2 for s in death_scores]))
case_scores = np.sqrt(np.mean([s**2 for s in case_scores]))
if 0:
print('train death: %0.3f' % train_death_score,
'train case: %0.3f' % train_case_score,
'val death: %0.3f' % death_scores,
'val case: %0.3f' % case_scores,
'val : %0.3f' %(( death_scores + case_scores)/ 2),
flush=True)
 else:
print('%0.4f' % case_scores,
', %0.4f' % death_scores,
'= %0.4f' %(( death_scores + case_scores)/ 2),
flush=True)
 death_pred = np.hstack(death_pred)
 case_pred = np.hstack(case_pred)
 return death_scores, case_scores, death_pred, case_pred
countries = [g.split('_')[0] for g in geo_data.index]
countries = pd.factorize(countries)[0]
country_ids_base = countries.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 country_ids_base = 0.2 * ohe.fit_transform(country_ids_base)
 country_ids_base.shape
geo_ids_base = np.arange(num_geo ).reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 geo_ids_base = 0.1 * ohe.fit_transform(geo_ids_base)
 geo_ids_base.shape
def val_score(true, pred):
pred = np.log1p(np.round(np.expm1(pred)- 0.2))
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def val_score(true, pred):
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
start_lag_death, end_lag_death = 14, 6,
num_train = 5
num_lag_case = 14
lag_period = max(start_lag_death, num_lag_case)
 def get_oof(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[start_val], start_val, num_val)
 train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = train[['Date', 'Id', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[sub.day >= start_val]
sub = sub[['Id', 'ConfirmedCases', 'Fatalities']].copy()
return sub
if save_oof:
for start_val_delta, date in zip(range(3, -8, -3),
['2020-03-22', '2020-03-19', '2020-03-16', '2020-03-13']):
print(date, end=' ')
 oof = get_oof(start_val_delta)
 oof.to_csv('.. /submissions/cpmp-%s.csv' % date, index=None)
 def get_sub(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = 14
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub.fillna(0)
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
return sub
return sub
known_test = train[['geo', 'day', 'ConfirmedCases', 'Fatalities']
].merge(test[['geo', 'day', 'ForecastId']], how='left', on=['geo', 'day'])
 known_test = known_test[['ForecastId', 'ConfirmedCases', 'Fatalities']][known_test.ForecastId.notnull() ].copy()
known_test
unknow_test = test[test.day > max_test_val_day]
unknow_test
def get_final_sub() :
start_val = max_test_val_day + 1
last_train = start_val - 1
num_val = max_test_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = num_val + 3
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
(_, _, val_death_preds, val_case_preds 
)= train_model(train_data, valid_data, start_lag_death, end_lag_death, num_lag_case, num_val, score=False)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
print(unknow_test.shape, pred_deaths.shape, pred_cases.shape)
 sub = unknow_test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
sub = pd.concat([known_test, sub])
 return sub
if save_public_test:
sub = get_sub()
else:
sub = get_final_sub()
return sub<compute_test_metric>
cats = ['issue_d','sub_grade',
'title','zip_code','addr_state','earliest_cr_line','initial_list_status','application_type']
oe = OrdinalEncoder(cols=cats, return_df=False)
 X_train[cats] = oe.fit_transform(X_train[cats])
 X_test[cats] = oe.transform(X_test[cats] )
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
m1s = []
m2s = []
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, d]), preds_c_cpmp[:, d-START_PUBLIC]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, d]), preds_f_cpmp[:, d-START_PUBLIC]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 m1s += [m1]
m2s += [m2]
print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c_cpmp[:, TRAIN_N-START_PUBLIC:TRAIN_N-START_PUBLIC+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f_cpmp[:, TRAIN_N-START_PUBLIC:TRAIN_N-START_PUBLIC+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
<load_from_csv>
TXT_train = X_train.emp_title.copy()
TXT_test = X_test.emp_title.copy()
TXT_train.head()
X_train.drop(['emp_title'], axis=1, inplace=True)
 X_test.drop(['emp_title'], axis=1, inplace=True)
 X_train.fillna(X_train.median() , inplace=True)
 X_test.fillna(X_train.median() , inplace=True )
COMP = '.. /input/covid19-global-forecasting-week-3'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '' ).fillna('N/A')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '' ).fillna('N/A')
 train['Location'] = train['Country_Region'].fillna('')+ '-' + train['Province_State'].fillna('N/A')
 test['Location'] = test['Country_Region'].fillna('')+ '-' + test['Province_State'].fillna('N/A')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 country_codes = pd.read_csv('.. /input/covid19-metadata/country_codes.csv', keep_default_na=False)
 train = train.merge(country_codes, on='Country_Region', how='left')
 test = test.merge(country_codes, on='Country_Region', how='left')
 train['continent'] = train['continent'].fillna('')
 test['continent'] = test['continent'].fillna('')
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in tqdm(df.groupby('Location')) :
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1
def get_beluga_sub() :
train, test, submission = get_comp_data(COMP)
 train.shape, test.shape, submission.shape
TRAIN_START = train.Date.min()
TEST_START = test.Date.min()
TRAIN_END = train.Date.max()
TEST_END = test.Date.max()
TRAIN_START, TRAIN_END, TEST_START, TEST_END
train_clean = process_each_location(train)
 train_clean['Geo
DECAY = 0.93
DECAY ** 7, DECAY ** 14, DECAY ** 21, DECAY ** 28
confirmed_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
confirmed_deltas['DELTA'] = GLOBAL_DELTA
confirmed_deltas.loc[confirmed_deltas.continent=='Africa', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.continent=='Oceania', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Korea South', 'DELTA'] = 0.011
confirmed_deltas.loc[confirmed_deltas.Country_Region=='US', 'DELTA'] = 0.15
confirmed_deltas.loc[confirmed_deltas.Country_Region=='China', 'DELTA'] = 0.01
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Japan', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Norway', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iceland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Austria', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Italy', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Spain', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Portugal', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Israel', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iran', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Germany', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Russia', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Brazil', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Turkey', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Philippines', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Location=='France-N/A', 'DELTA'] = 0.1
confirmed_deltas.loc[confirmed_deltas.Location=='United Kingdom-N/A', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Location=='Diamond Princess-N/A', 'DELTA'] = 0.00
confirmed_deltas.loc[confirmed_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Location=='San Marino-N/A', 'DELTA'] = 0.03
confirmed_deltas.shape, confirmed_deltas.DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].shape, confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA]
confirmed_deltas.describe()
daily_log_confirmed = train_clean.pivot('Location', 'Date', 'LogConfirmed' ).reset_index()
daily_log_confirmed = daily_log_confirmed.sort_values(TRAIN_END, ascending=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in confirmed_deltas.Location.values:
confirmed_delta = confirmed_deltas.loc[confirmed_deltas.Location == loc, 'DELTA'].values[0]
daily_log_confirmed.loc[daily_log_confirmed.Location == loc, new_day] = daily_log_confirmed.loc[daily_log_confirmed.Location == loc, last_day] + \
confirmed_delta * DECAY ** i
confirmed_prediciton = pd.melt(daily_log_confirmed, id_vars='Location')
 confirmed_prediciton['ConfirmedCases'] = to_exp(confirmed_prediciton['value'])
 confirmed_prediciton_cases = confirmed_prediciton.copy()
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_death_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_death_deltas = latest.merge(daily_death_deltas, on='Geo
death_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
death_deltas['DELTA'] = GLOBAL_DELTA
death_deltas.loc[death_deltas.Country_Region=='China', 'DELTA'] = 0.005
death_deltas.loc[death_deltas.continent=='Oceania', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Country_Region=='Korea South', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Japan', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='US', 'DELTA'] = 0.17
death_deltas.loc[death_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Norway', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Iceland', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Country_Region=='Austria', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Italy', 'DELTA'] = 0.07
death_deltas.loc[death_deltas.Country_Region=='Spain', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Portugal', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Israel', 'DELTA'] = 0.16
death_deltas.loc[death_deltas.Country_Region=='Iran', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='Germany', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Russia', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Brazil', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Turkey', 'DELTA'] = 0.22
death_deltas.loc[death_deltas.Country_Region=='Philippines', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Location=='France-N/A', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='United Kingdom-N/A', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='Diamond Princess-N/A', 'DELTA'] = 0.00
death_deltas.loc[death_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Location=='San Marino-N/A', 'DELTA'] = 0.05
death_deltas.shape
death_deltas.DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].shape
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA]
death_deltas.describe()
daily_log_deaths = train_clean.pivot('Location', 'Date', 'LogFatalities' ).reset_index()
daily_log_deaths = daily_log_deaths.sort_values(TRAIN_END, ascending=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in death_deltas.Location:
death_delta = death_deltas.loc[death_deltas.Location == loc, 'DELTA'].values[0]
daily_log_deaths.loc[daily_log_deaths.Location == loc, new_day] = daily_log_deaths.loc[daily_log_deaths.Location == loc, last_day] + \
death_delta * DECAY ** i
confirmed_prediciton = pd.melt(daily_log_deaths, id_vars='Location')
 confirmed_prediciton['Fatalities'] = to_exp(confirmed_prediciton['value'])
 confirmed_prediciton_fatalities = confirmed_prediciton.copy()
return confirmed_prediciton_cases, confirmed_prediciton_fatalities
preds_beluga_cases, preds_beluga_fatalities = get_beluga_sub()<define_variables>
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 cats = ['issue_d','sub_grade',
'title','zip_code','addr_state','earliest_cr_line','initial_list_status','application_type']
for col in cats:
summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
locs = p_f_beluga.index.values<set_options>
X_train.fillna(X_train.mean() , axis=0, inplace=True)
 X_test.fillna(X_train.mean() , axis=0, inplace=True )
warnings.filterwarnings('ignore' )<feature_engineering>
y_pred = np.zeros(X_test.shape[0])
 for i in range(30):
clf = LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.71,
importance_type='split', learning_rate=0.05, max_depth=-1,
min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
n_estimators=1000, n_jobs=-1, num_leaves=31, objective=None,
random_state=i, reg_alpha=1.0, reg_lambda=1.0, silent=True,
subsample=0.9, subsample_for_bin=200000, subsample_freq=0)
 clf.fit(X_train, y_train)
 y_pred += clf.predict_proba(X_test)[:,1]
y_pred /= 30
FIRST_TEST = test_orig['Date'].apply(lambda x: x.dayofyear ).min()<define_variables>
<concatenate>
def do_aggregation(df, col, mean_range):
df_new = copy.deepcopy(df)
 col_new = '{}_({}-{})'.format(col, mean_range[0], mean_range[1])
 df_new[col_new] = 0
tmp = df_new[col].rolling(mean_range[1]-mean_range[0]+1 ).mean()
df_new[col_new][mean_range[0]:] = tmp[:-(mean_range[0])]
df_new[col_new][pd.isna(df_new[col_new])] = 0
return df_new[[col_new]].reset_index(drop=True)
 def do_aggregations(df):
df = pd.concat([df, do_aggregation(df, 'cases/day', [1,1] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [1,7] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [8,14] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [15,21] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,1] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,7] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [8,14] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [15,21] ).reset_index(drop=True)], axis=1)
 for threshold in [1, 10, 100]:
days_under_threshold =(df['ConfirmedCases']<threshold ).sum()
tmp = df['day'].values - 22 - days_under_threshold
tmp[tmp<=0] = 0
df['days_since_{}cases'.format(threshold)] = tmp
for threshold in [1, 10, 100]:
days_under_threshold =(df['Fatalities']<threshold ).sum()
tmp = df['day'].values - 22 - days_under_threshold
tmp[tmp<=0] = 0
df['days_since_{}fatal'.format(threshold)] = tmp
if df['place_id'][0]=='China/Hubei':
df['days_since_1cases'] += 35
df['days_since_10cases'] += 35-13
df['days_since_100cases'] += 4
df['days_since_1fatal'] += 13
return df<categorify>
submission = pd.read_csv('.. /input/homework-for-students4plus/sample_submission.csv', index_col=0)
 submission.loan_condition = y_pred
submission.to_csv('submission.csv' )