Kaggle competition: COVID19 Global Forecasting (Week 4)
<merge><EOS> 
get_ipython().run_line_magic('matplotlib', 'inline')
 get_ipython().run_line_magic('config', "InlineBackend.figure_format = 'retina'")
 def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 testing_result=False
dateck='2020-04-03'
def get_cpmp_sub(save_oof=False, save_public_test=False):
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 if testing_result :train=train[train.Date<=dateck]
train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test
day_min = train['day'].min()
train['day'] -= day_min
test['day'] -= day_min
min_test_val_day = test.day.min()
max_test_val_day = train.day.max()
max_test_day = test.day.max()
num_days = max_test_day + 1
min_test_val_day, max_test_val_day, num_days
train['ForecastId'] = -1
test['Id'] = -1
test['ConfirmedCases'] = 0
test['Fatalities'] = 0
debug = False
data = pd.concat([train,
test[test.day > max_test_val_day][train.columns]
] ).reset_index(drop=True)
 if debug:
data = data[data['geo'] >= 'France_'].reset_index(drop=True)
 gc.collect()
dates = data[data['geo'] == 'France_'].Date.values
if 0:
gr = data.groupby('geo')
 data['ConfirmedCases'] = gr.ConfirmedCases.transform('cummax')
 data['Fatalities'] = gr.Fatalities.transform('cummax')
 geo_data = data.pivot(index='geo', columns='day', values='ForecastId')
 num_geo = geo_data.shape[0]
geo_data
geo_id = {}
for i,g in enumerate(geo_data.index):
geo_id[g] = i
ConfirmedCases = data.pivot(index='geo', columns='day', values='ConfirmedCases')
 Fatalities = data.pivot(index='geo', columns='day', values='Fatalities')
 if debug:
cases = ConfirmedCases.values
deaths = Fatalities.values
else:
cases = np.log1p(ConfirmedCases.values)
 deaths = np.log1p(Fatalities.values)
 def get_dataset(start_pred, num_train, lag_period):
days = np.arange(start_pred - num_train + 1, start_pred + 1)
 lag_cases = np.vstack([cases[:, d - lag_period : d] for d in days])
 lag_deaths = np.vstack([deaths[:, d - lag_period : d] for d in days])
 target_cases = np.vstack([cases[:, d : d + 1] for d in days])
 target_deaths = np.vstack([deaths[:, d : d + 1] for d in days])
 geo_ids = np.vstack([geo_ids_base for d in days])
 country_ids = np.vstack([country_ids_base for d in days])
 return lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days
def update_valid_dataset(data, pred_death, pred_case):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
day = days[-1] + 1
new_lag_cases = np.hstack([lag_cases[:, 1:], pred_case])
 new_lag_deaths = np.hstack([lag_deaths[:, 1:], pred_death])
 new_target_cases = cases[:, day:day+1]
new_target_deaths = deaths[:, day:day+1]
new_geo_ids = geo_ids
new_country_ids = country_ids
new_days = 1 + days
return new_lag_cases, new_lag_deaths, new_target_cases, new_target_deaths, new_geo_ids, new_country_ids, new_days
def fit_eval(lr_death, lr_case, data, start_lag_death, end_lag_death, num_lag_case, fit, score):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], country_ids])
 X_death = np.hstack([lag_deaths[:, -num_lag_case:], country_ids])
 X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], lag_deaths[:, -num_lag_case:], country_ids])
 y_death = target_deaths
y_death_prev = lag_deaths[:, -1:]
if fit:
if 0:
keep =(y_death > 0 ).ravel()
X_death = X_death[keep]
y_death = y_death[keep]
y_death_prev = y_death_prev[keep]
lr_death.fit(X_death, y_death)
 y_pred_death = lr_death.predict(X_death)
 y_pred_death = np.maximum(y_pred_death, y_death_prev)
 X_case = np.hstack([lag_cases[:, -num_lag_case:], geo_ids])
 X_case = lag_cases[:, -num_lag_case:]
y_case = target_cases
y_case_prev = lag_cases[:, -1:]
if fit:
lr_case.fit(X_case, y_case)
 y_pred_case = lr_case.predict(X_case)
 y_pred_case = np.maximum(y_pred_case, y_case_prev)
 if score:
death_score = val_score(y_death, y_pred_death)
 case_score = val_score(y_case, y_pred_case)
 else:
death_score = 0
case_score = 0
return death_score, case_score, y_pred_death, y_pred_case
def train_model(train, valid, start_lag_death, end_lag_death, num_lag_case, num_val, score=True):
alpha = 3
lr_death = Ridge(alpha=alpha, fit_intercept=False)
 lr_case = Ridge(alpha=alpha, fit_intercept=True)
(train_death_score, train_case_score, train_pred_death, train_pred_case, 
)= fit_eval(lr_death, lr_case, train, start_lag_death, end_lag_death, num_lag_case, fit=True, score=score)
 death_scores = []
case_scores = []
death_pred = []
case_pred = []
for i in range(num_val): 
(valid_death_score, valid_case_score, valid_pred_death, valid_pred_case, 
)= fit_eval(lr_death, lr_case, valid, start_lag_death, end_lag_death, num_lag_case, fit=False, score=score)
 death_scores.append(valid_death_score)
 case_scores.append(valid_case_score)
 death_pred.append(valid_pred_death)
 case_pred.append(valid_pred_case)
 if 0:
print('val death: %0.3f' % valid_death_score,
'val case: %0.3f' % valid_case_score,
'val : %0.3f' % np.mean([valid_death_score, valid_case_score]),
flush=True)
 valid = update_valid_dataset(valid, valid_pred_death, valid_pred_case)
 if score:
death_scores = np.sqrt(np.mean([s**2 for s in death_scores]))
case_scores = np.sqrt(np.mean([s**2 for s in case_scores]))
if 0:
print('train death: %0.3f' % train_death_score,
'train case: %0.3f' % train_case_score,
'val death: %0.3f' % death_scores,
'val case: %0.3f' % case_scores,
'val : %0.3f' %(( death_scores + case_scores)/ 2),
flush=True)
 else:
print('%0.4f' % case_scores,
', %0.4f' % death_scores,
'= %0.4f' %(( death_scores + case_scores)/ 2),
flush=True)
 death_pred = np.hstack(death_pred)
 case_pred = np.hstack(case_pred)
 return death_scores, case_scores, death_pred, case_pred
countries = [g.split('_')[0] for g in geo_data.index]
countries = pd.factorize(countries)[0]
country_ids_base = countries.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 country_ids_base = 0.2 * ohe.fit_transform(country_ids_base)
 country_ids_base.shape
geo_ids_base = np.arange(num_geo ).reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 geo_ids_base = 0.1 * ohe.fit_transform(geo_ids_base)
 geo_ids_base.shape
def val_score(true, pred):
pred = np.log1p(np.round(np.expm1(pred)- 0.2))
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def val_score(true, pred):
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
start_lag_death, end_lag_death = 14, 6,
num_train = 5
num_lag_case = 14
lag_period = max(start_lag_death, num_lag_case)
 def get_oof(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[start_val], start_val, num_val)
 train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = train[['Date', 'Id', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[sub.day >= start_val]
sub = sub[['Id', 'ConfirmedCases', 'Fatalities']].copy()
return sub
if save_oof:
for start_val_delta, date in zip(range(3, -8, -3),
['2020-03-22', '2020-03-19', '2020-03-16', '2020-03-13']):
print(date, end=' ')
 oof = get_oof(start_val_delta)
 oof.to_csv('.. /submissions/cpmp-%s.csv' % date, index=None)
 def get_sub(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = 14
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub.fillna(0)
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
return sub
return sub
known_test = train[['geo', 'day', 'ConfirmedCases', 'Fatalities']
].merge(test[['geo', 'day', 'ForecastId']], how='left', on=['geo', 'day'])
 known_test = known_test[['ForecastId', 'ConfirmedCases', 'Fatalities']][known_test.ForecastId.notnull() ].copy()
known_test
unknow_test = test[test.day > max_test_val_day]
unknow_test
def get_final_sub() :
start_val = max_test_val_day + 1
last_train = start_val - 1
num_val = max_test_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = num_val + 3
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
(_, _, val_death_preds, val_case_preds 
)= train_model(train_data, valid_data, start_lag_death, end_lag_death, num_lag_case, num_val, score=False)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
print(unknow_test.shape, pred_deaths.shape, pred_cases.shape)
 sub = unknow_test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
sub = pd.concat([known_test, sub])
 return sub
if save_public_test:
sub = get_sub()
else:
sub = get_final_sub()
return sub
sub1 = get_cpmp_sub()
sub1['ForecastId'] = sub1['ForecastId'].astype('int')
 sub1.sort_values("ForecastId", inplace=True)
 train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 dacheckv='2020-04-01'
train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train=train[train.Date>dacheckv]
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test1=test.merge(train,on=['Country_Region','Province_State','Date'],how='inner')
 print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['ConfirmedCases'].values,test1['ConfirmedCases'].values))
print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['Fatalities'].values,test1['Fatalities'].values))
BAGS = 3
SEED = 123
SET_FRAC = 0.01
TRUNCATED = False
DROPS = True
PRIVATE = True
USE_PRIORS = False
SUP_DROP = 0.0
ACTIONS_DROP = 0.0
PLACE_FRACTION = 1.0
LT_DECAY_MAX = 0.3
LT_DECAY_MIN = -0.4
SINGLE_MODEL = False
MODEL_Y = 'agg_dff'
pd.options.display.float_format = '{:.8}'.format
plt.rcParams["figure.figsize"] =(12, 4.75)
 InteractiveShell.ast_node_interactivity = "all"
pd.options.display.max_rows = 999
path = '/kaggle/input/c19week3'
input_path = '/kaggle/input/covid19-global-forecasting-week-4'
train = pd.read_csv(input_path + '/train.csv')
 if testing_result :train=train[train.Date<=dateck]
test = pd.read_csv(input_path + '/test.csv')
 sub = pd.read_csv(input_path + '/submission.csv')
 tt = pd.merge(train, test, on=['Country_Region', 'Province_State', 'Date'],
how='right', validate="1:1")\ 
.fillna(method = 'ffill')
 public = tt[['ForecastId', 'ConfirmedCases', 'Fatalities']]
train.Date.max()
test_dates = test.Date.unique()
test_dates
pp = 'public'
if PRIVATE:
test = test[ pd.to_datetime(test.Date)> train.Date.max() ]
pp = 'private'
test.Date.unique()
revised = pd.read_csv(path + '/outside_data' +
'/covid19_train_data_us_states_before_march_09_new.csv')
 revised = revised[['Province_State', 'Country_Region', 'Date', 'ConfirmedCases', 'Fatalities']]
train.tail()
revised.head()
train.Date = pd.to_datetime(train.Date)
 revised.Date = pd.to_datetime(revised.Date)
 rev_train = pd.merge(train, revised, on=['Province_State', 'Country_Region', 'Date'],
suffixes =('', '_r'), how='left')
 rev_train[~rev_train.ConfirmedCases_r.isnull() ].head()
rev_train.ConfirmedCases = np.where(( rev_train.ConfirmedCases == 0)&(( rev_train.ConfirmedCases_r > 0)) & 
(rev_train.Country_Region == 'US'),
rev_train.ConfirmedCases_r,
rev_train.ConfirmedCases)
 rev_train.Fatalities = np.where(~rev_train.Fatalities_r.isnull() & 
(rev_train.Fatalities == 0)&(( rev_train.Fatalities_r > 0)) & 
(rev_train.Country_Region == 'US')
 ,
rev_train.Fatalities_r,
rev_train.Fatalities)
 rev_train.drop(columns = ['ConfirmedCases_r', 'Fatalities_r'], inplace=True)
 train = rev_train
contain_data = pd.read_csv(path + '/outside_data' +
'/OxCGRT_Download_070420_160027_Full.csv')
 contain_data = contain_data[[c for c in contain_data.columns if
not any(z in c for z in ['_Notes','Unnamed', 'Confirmed',
'CountryCode',
'S8', 'S9', 'S10','S11',
'StringencyIndexForDisplay'])] ]\
contain_data.rename(columns = {'CountryName': "Country"}, inplace=True)
 contain_data.Date = contain_data.Date.astype(str ).apply(datetime.datetime.strptime, args=('%Y%m%d',))
contain_data_orig = contain_data.copy()
contain_data.columns
cds = []
for country in contain_data.Country.unique() :
cd = contain_data[contain_data.Country==country]
cd = cd.fillna(method = 'ffill' ).fillna(0)
 cd.StringencyIndex = cd.StringencyIndex.cummax()
col_count = cd.shape[1]
for col in [c for c in contain_data.columns if 'S' in c]:
col_diff = cd[col].diff()
cd[col+"_chg_5d_ewm"] = col_diff.ewm(span = 5 ).mean()
cd[col+"_chg_20_ewm"] = col_diff.ewm(span = 20 ).mean()
cd['StringencyIndex_5d_ewm'] = cd.StringencyIndex.ewm(span = 5 ).mean()
cd['StringencyIndex_20d_ewm'] = cd.StringencyIndex.ewm(span = 20 ).mean()
cd['S_data_days'] =(cd.Date - cd.Date.min() ).dt.days
for s in [1, 10, 20, 30, 50, ]:
cd['days_since_Stringency_{}'.format(s)] = np.clip(( cd.Date - cd[(cd.StringencyIndex > s)].Date.min() ).dt.days, 0, None)
 cds.append(cd.fillna(0)[['Country', 'Date'] + cd.columns.to_list() [col_count:]])
 contain_data = pd.concat(cds)
 contain_data.Date.max()
contain_data.columns
contain_data[contain_data.Country == 'Australia']
contain_data.shape
contain_data.Country.replace({ 'United States': "US",
'South Korea': "Korea, South",
'Taiwan': "Taiwan*",
'Myanmar': "Burma", 'Slovak Republic': "Slovakia",
'Czech Republic': 'Czechia',
}, inplace=True)
 set(contain_data.Country)- set(test.Country_Region)
 sup_data = pd.read_excel(path + '/outside_data' +
'/Data Join - Copy1.xlsx')
 sup_data.columns = [c.replace(' ', '_')for c in sup_data.columns.to_list() ]
sup_data.drop(columns = [c for c in sup_data.columns.to_list() if 'Unnamed:' in c], inplace=True)
 sup_data.drop(columns = [ 'Date', 'ConfirmedCases',
'Fatalities', 'log-cases', 'log-fatalities', 'continent'], inplace=True)
 sup_data['Migrants_in'] = np.clip(sup_data.Migrants, 0, None)
 sup_data['Migrants_out'] = -np.clip(sup_data.Migrants, None, 0)
 sup_data.drop(columns = 'Migrants', inplace=True)
 sup_data.head()
sup_data.shape
train.Date = pd.to_datetime(train.Date)
 test.Date = pd.to_datetime(test.Date)
 train.rename(columns={'Country_Region': 'Country'}, inplace=True)
 test.rename(columns={'Country_Region': 'Country'}, inplace=True)
 sup_data.rename(columns={'Country_Region': 'Country'}, inplace=True)
 train['Place'] = train.Country + train.Province_State.fillna("")
 test['Place'] = test.Country + test.Province_State.fillna("")
 sup_data['Place'] = sup_data.Country + sup_data.Province_State.fillna("")
 len(train.Place.unique())
 sup_data = sup_data[
sup_data.columns.to_list() [2:]]
sup_data = sup_data.replace('N.A.', np.nan ).fillna(-0.5)
 for c in sup_data.columns[:-1]:
m = sup_data[c].max()
if m > 300 and c!='TRUE_POPULATION':
print(c)
 sup_data[c] = np.log(sup_data[c] + 1)
 assert sup_data[c].min() > -1
for c in sup_data.columns[:-1]:
m = sup_data[c].max()
if m > 300:
print(c)
 DEATHS = 'Fatalities'
len(train.Place.unique())
 train.ConfirmedCases = np.where(
(train.ConfirmedCases.shift(1)> train.ConfirmedCases)& 
(train.ConfirmedCases.shift(1)> 0)&(train.ConfirmedCases.shift(-1)> 0)& 
(train.Place == train.Place.shift(1)) &(train.Place == train.Place.shift(-1)) &
~train.ConfirmedCases.shift(-1 ).isnull() ,
np.sqrt(train.ConfirmedCases.shift(1)* train.ConfirmedCases.shift(-1)) ,
train.ConfirmedCases)
 train.Fatalities = np.where(
(train.Fatalities.shift(1)> train.Fatalities)& 
(train.Fatalities.shift(1)> 0)&(train.Fatalities.shift(-1)> 0)& 
(train.Place == train.Place.shift(1)) &(train.Place == train.Place.shift(-1)) &
~train.Fatalities.shift(-1 ).isnull() ,
np.sqrt(train.Fatalities.shift(1)* train.Fatalities.shift(-1)) ,
train.Fatalities)
 for i in [0, -1]:
train.ConfirmedCases = np.where(
(train.ConfirmedCases.shift(2+ i)> train.ConfirmedCases)& 
(train.ConfirmedCases.shift(2+ i)> 0)&(train.ConfirmedCases.shift(-1+ i)> 0)& 
(train.Place == train.Place.shift(2+ i)) &(train.Place == train.Place.shift(-1+ i)) &
~train.ConfirmedCases.shift(-1+ i ).isnull() ,
np.sqrt(train.ConfirmedCases.shift(2+ i)* train.ConfirmedCases.shift(-1+ i)) ,
train.ConfirmedCases)
 train[train.Place=='USVirgin Islands'][-10:]
train[(train.ConfirmedCases.shift(2)> 2* train.ConfirmedCases)& 
(train.Place == train.Place.shift(2)) &(train.ConfirmedCases < 100000)]
train[(train.Fatalities.shift(1)> train.Fatalities)& 
(train.Place == train.Place.shift(1)) &(train.Fatalities < 10000)]
train_bk = train.copy()
full_train = train.copy()
train_c = train[train.Country == 'China']
train_nc = train[train.Country != 'China']
train_us = train[train.Country == 'US']
def lplot(data, minDate = datetime.datetime(2000, 1, 1),
columns = ['ConfirmedCases', 'Fatalities']):
return
REAL = datetime.datetime(2020, 2, 10)
 dataset = train.copy()
if TRUNCATED:
dataset = dataset[dataset.Country.isin(
 ['Italy', 'Spain', 'Germany', 'Portugal', 'Belgium', 'Austria', 'Switzerland' ])]
dataset.head()
def rollDates(df, i, preserve=False):
df = df.copy()
if preserve:
df['Date_i'] = df.Date
df.Date = df.Date + datetime.timedelta(i)
 return df
WINDOWS = [1, 2, 4, 7, 12, 20, 30]
for window in WINDOWS:
csuffix = '_{}d_prior_value'.format(window)
 base = rollDates(dataset, window)
 dataset = pd.merge(dataset, base[['Date', 'Place',
'ConfirmedCases', 'Fatalities']], on = ['Date', 'Place'],
suffixes =('', csuffix), how='left')
 for c in ['ConfirmedCases', 'Fatalities']:
dataset[c+ csuffix].fillna(0, inplace=True)
 dataset[c+ csuffix] = np.log(dataset[c + csuffix] + 1)
 dataset[c+ '_{}d_prior_slope'.format(window)] =(np.log(dataset[c] + 1)- dataset[c+ csuffix])/ window
dataset[c+ '_{}d_ago_zero'.format(window)] = 1.0*(dataset[c+ csuffix] == 0)
 for window1 in WINDOWS:
for window2 in WINDOWS:
for c in ['ConfirmedCases', 'Fatalities']:
if window1 * 1.3 < window2 and window1 * 5 > window2:
dataset[ c +'_{}d_{}d_prior_slope_chg'.format(window1, window2)] = dataset[c+ '_{}d_prior_slope'.format(window1)] - dataset[c+ '_{}d_prior_slope'.format(window2)]
first_case = dataset[dataset.ConfirmedCases >= 1].groupby('Place' ).min()
tenth_case = dataset[dataset.ConfirmedCases >= 10].groupby('Place' ).min()
hundredth_case = dataset[dataset.ConfirmedCases >= 100].groupby('Place' ).min()
thousandth_case = dataset[dataset.ConfirmedCases >= 1000].groupby('Place' ).min()
first_fatality = dataset[dataset.Fatalities >= 1].groupby('Place' ).min()
tenth_fatality = dataset[dataset.Fatalities >= 10].groupby('Place' ).min()
hundredth_fatality = dataset[dataset.Fatalities >= 100].groupby('Place' ).min()
thousandth_fatality = dataset[dataset.Fatalities >= 1000].groupby('Place' ).min()
dataset['days_since_first_case'] = np.clip(( dataset.Date - first_case.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_tenth_case'] = np.clip(( dataset.Date - tenth_case.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_hundredth_case'] = np.clip(( dataset.Date - hundredth_case.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_thousandth_case'] = np.clip(( dataset.Date - thousandth_case.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_first_fatality'] = np.clip(( dataset.Date - first_fatality.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_tenth_fatality'] = np.clip(( dataset.Date - tenth_fatality.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_hundredth_fatality'] = np.clip(( dataset.Date - hundredth_fatality.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_thousandth_fatality'] = np.clip(( dataset.Date - thousandth_fatality.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['case_rate_since_first_case'] = np.clip(( np.log(dataset.ConfirmedCases + 1)- np.log(first_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['case_rate_since_tenth_case'] = np.clip(( np.log(dataset.ConfirmedCases + 1)- np.log(tenth_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) /(dataset.days_since_tenth_case+0.01), 0, 1)
 dataset['case_rate_since_hundredth_case'] = np.clip(( np.log(dataset.ConfirmedCases + 1)- np.log(hundredth_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['case_rate_since_thousandth_case'] = np.clip(( np.log(dataset.ConfirmedCases + 1)- np.log(thousandth_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_first_case'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(first_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_tenth_case'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(tenth_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_hundredth_case'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(hundredth_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_thousandth_case'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(thousandth_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_first_fatality'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(first_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_first_fatality+0.01), 0, 1)
 dataset['fatality_rate_since_tenth_fatality'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(tenth_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_tenth_fatality+0.01), 0, 1)
 dataset['fatality_rate_since_hundredth_fatality'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(hundredth_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_hundredth_fatality+0.01), 0, 1)
 dataset['fatality_rate_since_thousandth_fatality'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(thousandth_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_thousandth_fatality+0.01), 0, 1)
 dataset['first_case_ConfirmedCases'] = np.log(first_case.loc[dataset.Place].ConfirmedCases.values + 1)
 dataset['first_case_Fatalities'] = np.log(first_case.loc[dataset.Place].Fatalities.values + 1)
 dataset['first_fatality_ConfirmedCases'] = np.log(first_fatality.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)*(dataset.days_since_first_fatality >= 0)
 dataset['first_fatality_Fatalities'] = np.log(first_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)*(dataset.days_since_first_fatality >= 0)
 dataset['first_fatality_cfr'] = np.where(dataset.days_since_first_fatality < 0,
-8, 
(dataset.first_fatality_Fatalities)- 
(dataset.first_fatality_ConfirmedCases))
dataset['first_fatality_lag_vs_first_case'] = np.where(dataset.days_since_first_fatality >= 0,
dataset.days_since_first_case - dataset.days_since_first_fatality , -1)
 dataset['case_chg'] = np.clip(np.log(dataset.ConfirmedCases + 1)- np.log(dataset.ConfirmedCases.shift(1)+1), 0, None ).fillna(0)
 dataset['case_chg_ema_3d'] = dataset.case_chg.ewm(span = 3 ).mean() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1)
 dataset['case_chg_ema_10d'] = dataset.case_chg.ewm(span = 10 ).mean() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1)
 dataset['case_chg_stdev_5d'] = dataset.case_chg.rolling(5 ).std() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/5, 0, 1)
 dataset['case_chg_stdev_15d'] = dataset.case_chg.rolling(15 ).std() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/15, 0, 1)
 dataset['case_update_pct_3d_ewm'] =(dataset.case_chg > 0 ).ewm(span = 3 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1), 2)
 dataset['case_update_pct_10d_ewm'] =(dataset.case_chg > 0 ).ewm(span = 10 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1), 2)
 dataset['case_update_pct_30d_ewm'] =(dataset.case_chg > 0 ).ewm(span = 30 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/30, 0, 1), 2)
 dataset['fatality_chg'] = np.clip(np.log(dataset.Fatalities + 1)- np.log(dataset.Fatalities.shift(1)+1), 0, None ).fillna(0)
 dataset['fatality_chg_ema_3d'] = dataset.fatality_chg.ewm(span = 3 ).mean() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/33, 0, 1)
 dataset['fatality_chg_ema_10d'] = dataset.fatality_chg.ewm(span = 10 ).mean() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1)
 dataset['fatality_chg_stdev_5d'] = dataset.fatality_chg.rolling(5 ).std() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/5, 0, 1)
 dataset['fatality_chg_stdev_15d'] = dataset.fatality_chg.rolling(15 ).std() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/15, 0, 1)
 dataset['fatality_update_pct_3d_ewm'] =(dataset.fatality_chg > 0 ).ewm(span = 3 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1), 2)
 dataset['fatality_update_pct_10d_ewm'] =(dataset.fatality_chg > 0 ).ewm(span = 10 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1), 2)
 dataset['fatality_update_pct_30d_ewm'] =(dataset.fatality_chg > 0 ).ewm(span = 30 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/30, 0, 1), 2)
 dataset.tail()
contain_data.Date = contain_data.Date + datetime.timedelta(7)
 contain_data.Date.max()
assert set(dataset.Place.unique())== set(dataset.Place.unique())
 dataset = pd.merge(dataset, sup_data, on='Place', how='left', validate='m:1')
 dataset = pd.merge(dataset, contain_data, on = ['Country', 'Date'], how='left', validate='m:1')
 dataset['log_true_population'] = np.log(dataset.TRUE_POPULATION + 1)
 dataset['ConfirmedCases_percapita'] = np.log(dataset.ConfirmedCases + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['Fatalities_percapita'] = np.log(dataset.Fatalities + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['log_cfr'] = np.log(( dataset.Fatalities + np.clip(0.015 * dataset.ConfirmedCases, 0, 0.3)) /(dataset.ConfirmedCases + 0.1))
def cfr(case, fatality):
cfr_calc = np.log(( fatality + np.clip(0.015 * case, 0, 0.3)) /(case + 0.1))
return np.where(np.isnan(cfr_calc)| np.isinf(cfr_calc),
BLCFR, cfr_calc)
 BLCFR = np.median(dataset[dataset.ConfirmedCases==1].log_cfr[::10])
 dataset.log_cfr.fillna(BLCFR, inplace=True)
 dataset.log_cfr = np.where(dataset.log_cfr.isnull() | np.isinf(dataset.log_cfr),
BLCFR, dataset.log_cfr)
 BLCFR
dataset['log_cfr_3d_ewm'] = BLCFR +(dataset.log_cfr - BLCFR ).ewm(span = 3 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1), 2)
 dataset['log_cfr_8d_ewm'] = BLCFR +(dataset.log_cfr - BLCFR ).ewm(span = 8 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/8, 0, 1), 2)
 dataset['log_cfr_20d_ewm'] = BLCFR +(dataset.log_cfr - BLCFR ).ewm(span = 20 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/20, 0, 1), 2)
 dataset['log_cfr_3d_20d_ewm_crossover'] = dataset.log_cfr_3d_ewm - dataset.log_cfr_20d_ewm
dataset.drop(columns = 'log_cfr', inplace=True)
 date_totals = dataset.groupby('Date' ).sum()
mean_7d_c_slope = dataset.groupby('Date')[['ConfirmedCases_7d_prior_slope']].apply(lambda x:
np.mean(x[x > 0])).ewm(span = 3 ).mean()
mean_7d_f_slope = dataset.groupby('Date')[['Fatalities_7d_prior_slope']].apply(lambda x:
np.mean(x[x > 0])).ewm(span = 7 ).mean()
dataset['ConfirmedCases_percapita_vs_world'] = np.log(dataset.ConfirmedCases + 1)- np.log(dataset.TRUE_POPULATION + 1)-(
 np.log(date_totals.loc[dataset.Date].ConfirmedCases + 1)
 -np.log(date_totals.loc[dataset.Date].TRUE_POPULATION + 1)
 ).values
dataset['Fatalities_percapita_vs_world'] = np.log(dataset.Fatalities + 1)- np.log(dataset.TRUE_POPULATION + 1)-(
 np.log(date_totals.loc[dataset.Date].Fatalities + 1)
 -np.log(date_totals.loc[dataset.Date].TRUE_POPULATION + 1)
 ).values
dataset['cfr_vs_world'] = dataset.log_cfr_3d_ewm - np.log(date_totals.loc[dataset.Date].Fatalities / date_totals.loc[dataset.Date].ConfirmedCases ).values
cont_date_totals = dataset.groupby(['Date', 'continent_generosity'] ).sum()
len(dataset)
 dataset['ConfirmedCases_percapita_vs_continent_mean'] = 0
dataset['Fatalities_percapita_vs_continent_mean'] = 0
dataset['ConfirmedCases_percapita_vs_continent_median'] = 0
dataset['Fatalities_percapita_vs_continent_median'] = 0
for cg in dataset.continent_generosity.unique() :
ps = dataset.groupby("Place" ).last()
tp = ps[ps.continent_generosity==cg].TRUE_POPULATION.sum()
print(tp / 1e9)
 for Date in dataset.Date.unique() :
cd = dataset[(dataset.Date == Date)& 
(dataset.continent_generosity == cg)]\
[['ConfirmedCases', 'Fatalities', 'TRUE_POPULATION']]
cmedian = np.median(np.log(cd.ConfirmedCases + 1)- np.log(cd.TRUE_POPULATION+1))
cmean = np.log(cd.ConfirmedCases.sum() + 1)- np.log(tp + 1)
 fmedian = np.median(np.log(cd.Fatalities + 1)- np.log(cd.TRUE_POPULATION+1))
fmean = np.log(cd.Fatalities.sum() + 1)- np.log(tp + 1)
 cfrmean = cfr(cd.ConfirmedCases.sum() , cd.Fatalities.sum())
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'ConfirmedCases_percapita_vs_continent_mean'] = \
dataset['ConfirmedCases_percapita'] \
-(cmean)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'ConfirmedCases_percapita_vs_continent_median'] = \
dataset['ConfirmedCases_percapita'] \
-(cmedian)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'Fatalities_percapita_vs_continent_mean'] = \
dataset['Fatalities_percapita']\
-(fmean)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'Fatalities_percapita_vs_continent_median'] = \
dataset['Fatalities_percapita']\
-(fmedian)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'cfr_vs_continent'] = \
dataset.log_cfr_3d_ewm \
- cfrmean
all_places = dataset[['Place', 'latitude', 'longitude']].drop_duplicates().set_index('Place',
drop=True)
 all_places.head()
def surroundingPlaces(place, d = 10):
dist =(all_places.latitude - all_places.loc[place].latitude)**2 +(all_places.longitude - all_places.loc[place].longitude)** 2
return all_places[dist < d**2][1:n+1]
def nearestPlaces(place, n = 10):
dist =(all_places.latitude - all_places.loc[place].latitude)**2 +(all_places.longitude - all_places.loc[place].longitude)** 2
ranked = np.argsort(dist)
 return all_places.iloc[ranked][1:n+1]
dgp = dataset.groupby('Place' ).last()
for n in [5, 10, 20]:
for place in dataset.Place.unique() :
nps = nearestPlaces(place, n)
 tp = dgp.loc[nps.index].TRUE_POPULATION.sum()
dataset.loc[dataset.Place==place,
'ratio_population_vs_nearest{}'.format(n)] = \
np.log(dataset.loc[dataset.Place==place].TRUE_POPULATION.mean() + 1)\
- np.log(tp+1)
 nbps = dataset[(dataset.Place.isin(nps.index)) ].groupby('Date')[['ConfirmedCases', 'Fatalities']].sum()
nppc =(np.log(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).ConfirmedCases + 1)- np.log(tp + 1))
nppf =(np.log(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).Fatalities + 1)- np.log(tp + 1))
npp_cfr = cfr(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).ConfirmedCases,
nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).Fatalities)
 dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_percapita_vs_nearest{}'.format(n)] = \
dataset[(dataset.Place == place)].ConfirmedCases_percapita \
- nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_percapita_vs_nearest{}'.format(n)] = \
dataset[(dataset.Place == place)].Fatalities_percapita \
- nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_vs_nearest{}'.format(n)] = \
dataset[(dataset.Place == place)].log_cfr_3d_ewm \
- npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_nearest{}_percapita'.format(n)] = nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_nearest{}_percapita'.format(n)] = nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_nearest{}'.format(n)] = npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_nearest{}_10d_slope'.format(n)] = \ 
(nppc.ewm(span = 1 ).mean() - nppc.ewm(span = 10 ).mean() ).values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_nearest{}_10d_slope'.format(n)] = \ 
(nppf.ewm(span = 1 ).mean() - nppf.ewm(span = 10 ).mean() ).values
npp_cfr_s = pd.Series(npp_cfr)
 dataset.loc[ 
(dataset.Place == place),
'cfr_nearest{}_10d_slope'.format(n)] = \ 
(npp_cfr_s.ewm(span = 1 ).mean() \
- npp_cfr_s.ewm(span = 10 ).mean() ).values
dgp = dataset.groupby('Place' ).last()
for d in [5, 10, 20]:
for place in dataset.Place.unique() :
nps = surroundingPlaces(place, d)
 dataset.loc[dataset.Place==place, 'num_surrounding_places_{}_degrees'.format(d)] = len(nps)
 tp = dgp.loc[nps.index].TRUE_POPULATION.sum()
dataset.loc[dataset.Place==place,
'ratio_population_vs_surrounding_places_{}_degrees'.format(d)] = \
np.log(dataset.loc[dataset.Place==place].TRUE_POPULATION.mean() + 1)\
- np.log(tp+1)
 if len(nps)==0:
continue;
nbps = dataset[(dataset.Place.isin(nps.index)) ].groupby('Date')[['ConfirmedCases', 'Fatalities']].sum()
nppc =(np.log(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).ConfirmedCases + 1)- np.log(tp + 1))
nppf =(np.log(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).Fatalities + 1)- np.log(tp + 1))
npp_cfr = cfr(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).ConfirmedCases,
nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).Fatalities)
 dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_percapita_vs_surrounding_places_{}_degrees'.format(d)] = \
dataset[(dataset.Place == place)].ConfirmedCases_percapita \
- nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_percapita_vs_surrounding_places_{}_degrees'.format(d)] = \
dataset[(dataset.Place == place)].Fatalities_percapita \
- nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_vs_surrounding_places_{}_degrees'.format(d)] = \
dataset[(dataset.Place == place)].log_cfr_3d_ewm \
- npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_surrounding_places_{}_degrees_percapita'.format(d)] = nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_surrounding_places_{}_degrees_percapita'.format(d)] = nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_surrounding_places_{}_degrees'.format(d)] = npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_surrounding_places_{}_degrees_10d_slope'.format(d)] = \ 
(nppc.ewm(span = 1 ).mean() - nppc.ewm(span = 10 ).mean() ).values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_surrounding_places_{}_degrees_10d_slope'.format(d)] = \ 
(nppf.ewm(span = 1 ).mean() - nppf.ewm(span = 10 ).mean() ).values
npp_cfr_s = pd.Series(npp_cfr)
 dataset.loc[ 
(dataset.Place == place),
'cfr_surrounding_places_{}_degrees_10d_slope'.format(d)] = \ 
(npp_cfr_s.ewm(span = 1 ).mean() \
- npp_cfr_s.ewm(span = 10 ).mean() ).values
for col in [c for c in dataset.columns if 'surrounding_places' in c and 'num_sur' not in c]:
dataset[col] = dataset[col].fillna(0)
 n_col = 'num_surrounding_places_{}_degrees'.format(col.split('degrees')[0].split('_')[-2])
 print(col)
 dataset[col + "_times_num_places"] = dataset[col] * np.sqrt(dataset[n_col])
 dataset[dataset.Country=='US'][['Place', 'Date'] + [c for c in dataset.columns if 'ratio_p' in c]] [::50]
dataset.TRUE_POPULATION
dataset.TRUE_POPULATION.sum()
dataset.groupby('Date' ).sum().TRUE_POPULATION
dataset['first_case_ConfirmedCases_percapita'] = np.log(dataset.first_case_ConfirmedCases + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['first_case_Fatalities_percapita'] = np.log(dataset.first_case_Fatalities + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['first_fatality_Fatalities_percapita'] = np.log(dataset.first_fatality_Fatalities + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['first_fatality_ConfirmedCases_percapita'] = np.log(dataset.first_fatality_ConfirmedCases + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['days_to_saturation_ConfirmedCases_4d'] =(- np.log(dataset.ConfirmedCases + 1)+ np.log(dataset.TRUE_POPULATION + 1)) / dataset.ConfirmedCases_4d_prior_slope
dataset['days_to_saturation_ConfirmedCases_7d'] =(- np.log(dataset.ConfirmedCases + 1)+ np.log(dataset.TRUE_POPULATION + 1)) / dataset.ConfirmedCases_7d_prior_slope
dataset['days_to_saturation_Fatalities_20d_cases'] =(- np.log(dataset.Fatalities + 1)+ np.log(dataset.TRUE_POPULATION + 1)) / dataset.ConfirmedCases_20d_prior_slope
dataset['days_to_saturation_Fatalities_12d_cases'] =(- np.log(dataset.Fatalities + 1)+ np.log(dataset.TRUE_POPULATION + 1)) / dataset.ConfirmedCases_12d_prior_slope
dataset['days_to_3pct_ConfirmedCases_4d'] =(- np.log(dataset.ConfirmedCases + 1)+ np.log(dataset.TRUE_POPULATION + 1)- 3.5)/ dataset.ConfirmedCases_4d_prior_slope
dataset['days_to_3pct_ConfirmedCases_7d'] =(- np.log(dataset.ConfirmedCases + 1)+ np.log(dataset.TRUE_POPULATION + 1)- 3.5)/ dataset.ConfirmedCases_7d_prior_slope
dataset['days_to_0.3pct_Fatalities_20d_cases'] =(- np.log(dataset.Fatalities + 1)+ np.log(dataset.TRUE_POPULATION + 1)- 5.8)/ dataset.ConfirmedCases_20d_prior_slope
dataset['days_to_0.3pct_Fatalities_12d_cases'] =(- np.log(dataset.Fatalities + 1)+ np.log(dataset.TRUE_POPULATION + 1)- 5.8)/ dataset.ConfirmedCases_12d_prior_slope
dataset.tail()
dataset = dataset[dataset.ConfirmedCases > 0]
len(dataset)
 datas = []
for window in range(1, 35):
base = rollDates(dataset, window, True)
 datas.append(pd.merge(dataset[['Date', 'Place',
'ConfirmedCases', 'Fatalities']], base, on = ['Date', 'Place'],
how = 'right',
suffixes =('_f', '')))
 data = pd.concat(datas, axis =0 ).astype(np.float32, errors ='ignore')
 len(data)
 data['Date_f'] = data.Date
data.Date = data.Date_i
data['elapsed'] =(data.Date_f - data.Date_i ).dt.days
data['CaseChgRate'] =(np.log(data.ConfirmedCases_f + 1)- np.log(data.ConfirmedCases + 1)) / data.elapsed;
data['FatalityChgRate'] =(np.log(data.Fatalities_f + 1)- np.log(data.Fatalities + 1)) / data.elapsed;
data.elapsed
falloff_hash = {}
def true_agg(rate_i, elapsed, bend_rate):
elapsed = int(elapsed)
 if(bend_rate, elapsed)not in falloff_hash:
falloff_hash[(bend_rate, elapsed)] = np.sum([ np.power(bend_rate, e)for e in range(1, elapsed+1)])
 return falloff_hash[(bend_rate, elapsed)] * rate_i
true_agg(0.3, 30, 0.9)
 slope_cols = [c for c in data.columns if
any(z in c for z in ['prior_slope', 'chg', 'rate'])
 and not any(z in c for z in ['bend', 'prior_slope_chg', 'Country', 'ewm',
])]
print(slope_cols)
 bend_rates = [1, 0.95, 0.90]
for bend_rate in bend_rates:
bend_agg = data[['elapsed']].apply(lambda x: true_agg(1, *x, bend_rate), axis=1)
 for sc in slope_cols:
if bend_rate < 1:
data[sc+"_slope_bend_{}".format(bend_rate)] = data[sc] * np.power(( bend_rate + 1)/2, data.elapsed)
 data[sc+"_true_slope_bend_{}".format(bend_rate)] = bend_agg * data[sc] / data.elapsed
data[sc+"_agg_bend_{}".format(bend_rate)] = data[sc] * data.elapsed * np.power(( bend_rate + 1)/2, data.elapsed)
 data[sc+"_true_agg_bend_{}".format(bend_rate)] = bend_agg * data[sc]
slope_cols[:5]
for col in [c for c in data.columns if any(z in c for z in
['vs_continent', 'nearest', 'vs_world', 'surrounding_places'])]:
data[col + '_times_days'] = data[col] * data.elapsed
data['saturation_slope_ConfirmedCases'] =(- np.log(data.ConfirmedCases + 1)+ np.log(data.TRUE_POPULATION + 1)) / data.elapsed
data['saturation_slope_Fatalities'] =(- np.log(data.Fatalities + 1)+ np.log(data.TRUE_POPULATION + 1)) / data.elapsed
data['dist_to_ConfirmedCases_saturation_times_days'] =(- np.log(data.ConfirmedCases + 1)+ np.log(data.TRUE_POPULATION + 1)) * data.elapsed
data['dist_to_Fatalities_saturation_times_days'] =(- np.log(data.Fatalities + 1)+ np.log(data.TRUE_POPULATION + 1)) * data.elapsed
data['slope_to_1pct_ConfirmedCases'] =(- np.log(data.ConfirmedCases + 1)+ np.log(data.TRUE_POPULATION + 1)- 4.6)/ data.elapsed
data['slope_to_0.1pct_Fatalities'] =(- np.log(data.Fatalities + 1)+ np.log(data.TRUE_POPULATION + 1)- 6.9)/ data.elapsed
data['dist_to_1pct_ConfirmedCases_times_days'] =(- np.log(data.ConfirmedCases + 1)+ np.log(data.TRUE_POPULATION + 1)- 4.6)* data.elapsed
data['dist_to_0.1pct_Fatalities_times_days'] =(- np.log(data.Fatalities + 1)+ np.log(data.TRUE_POPULATION + 1)- 6.9)* data.elapsed
data['trendline_per_capita_ConfirmedCases_4d_slope'] =(np.log(data.ConfirmedCases + 1)- np.log(data.TRUE_POPULATION + 1)) +(data.ConfirmedCases_4d_prior_slope * data.elapsed)
 data['trendline_per_capita_ConfirmedCases_7d_slope'] =(np.log(data.ConfirmedCases + 1)- np.log(data.TRUE_POPULATION + 1)) +(data.ConfirmedCases_7d_prior_slope * data.elapsed)
 data['trendline_per_capita_Fatalities_12d_slope'] =(np.log(data.Fatalities + 1)- np.log(data.TRUE_POPULATION + 1)) +(data.ConfirmedCases_12d_prior_slope * data.elapsed)
 data['trendline_per_capita_Fatalities_20d_slope'] =(np.log(data.Fatalities + 1)- np.log(data.TRUE_POPULATION + 1)) +(data.ConfirmedCases_20d_prior_slope * data.elapsed)
 len(data)
 data.groupby('Place' ).last()
def logHist(x, b = 150):
return
data['log_fatalities'] = np.log(data.Fatalities + 1)
 data['log_cases'] = np.log(data.ConfirmedCases + 1)
 data['is_China'] =(data.Country=='China')&(~data.Place.isin(['Hong Kong', 'Macau']))
for col in [c for c in data.columns if 'd_ewm' in c]:
data[col] += np.random.normal(0, 1, len(data)) * np.std(data[col])* 0.2
data['is_province'] = 1.0*(~data.Province_State.isnull())
 data['log_elapsed'] = np.log(data.elapsed + 1)
 data.columns
data.columns[::19]
data.shape
logHist(data.ConfirmedCases)
 data.drop(columns = ['TRUE_POPULATION'], inplace=True)
 data['final_day_of_week'] = data.Date_f.apply(datetime.datetime.weekday)
 data['base_date_day_of_week'] = data.Date.apply(datetime.datetime.weekday)
 data['date_difference_modulo_7_days'] =(data.Date_f - data.Date ).dt.days % 7
for c in data.columns.to_list() :
if 'days_to' in c:
data[c] = data[c].where(~np.isinf(data[c]), 1e3)
 data[c] = np.clip(data[c], 0, 365)
 data[c] = np.sqrt(data[c])
 new_places = train[(train.Date == test.Date.min() - datetime.timedelta(1)) & 
(train.ConfirmedCases == 0)
 ].Place
model_data = data[(( len(test)==0)|(data.Date_f < test.Date.min()))
& 
(data.ConfirmedCases > 0)& 
(~data.ConfirmedCases_f.isnull())].copy()
test.Date.min()
model_data.Date_f.max()
model_data.Date_f.max()
model_data.Date.max()
model_data.Date_f.min()
model_data = model_data[~(
(np.random.rand(len(model_data)) < 0.8)& 
(model_data.Country == 'China')& 
(model_data.Date < datetime.datetime(2020, 2, 15)))]
x_dates = model_data[['Date_i', 'Date_f', 'Place']]
x = model_data[
model_data.columns.to_list() [
model_data.columns.to_list().index('ConfirmedCases_1d_prior_value'):]]\ 
.drop(columns = ['Date_i', 'Date_f', 'CaseChgRate', 'FatalityChgRate'])
 test.Date
if PRIVATE:
data_test = data[(data.Date_i == train.Date.max())& 
(data.Date_f.isin(test.Date.unique())) ].copy()
else:
data_test = data[(data.Date_i == test.Date.min() - datetime.timedelta(1)) & 
(data.Date_f.isin(test.Date.unique())) ].copy()
data_test.Date.unique()
test.Date.unique()
x_test = data_test[x.columns].copy()
train.Date.max()
test.Date.max()
if MODEL_Y is 'slope':
y_cases = model_data.CaseChgRate
y_fatalities = model_data.FatalityChgRate
else:
y_cases = model_data.CaseChgRate * model_data.elapsed
y_fatalities = model_data.FatalityChgRate * model_data.elapsed
y_cfr = np.log(( model_data.Fatalities_f + np.clip(0.015 * model_data.ConfirmedCases_f, 0, 0.3)) /(model_data.ConfirmedCases_f + 0.1))
groups = model_data.Country
places = model_data.Place
np.random.seed(SEED)
 enet_params = { 'alpha': [ 3e-6, 1e-5, 3e-5, 1e-4, 3e-4, 1e-3, ],
'l1_ratio': [ 0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 0.8, 0.9, 0.97, 0.99 ]}
et_params = { 'n_estimators': [50, 70, 100, 140],
'max_depth': [3, 5, 7, 8, 9, 10],
'min_samples_leaf': [30, 50, 70, 100, 130, 165, 200, 300, 600],
'max_features': [0.4, 0.5, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85],
'min_impurity_decrease': [0, 1e-5 ],
'bootstrap': [ True, False],
}
lgb_params = {
'max_depth': [5, 12],
'n_estimators': [ 100, 200, 300, 500],
'min_split_gain': [0, 0, 1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2],
'min_child_samples': [ 7, 10, 14, 20, 30, 40, 70, 100, 200, 400, 700, 1000, 2000],
'min_child_weight': [0],
'num_leaves': [5, 10, 20, 30],
'learning_rate': [0.05, 0.07, 0.1],
'colsample_bytree': [0.1, 0.2, 0.33, 0.5, 0.65, 0.8, 0.9],
'colsample_bynode':[0.1, 0.2, 0.33, 0.5, 0.65, 0.81],
'reg_lambda': [1e-5, 3e-5, 1e-4, 1e-3, 1e-2, 0.1, 1, 10, 100, 1000, ],
'reg_alpha': [1e-5, 1e-4, 1e-3, 1e-2, 0.1, 1, 30, 1000,],
'subsample': [ 0.8, 0.9, 1],
'subsample_freq': [1],
'max_bin': [ 7, 15, 31, 63, 127, 255],
}
MSE = 'neg_mean_squared_error'
MAE = 'neg_mean_absolute_error'
def trainENet(x, y, groups, cv = 0, **kwargs):
return trainModel(x, y, groups,
clf = ElasticNet(normalize = True, selection = 'random',
max_iter = 3000),
params = enet_params,
cv = cv, **kwargs)
 def trainETR(x, y, groups, cv = 0, n_jobs = 5, **kwargs):
clf = ExtraTreesRegressor(n_jobs = 1)
 params = et_params
return trainModel(x, y, groups, clf, params, cv, n_jobs, **kwargs)
 def trainLGB(x, y, groups, cv = 0, n_jobs = 4, **kwargs):
clf = lgb.LGBMRegressor(verbosity=-1, hist_pool_size = 1000, 
)
 params = lgb_params
return trainModel(x, y, groups, clf, params, cv, n_jobs, **kwargs)
 def trainModel(x, y, groups, clf, params, cv = 0, n_jobs = None,
verbose=0, splits=None, **kwargs):
if n_jobs is None:
n_jobs = 4
if np.random.rand() < 0.8:
folds = GroupShuffleSplit(n_splits=4,
test_size= 0.2 + 0.10 * np.random.rand())
 else:
folds = GroupKFold(4)
 clf = RandomizedSearchCV(clf, params,
cv= folds,
n_iter=12,
verbose = 0, n_jobs = n_jobs, scoring = MSE)
 f = clf.fit(x, y, groups)
 print(pd.DataFrame(clf.cv_results_['mean_test_score'])) ; print() ;
best = clf.best_estimator_; print(best)
 print("Best Score: {}".format(np.round(clf.best_score_,4)))
 return best
np.mean(y_cases)
 def getSparseColumns(x, verbose = 0):
sc = []
for c in x.columns.to_list() :
u = len(x[c].unique())
 if u > 10 and u < 0.01*len(x):
sc.append(c)
 if verbose > 0:
print("{}: {}".format(c, u))
return sc
def noisify(x, noise = 0.1):
x = x.copy()
cols = getSparseColumns(x)
 for c in cols:
u = len(x[c].unique())
 if u > 50:
x[c].values[:] = x[c].values + np.random.normal(0, noise, len(x)) * np.std(x[c])
 return x;
def getMaxOverlap(row, df):
df_place = df[df.Place == row.Place]
if len(df_place)==0:
return 0
overlap =(np.clip(df_place.Date_f, None, row.Date_f)- np.clip(df_place.Date_i, row.Date_i, None)).dt.days
overlap = np.clip(overlap, 0, None)
 length = np.clip(( df_place.Date_f - df_place.Date_i ).dt.days, 
(row.Date_f - row.Date_i ).days, None)
 return np.amax(overlap / length)
 def getSampleWeight(x, groups):
counter = Counter(groups)
 median_count = np.median([counter[group] for group in groups.unique() ])
 c_count = [counter[group] for group in groups]
e_decay = np.round(LT_DECAY_MIN + np.random.rand() *(LT_DECAY_MAX - LT_DECAY_MIN), 1)
 print("LT weight decay: {:.2f}".format(e_decay)) ;
ssr = np.power(1 / np.clip(c_count / median_count , 0.1, 30),
0.1 + np.random.rand() * 0.6)\
/ np.power(x.elapsed / 3, e_decay)\
* SET_FRAC * np.exp(- np.random.rand())
 group_drop = dict([(group, np.random.rand() < 0.15)for group in groups.unique() ])
 ssr = ssr *([ 1 -group_drop[group] for group in groups])
 return ssr;
def runBags(x, y, groups, cv, bags = 3, model_type = trainLGB,
noise = 0.1, splits = None, weights = None, **kwargs):
models = []
for bag in range(bags):
print("
BAG {}".format(bag+1))
x = x.copy()
if DROPS:
for col in [c for c in x.columns if any(z in c for z in ['bend', 'slope', 'prior'])]:
if np.random.rand() < np.sqrt(np.random.rand())* 0.7:
x[col].values[:] = 0
if DROPS and(np.random.rand() < 0.30):
print('dropping nearest features')
 for col in [c for c in x.columns if 'nearest' in c]:
x[col].values[:] = 0
if DROPS and(np.random.rand() < 0.25):
print('dropping 'surrounding places' features')
 for col in [c for c in x.columns if 'surrounding_places' in c]:
x[col].values[:] = 0
col_drop_frac = np.sqrt(np.random.rand())* 0.5
for col in [c for c in x.columns if 'elapsed' not in c ]:
if np.random.rand() < col_drop_frac:
x[col].values[:] = 0
x = noisify(x, noise)
 if DROPS and(np.random.rand() < SUP_DROP):
print("Dropping supplemental country data")
 for col in x[[c for c in x.columns if c in sup_data.columns]]:
x[col].values[:] = 0
if DROPS and(np.random.rand() < ACTIONS_DROP):
for col in x[[c for c in x.columns if c in contain_data.columns]]:
x[col].values[:] = 0
else:
print("*using containment data")
 if np.random.rand() < 0.6:
x.S_data_days = 0
ssr = getSampleWeight(x, groups)
 date_falloff = 0 +(1/30)* np.random.rand()
if weights is not None:
ssr = ssr * np.exp(-weights * date_falloff)
 ss =(np.random.rand(len(y)) < ssr)
 print("n={}".format(len(x[ss])))
 p1 =x.elapsed[ss].plot(kind='hist', bins = int(x.elapsed.max() - x.elapsed.min() + 1))
p1 = plt.figure() ;
print(( ss ).sum())
 models.append(model_type(x[ss], y[ss], groups[ss], cv, **kwargs))
return models
x = x.astype(np.float32)
 BAG_MULT = 1
x.shape
lgb_c_clfs = []; lgb_c_noise = []
date_weights = np.abs(( model_data.Date_f - test.Date.min() ).dt.days)
 for iteration in range(0, int(math.ceil(1.1 * BAGS))):
for noise in [ 0.05, 0.1, 0.2, 0.3, 0.4 ]:
print("
---
Noise of {}".format(noise)) ;
num_bags = 1 * BAG_MULT;
if np.random.rand() < PLACE_FRACTION:
cv_group = places
print("CV by Place")
 else:
cv_group = groups
print("CV by Country")
 lgb_c_clfs.extend(runBags(x, y_cases,
cv_group,
MSE, num_bags, trainLGB, verbose = 0,
noise = noise, weights = date_weights 
))
lgb_c_noise.extend([noise] * num_bags)
 if SINGLE_MODEL:
break;
lgb_f_clfs = []; lgb_f_noise = []
for iteration in range(0, int(np.ceil(np.sqrt(BAGS)))) :
for noise in [ 0.5, 1, 2, 3, ]:
print("
---
Noise of {}".format(noise)) ;
num_bags = 1 * int(np.ceil(np.sqrt(BAG_MULT)))
 if np.random.rand() < PLACE_FRACTION :
cv_group = places
print("CV by Place")
 else:
cv_group = groups
print("CV by Country")
 lgb_f_clfs.extend(runBags(x, y_fatalities,
cv_group,
MSE, num_bags, trainLGB,
verbose = 0, noise = noise,
weights = date_weights 
))
lgb_f_noise.extend([noise] * num_bags)
 if SINGLE_MODEL:
break;
lgb_cfr_clfs = []; lgb_cfr_noise = [];
for iteration in range(0, int(np.ceil(np.sqrt(BAGS)))) :
for noise in [ 0.4, 1, 2, 3]:
print("
---
Noise of {}".format(noise)) ;
num_bags = 1 * BAG_MULT;
if np.random.rand() < 0.5 * PLACE_FRACTION :
cv_group = places
print("CV by Place")
 else:
cv_group = groups
print("CV by Country")
 lgb_cfr_clfs.extend(runBags(x, y_cfr,
cv_group,
MSE, num_bags, trainLGB, verbose = 0,
noise = noise,
weights = date_weights 
))
lgb_cfr_noise.extend([noise] * num_bags)
 if SINGLE_MODEL:
break;
lgb_cfr_clfs[0].predict(x_test)
 def show_FI(model, featNames, featCount):
fis = model.feature_importances_
fig, ax = plt.subplots(figsize=(6, 5))
indices = np.argsort(fis)[::-1][:featCount]
g = sns.barplot(y=featNames[indices][:featCount],
x = fis[indices][:featCount] , orient='h')
 g.set_xlabel("Relative importance")
 g.set_ylabel("Features")
 g.tick_params(labelsize=12)
 g.set_title(" feature importance")
 def avg_FI(all_clfs, featNames, featCount):
clfs = []
for clf_set in all_clfs:
for clf in clf_set:
clfs.append(clf);
print("{} classifiers".format(len(clfs)))
 fi = np.zeros(( len(clfs), len(clfs[0].feature_importances_)))
 for idx, clf in enumerate(clfs):
fi[idx, :] = clf.feature_importances_
avg_fi = np.mean(fi, axis = 0)
 fis = avg_fi
fig, ax = plt.subplots(figsize=(6, 5))
indices = np.argsort(fis)[::-1]
g = sns.barplot(y=featNames[indices][:featCount],
x = fis[indices][:featCount] , orient='h')
 g.set_xlabel("Relative importance")
 g.set_ylabel("Features")
 g.tick_params(labelsize=12)
 g.set_title(" feature importance")
 return pd.Series(fis[indices], featNames[indices])
 def linear_FI_plot(fi, featNames, featCount):
fig, ax = plt.subplots(figsize=(6, 5))
indices = np.argsort(np.absolute(fi)) [::-1]
g = sns.barplot(y=featNames[indices][:featCount],
x = fi[indices][:featCount] , orient='h')
 g.set_xlabel("Relative importance")
 g.set_ylabel("Features")
 g.tick_params(labelsize=12)
 g.set_title(" feature importance")
 return pd.Series(fi[indices], featNames[indices])
 f = avg_FI([lgb_c_clfs], x.columns, 25)
 for feat in ['bend', 'capita', 'cfr', 'slope', 'since', 'chg', 'ersonal',
'world', 'continent', 'nearest', 'surrounding']:
print("{}: {:.2f}".format(feat, f.filter(like=feat ).sum() / f.sum()))
f[:100:3]
print("{}: {:.2f}".format('sup_data',
f[[c for c in f.index if c in sup_data.columns]].sum() / f.sum()))
print("{}: {:.2f}".format('contain_data',
f[[c for c in f.index if c in contain_data.columns]].sum() / f.sum()))
f = avg_FI([lgb_f_clfs], x.columns, 25)
 for feat in ['bend', 'capita', 'cfr', 'slope', 'since', 'chg', 'ersonal',
'world', 'continent', 'nearest', 'surrounding']:
print("{}: {:.2f}".format(feat, f.filter(like=feat ).sum() / f.sum()))
print("{}: {:.2f}".format('sup_data',
f[[c for c in f.index if c in sup_data.columns]].sum() / f.sum()))
print("{}: {:.2f}".format('contain_data',
f[[c for c in f.index if c in contain_data.columns]].sum() / f.sum()))
f = avg_FI([lgb_cfr_clfs], x.columns, 25)
 for feat in ['bend', 'capita', 'cfr', 'slope', 'since', 'chg', 'ersonal',
'world', 'continent', 'nearest', 'surrounding']:
print("{}: {:.2f}".format(feat, f.filter(like=feat ).sum() / f.sum()))
print("{}: {:.2f}".format('sup_data',
f[[c for c in f.index if c in sup_data.columns]].sum() / f.sum()))
print("{}: {:.2f}".format('contain_data',
f[[c for c in f.index if c in contain_data.columns]].sum() / f.sum()))
all_c_clfs = [lgb_c_clfs, ]
all_f_clfs = [lgb_f_clfs]
all_cfr_clfs = [lgb_cfr_clfs]
all_c_noise = [lgb_c_noise]
all_f_noise = [lgb_f_noise]
all_cfr_noise = [lgb_cfr_noise]
NUM_TEST_RUNS = 1
c_preds = np.zeros(( NUM_TEST_RUNS * sum([len(x)for x in all_c_clfs]), len(x_test)))
 f_preds = np.zeros(( NUM_TEST_RUNS * sum([len(x)for x in all_f_clfs]), len(x_test)))
 cfr_preds = np.zeros(( NUM_TEST_RUNS * sum([len(x)for x in all_cfr_clfs]), len(x_test)))
 def avg(x):
return(np.mean(x, axis=0)+ np.median(x, axis=0)) /2
count = 0
for idx, clf in enumerate(lgb_c_clfs):
for i in range(0, NUM_TEST_RUNS):
noise = lgb_c_noise[idx]
c_preds[count,:] = np.clip(clf.predict(noisify(x_test, noise)) , -1 , 10)
 count += 1
count = 0
for idx, clf in enumerate(lgb_f_clfs):
for i in range(0, NUM_TEST_RUNS):
noise = lgb_f_noise[idx]
f_preds[count,:] = np.clip(clf.predict(noisify(x_test, noise)) , -1 , 10)
 count += 1
count = 0
for idx, clf in enumerate(lgb_cfr_clfs):
for i in range(0, NUM_TEST_RUNS):
noise = lgb_cfr_noise[idx]
cfr_preds[count,:] = np.clip(clf.predict(noisify(x_test, noise)) , -10 , 10)
 count += 1
def qPred(preds, pctile, simple=False):
q = np.percentile(preds, pctile, axis = 0)
 if simple:
return q;
resid = preds - q
resid_wtg = 2/100/len(preds)*(np.clip(resid, 0, None)*(pctile)+ np.clip(resid, None, 0)*(100- pctile))
adj = np.sum(resid_wtg, axis = 0)
 return q + adj
q = 50
y_cases_pred_blended_full = qPred(c_preds, q)
 y_fatalities_pred_blended_full = qPred(f_preds, q)
 y_cfr_pred_blended_full = qPred(cfr_preds, q)
 print(np.mean(np.corrcoef(c_preds[::NUM_TEST_RUNS]),axis=0))
print(np.mean(np.corrcoef(f_preds[::NUM_TEST_RUNS]), axis=0))
print(np.mean(np.corrcoef(cfr_preds[::NUM_TEST_RUNS]), axis = 0))
pd.Series(np.std(c_preds, axis = 0)).plot(kind='hist', bins = 50)
 pd.Series(np.std(f_preds, axis = 0)).plot(kind='hist', bins = 50)
 pd.Series(np.std(cfr_preds, axis = 0)).plot(kind='hist', bins = 50)
 y_cfr 
(groups == 'Sierra Leone' ).sum()
pred = pd.DataFrame(np.hstack(( np.transpose(c_preds),
np.transpose(f_preds))), index=x_test.index)
 pred['Place'] = data_test.Place
pred['Date'] = data_test.Date
pred['Date_f'] = data_test.Date_f
pred[(pred.Date == pred.Date.max())&(pred.Date_f == pred.Date_f.max())][30: 60] 
(pred.Place=='Sierra Leone' ).sum()
np.round(pred[(pred.Date == pred.Date.max())&(pred.Date_f == pred.Date_f.max())], 2)[190:220:]
np.round(pred[(pred.Date == pred.Date.max())&(pred.Date_f == pred.Date_f.max())][220:-20],2)
 c_preds.shape
x_test.shape
data_wp = data_test.copy()
if MODEL_Y is 'slope':
data_wp['case_slope'] = y_cases_pred_blended_full
data_wp['fatality_slope'] = y_fatalities_pred_blended_full
else:
data_wp['case_slope'] = y_cases_pred_blended_full / x_test.elapsed
data_wp['fatality_slope'] = y_fatalities_pred_blended_full / x_test.elapsed
data_wp['cfr_pred'] = y_cfr_pred_blended_full
train.Date.max()
test.Date.min()
if len(test)> 0:
base_date = test.Date.min() - datetime.timedelta(1)
 else:
base_date = train.Date.max()
base_date
data_wp_ss = data_wp[data_wp.Date == base_date]
data_wp_ss = data_wp_ss.drop(columns='Date' ).rename(columns = {'Date_f': 'Date'})
 test_wp = pd.merge(test, data_wp_ss[['Date', 'Place', 'case_slope', 'fatality_slope', 'cfr_pred',
'elapsed']],
how='left', on = ['Date', 'Place'])
 first_c_slope = test_wp[~test_wp.case_slope.isnull() ].groupby('Place' ).first()
last_c_slope = test_wp[~test_wp.case_slope.isnull() ].groupby('Place' ).last()
first_f_slope = test_wp[~test_wp.fatality_slope.isnull() ].groupby('Place' ).first()
last_f_slope = test_wp[~test_wp.fatality_slope.isnull() ].groupby('Place' ).last()
first_cfr_pred = test_wp[~test_wp.cfr_pred.isnull() ].groupby('Place' ).first()
last_cfr_pred = test_wp[~test_wp.cfr_pred.isnull() ].groupby('Place' ).last()
test_wp.case_slope = np.where(test_wp.case_slope.isnull() & 
(test_wp.Date < first_c_slope.loc[test_wp.Place].Date.values),
first_c_slope.loc[test_wp.Place].case_slope.values,
test_wp.case_slope 
)
 test_wp.case_slope = np.where(test_wp.case_slope.isnull() & 
(test_wp.Date > last_c_slope.loc[test_wp.Place].Date.values),
last_c_slope.loc[test_wp.Place].case_slope.values,
test_wp.case_slope 
)
 test_wp.fatality_slope = np.where(test_wp.fatality_slope.isnull() & 
(test_wp.Date < first_f_slope.loc[test_wp.Place].Date.values),
first_f_slope.loc[test_wp.Place].fatality_slope.values,
test_wp.fatality_slope 
)
 test_wp.fatality_slope = np.where(test_wp.fatality_slope.isnull() & 
(test_wp.Date > last_f_slope.loc[test_wp.Place].Date.values),
last_f_slope.loc[test_wp.Place].fatality_slope.values,
test_wp.fatality_slope 
)
 test_wp.cfr_pred = np.where(test_wp.cfr_pred.isnull() & 
(test_wp.Date < first_cfr_pred.loc[test_wp.Place].Date.values),
first_cfr_pred.loc[test_wp.Place].cfr_pred.values,
test_wp.cfr_pred 
)
 test_wp.cfr_pred = np.where(test_wp.cfr_pred.isnull() & 
(test_wp.Date > last_cfr_pred.loc[test_wp.Place].Date.values),
last_cfr_pred.loc[test_wp.Place].cfr_pred.values,
test_wp.cfr_pred 
)
 test_wp.case_slope = test_wp.case_slope.interpolate('linear')
 test_wp.fatality_slope = test_wp.fatality_slope.interpolate('linear')
 test_wp.cfr_pred = test_wp.cfr_pred.interpolate('linear')
 test_wp.case_slope = test_wp.case_slope.fillna(0)
 test_wp.fatality_slope = test_wp.fatality_slope.fillna(0)
 LAST_DATE = test.Date.min() - datetime.timedelta(1)
 final = train_bk[train_bk.Date == LAST_DATE ]
test_wp = pd.merge(test_wp, final[['Place', 'ConfirmedCases', 'Fatalities']], on='Place',
how ='left', validate='m:1')
 LAST_DATE
test_wp.ConfirmedCases = np.exp(
 np.log(test_wp.ConfirmedCases + 1)\
+ test_wp.case_slope * 
(test_wp.Date - LAST_DATE ).dt.days)- 1
test_wp.Fatalities = np.exp(
 np.log(test_wp.Fatalities + 1)\
+ test_wp.fatality_slope * 
(test_wp.Date - LAST_DATE ).dt.days)-1
LAST_DATE
test_wp[(test_wp.Country == 'Italy')].groupby('Date' ).sum() [:10]
test_wp[(test_wp.Country == 'US')].groupby('Date' ).sum().iloc[-5:]
final = train_bk[train_bk.Date == test.Date.min() - datetime.timedelta(1)]
final.head()
test['elapsed'] =(test.Date - final.Date.max() ).dt.days
test.elapsed
full_bk = test_wp.copy()
full = test_wp.copy()
BASE_RATE = 0.01
CFR_CAP = 0.13
lplot(full_bk)
 lplot(full_bk, columns = ['case_slope', 'fatality_slope'])
 full['cfr_imputed_fatalities_low'] = full.ConfirmedCases * np.exp(full.cfr_pred)/ np.exp(0.5)
 full['cfr_imputed_fatalities_high'] = full.ConfirmedCases * np.exp(full.cfr_pred)* np.exp(0.5)
 full['cfr_imputed_fatalities'] = full.ConfirmedCases * np.exp(full.cfr_pred)
 full[(full.case_slope > 0.02)& 
(full.Fatalities < full.cfr_imputed_fatalities_low)& 
(full.cfr_imputed_fatalities_low > 0.3)& 
(full.Fatalities < 100000)& 
(full.Country!='China')& 
(full.Date == datetime.datetime(2020, 4,15)) ] \ 
.groupby('Place' ).last() \ 
.sort_values('Fatalities', ascending=False ).iloc[:, 9:] 
(np.log(full.Fatalities + 1)-np.log(full.cfr_imputed_fatalities)).plot(kind='hist', bins = 250)
 full[(full.case_slope > 0.02)& 
(full.Fatalities < full.cfr_imputed_fatalities_low)& 
(full.cfr_imputed_fatalities_low > 0.3)& 
(full.Fatalities < 100000)& 
(~full.Country.isin(['China', 'Korea, South'])) ][full.Date==train.Date.max() ]\ 
.groupby('Place' ).first() \ 
.sort_values('cfr_imputed_fatalities', ascending=False ).iloc[:, 9:]
full.Fatalities = np.where(
(full.case_slope > 0.02)& 
(full.Fatalities <= full.cfr_imputed_fatalities_low)& 
(full.cfr_imputed_fatalities_low > 0.3)& 
(full.Fatalities < 100000)& 
(~full.Country.isin(['China', 'Korea, South'])) , 
(full.cfr_imputed_fatalities_high + full.cfr_imputed_fatalities)/2,
full.Fatalities)
 full['elapsed'] =(test_wp.Date - LAST_DATE ).dt.days
full[(full.case_slope > 0.02)& 
(np.log(full.Fatalities + 1)< np.log(full.ConfirmedCases * BASE_RATE + 1)- 0.5)& 
(full.Country != 'China')]\
[full.Date == datetime.datetime(2020, 4, 5)] \ 
.groupby('Place' ).last().sort_values('ConfirmedCases', ascending=False ).iloc[:,8:]
full.Fatalities = np.where(( full.case_slope > 0.02)& 
(full.Fatalities < full.ConfirmedCases * BASE_RATE)& 
(full.Country != 'China'),
np.exp(
 np.log(full.ConfirmedCases * BASE_RATE + 1)\
* np.clip(0.5*(full.elapsed - 1)/ 30, 0, 1)\
+ np.log(full.Fatalities +1)\
* np.clip(1 - 0.5*(full.elapsed - 1)/ 30, 0, 1)
)-1
,
full.Fatalities)
 full[(full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China')]\ 
.groupby('Place' ).count() \ 
.sort_values('ConfirmedCases', ascending=False ).iloc[:, 8:]
full[(full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high * 2)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China')]\ 
.groupby('Place' ).last() \ 
.sort_values('ConfirmedCases', ascending=False ).iloc[:, 8:]
full[(full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high * 1.5)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China')][full.Date==train.Date.max() ]\ 
.groupby('Place' ).first() \ 
.sort_values('ConfirmedCases', ascending=False ).iloc[:, 8:]
full.Fatalities = np.where(( full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high * 2)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China'),
full.cfr_imputed_fatalities,
full.Fatalities)
 full.Fatalities = np.where(( full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China'),
np.exp(
 0.6667 * np.log(full.Fatalities + 1)\
+ 0.3333 * np.log(full.cfr_imputed_fatalities + 1)
)- 1,
full.Fatalities)
 full[(full.Fatalities > full.ConfirmedCases * CFR_CAP)& 
(full.ConfirmedCases > 1000)
 ].groupby('Place' ).last().sort_values('Fatalities', ascending=False)
(np.log(full.Fatalities + 1)-np.log(full.cfr_imputed_fatalities)).plot(kind='hist', bins = 250)
 assert len(pd.merge(full, final, on='Place', suffixes =('', '_i'), validate='m:1')) == len(full)
 ffm = pd.merge(full, final, on='Place', suffixes =('', '_i'), validate='m:1')
 ffm['fatality_slope'] =(np.log(ffm.Fatalities + 1)- np.log(ffm.Fatalities_i + 1)) / ffm.elapsed
ffm['case_slope'] =(np.log(ffm.ConfirmedCases + 1)- np.log(ffm.ConfirmedCases_i + 1)) / ffm.elapsed
ffm[np.log(ffm.Fatalities+1)< np.log(ffm.Fatalities_i+1)- 0.2] [['Place', 'Date', 'elapsed', 'Fatalities', 'Fatalities_i']]
ffm[np.log(ffm.ConfirmedCases + 1)< np.log(ffm.ConfirmedCases_i+1)- 0.2] [['Place', 'elapsed', 'ConfirmedCases', 'ConfirmedCases_i']]
full_bk[(full_bk.Date == test.Date.max())& 
(~full_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)
 lplot(ffm[~ffm.Place.isin(new_places)])
 lplot(ffm[~ffm.Place.isin(new_places)], columns = ['case_slope', 'fatality_slope'])
 ffm.fatality_slope = np.clip(ffm.fatality_slope, None, 0.5)
 for lr in [0.2, 0.14, 0.1, 0.07, 0.05, 0.03, 0.01 ]:
ffm.loc[(ffm.Place==ffm.Place.shift(4))
&(ffm.Place==ffm.Place.shift(-4)) , 'fatality_slope'] = \ 
(ffm.fatality_slope.shift(-2)* 0.25 \
+ ffm.fatality_slope.shift(-1)* 0.5 \
+ ffm.fatality_slope \
+ ffm.fatality_slope.shift(1)* 0.5 \
+ ffm.fatality_slope.shift(2)* 0.25)/ 2.5
ffm.ConfirmedCases = np.exp(
 np.log(ffm.ConfirmedCases_i + 1)\
+ ffm.case_slope *
ffm.elapsed)- 1
ffm.Fatalities = np.exp(
 np.log(ffm.Fatalities_i + 1)\
+ ffm.fatality_slope *
ffm.elapsed)- 1
lplot(ffm[~ffm.Place.isin(new_places)])
 lplot(ffm[~ffm.Place.isin(new_places)], columns = ['case_slope', 'fatality_slope'])
 ffm[(ffm.Date == test.Date.max())& 
(~ffm.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)
 ffm_bk = ffm.copy()
ffm = ffm_bk.copy()
counter = Counter(data.Place)
 median_count = np.median([ counter[group] for group in ffm.Place])
 c_count = [ np.clip(
 np.power(counter[group] / median_count, -1.5), None, 2.5)for group in ffm.Place]
RATE_MULT = 0.00
RATE_ADD = 0.003
LAG_FALLOFF = 15
ma_factor = np.clip(( ffm.elapsed - 14)/ 14 , 0, 1)
 ffm.case_slope = np.where(ffm.elapsed > 0,
0.7 * ffm.case_slope *(1+ ma_factor * RATE_MULT)\
+ 0.3 *(ffm.case_slope.ewm(span=LAG_FALLOFF ).mean() \
* np.clip(ma_factor, 0, 1)
 + ffm.case_slope * np.clip(1 - ma_factor, 0, 1))
+ RATE_ADD * ma_factor * c_count,
ffm.case_slope)
 RATE_MULT = 0
RATE_ADD = 0.015
LAG_FALLOFF = 15
ma_factor = np.clip(( ffm.elapsed - 10)/ 14 , 0, 1)
 ffm.fatality_slope = np.where(ffm.elapsed > 0,
0.3 * ffm.fatality_slope *(1+ ma_factor * RATE_MULT)\
+ 0.7*(ffm.fatality_slope.ewm(span=LAG_FALLOFF ).mean() \
* np.clip(ma_factor, 0, 1)
 + ffm.fatality_slope * np.clip(1 - ma_factor, 0, 1))
+ RATE_ADD * ma_factor * c_count \
*(ffm.Country != 'China')
 ,
ffm.case_slope)
 ffm.ConfirmedCases = np.exp(
 np.log(ffm.ConfirmedCases_i + 1)\
+ ffm.case_slope *
ffm.elapsed)- 1
ffm.Fatalities = np.exp(
 np.log(ffm.Fatalities_i + 1)\
+ ffm.fatality_slope *
ffm.elapsed)- 1
lplot(ffm[~ffm.Place.isin(new_places)])
 lplot(ffm[~ffm.Place.isin(new_places)], columns = ['case_slope', 'fatality_slope'])
 ffm_bk[(ffm_bk.Date == test.Date.max())& 
(~ffm_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[:15]
ffm[(ffm.Date == test.Date.max())& 
(~ffm.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[:15]
ffm_bk[(ffm_bk.Date == test.Date.max())& 
(~ffm_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[-50:]
ffm[(ffm.Date == test.Date.max())& 
(~ffm.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).loc[ffm_bk[(ffm_bk.Date == test.Date.max())& 
(~ffm_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[-50:].index]
NUM_TEST_DATES = len(test.Date.unique())
 base = np.zeros(( 2, NUM_TEST_DATES))
base2 = np.zeros(( 2, NUM_TEST_DATES))
for idx, c in enumerate(['ConfirmedCases', 'Fatalities']):
for n in range(0, NUM_TEST_DATES):
base[idx,n] = np.mean(
 np.log(train[(( train.Date < test.Date.min())) & 
(train.ConfirmedCases > 0)].groupby('Country' ).nth(n)[c]+1))
base = np.pad(base,(( 0,0),(6,0)) , mode='constant', constant_values = 0)
 for n in range(0, base2.shape[1]):
base2[:, n] = np.mean(base[:, n+0: n+7], axis = 1)
 new_places = train[(train.Date == test.Date.min() - datetime.timedelta(1)) & 
(train.ConfirmedCases == 0)
 ].Place
ffm.ConfirmedCases = np.where(ffm.Place.isin(new_places),
base2[ 0,(ffm.Date - test.Date.min() ).dt.days],
ffm.ConfirmedCases)
 ffm.Fatalities = np.where(ffm.Place.isin(new_places),
base2[ 1,(ffm.Date - test.Date.min() ).dt.days],
ffm.Fatalities)
 ffm[ffm.Country=='US'].groupby('Date' ).agg(
 {'ForecastId': 'count',
'case_slope': 'mean',
'fatality_slope': 'mean',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
})
 sub = pd.read_csv(input_path + '/submission.csv')
 scl = sub.columns.to_list()
print(full_bk.groupby('Place' ).last() [['Date', 'ConfirmedCases', 'Fatalities']])
 print(ffm.groupby('Place' ).last() [['Date', 'ConfirmedCases', 'Fatalities']])
 if ffm[scl].isnull().sum().sum() == 0:
out = full_bk[scl] * 0.7 + ffm[scl] * 0.3
else:
print('using full-bk')
 out = full_bk[scl]
out = out[sub.columns.to_list() ]
out.ForecastId = np.round(out.ForecastId, 0 ).astype(int)
 out = np.round(out, 2)
 private = out
full_pred = pd.concat(( private, public[~public.ForecastId.isin(private.ForecastId)]),
ignore_index=True ).sort_values('ForecastId')
 sub1.sort_values("ForecastId", inplace=True)
 sub4=full_pred
sub4.sort_values("ForecastId", inplace=True)
 train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 dacheckv='2020-04-01'
train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train=train[train.Date>dacheckv]
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test1=test.merge(train,on=['Country_Region','Province_State','Date'],how='inner')
 print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['ConfirmedCases'].values,test1['ConfirmedCases'].values))
print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['Fatalities'].values,test1['Fatalities'].values))
print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['ConfirmedCases'].values,test1['ConfirmedCases'].values))
print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['Fatalities'].values,test1['Fatalities'].values))
print(RMSLE(sub4[sub4.ForecastId.isin(test1.ForecastId.values)]['ConfirmedCases'].values,test1['ConfirmedCases'].values))
print(RMSLE(sub4[sub4.ForecastId.isin(test1.ForecastId.values)]['Fatalities'].values,test1['Fatalities'].values))
sub1=sub1.reset_index(drop=True)
 sub4=sub4.reset_index(drop=True)
 sub_df = sub1.copy()
TARGETS = ["ConfirmedCases", "Fatalities"]
for t in TARGETS:
if t=='ConfirmedCases':
sub_df[t] = sub1[t]*0.01 + sub4[t]*0.99
else:
sub_df[t] = sub1[t]*0.01 + sub4[t]*0.99
sub_df.to_csv("submission.csv", index=False)
 print(RMSLE(sub_df[sub_df.ForecastId.isin(test1.ForecastId.values)]['ConfirmedCases'].values,test1['ConfirmedCases'].values))
print(RMSLE(sub_df[sub_df.ForecastId.isin(test1.ForecastId.values)]['Fatalities'].values,test1['Fatalities'].values))