Kaggle competition: APTOS 2019 Blindness Detection
def split_date(date):
date = date.split('-')
 date[0] = int(date[0])
 if(date[1][0] == '0'):
date[1] = int(date[1][1])
 else:
date[1] = int(date[1])
 if(date[2][0] == '0'):
date[2] = int(date[2][1])
 else:
date[2] = int(date[2])
 return date
train_df.Date = train_df.Date.apply(split_date)
 test_df.Date = test_df.Date.apply(split_date )<feature_engineering>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
year = []
month = []
day = []
for i in train_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2] )<feature_engineering>
from fastai import *
from fastai.vision import *
import torch
train_df['Year'] = year
train_df['Month'] = month
train_df['Day'] = day
del train_df['Date']<feature_engineering>
path = Path('.. /input/aptos2019-blindness-detection')
 path_train = path/'train_images'
path_test = path/'test_images'
path, path_train, path_test
year = []
month = []
day = []
for i in test_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2] )<feature_engineering>
labels = pd.read_csv(path/'train.csv')
 labels.head()
test_df['Year'] = year
test_df['Month'] = month
test_df['Day'] = day
del test_df['Date']
del train_df['Id']
del test_df['ForecastId']<train_model>
tfms = get_transforms(
 do_flip=True,
flip_vert=False,
max_warp=0.2,
max_rotate=360.,
max_zoom=1.2,
max_lighting=0.1,
p_lighting=0.5
)
rf = XGBRegressor(n_estimators = 1500 , random_state = 0 , max_depth = 15)
 rf.fit(x_train,cases)
 cases_pred = rf.predict(x_test)
 cases_pred = np.around(cases_pred,decimals = 0)
 x_train_cas = []
for i in range(len(x_train)) :
x = list(x_train[i])
 x.append(cases[i])
 x_train_cas.append(x)
 x_train_cas[0]
x_train_cas = np.array(x_train_cas)
 rf = XGBRegressor(n_estimators = 1500 , random_state = 0 , max_depth = 15)
 rf.fit(x_train_cas,fatalities)
 x_test_cas = []
for i in range(len(x_test)) :
x = list(x_test[i])
 x.append(cases_pred[i])
 x_test_cas.append(x)
 x_test_cas[0]
x_test_cas = np.array(x_test_cas)
 fatalities_pred = rf.predict(x_test_cas)
 fatalities_pred = np.around(fatalities_pred,decimals = 0)
 submission['ConfirmedCases'] = cases_pred
submission['Fatalities'] = fatalities_pred
submission.to_csv("submission.csv" , index = False )<set_options>
aptos19_stats =([0.42, 0.22, 0.075], [0.27, 0.15, 0.081] )
get_ipython().run_line_magic('matplotlib', 'inline')
 get_ipython().run_line_magic('config', "InlineBackend.figure_format = 'retina'")
 def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 testing_result=False
dateck='2020-04-03'
def get_cpmp_sub(save_oof=False, save_public_test=False):
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 if testing_result :train=train[train.Date<=dateck]
train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test
day_min = train['day'].min()
train['day'] -= day_min
test['day'] -= day_min
min_test_val_day = test.day.min()
max_test_val_day = train.day.max()
max_test_day = test.day.max()
num_days = max_test_day + 1
min_test_val_day, max_test_val_day, num_days
train['ForecastId'] = -1
test['Id'] = -1
test['ConfirmedCases'] = 0
test['Fatalities'] = 0
debug = False
data = pd.concat([train,
test[test.day > max_test_val_day][train.columns]
] ).reset_index(drop=True)
 if debug:
data = data[data['geo'] >= 'France_'].reset_index(drop=True)
 gc.collect()
dates = data[data['geo'] == 'France_'].Date.values
if 0:
gr = data.groupby('geo')
 data['ConfirmedCases'] = gr.ConfirmedCases.transform('cummax')
 data['Fatalities'] = gr.Fatalities.transform('cummax')
 geo_data = data.pivot(index='geo', columns='day', values='ForecastId')
 num_geo = geo_data.shape[0]
geo_data
geo_id = {}
for i,g in enumerate(geo_data.index):
geo_id[g] = i
ConfirmedCases = data.pivot(index='geo', columns='day', values='ConfirmedCases')
 Fatalities = data.pivot(index='geo', columns='day', values='Fatalities')
 if debug:
cases = ConfirmedCases.values
deaths = Fatalities.values
else:
cases = np.log1p(ConfirmedCases.values)
 deaths = np.log1p(Fatalities.values)
 def get_dataset(start_pred, num_train, lag_period):
days = np.arange(start_pred - num_train + 1, start_pred + 1)
 lag_cases = np.vstack([cases[:, d - lag_period : d] for d in days])
 lag_deaths = np.vstack([deaths[:, d - lag_period : d] for d in days])
 target_cases = np.vstack([cases[:, d : d + 1] for d in days])
 target_deaths = np.vstack([deaths[:, d : d + 1] for d in days])
 geo_ids = np.vstack([geo_ids_base for d in days])
 country_ids = np.vstack([country_ids_base for d in days])
 return lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days
def update_valid_dataset(data, pred_death, pred_case):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
day = days[-1] + 1
new_lag_cases = np.hstack([lag_cases[:, 1:], pred_case])
 new_lag_deaths = np.hstack([lag_deaths[:, 1:], pred_death])
 new_target_cases = cases[:, day:day+1]
new_target_deaths = deaths[:, day:day+1]
new_geo_ids = geo_ids
new_country_ids = country_ids
new_days = 1 + days
return new_lag_cases, new_lag_deaths, new_target_cases, new_target_deaths, new_geo_ids, new_country_ids, new_days
def fit_eval(lr_death, lr_case, data, start_lag_death, end_lag_death, num_lag_case, fit, score):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], country_ids])
 X_death = np.hstack([lag_deaths[:, -num_lag_case:], country_ids])
 X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], lag_deaths[:, -num_lag_case:], country_ids])
 y_death = target_deaths
y_death_prev = lag_deaths[:, -1:]
if fit:
if 0:
keep =(y_death > 0 ).ravel()
X_death = X_death[keep]
y_death = y_death[keep]
y_death_prev = y_death_prev[keep]
lr_death.fit(X_death, y_death)
 y_pred_death = lr_death.predict(X_death)
 y_pred_death = np.maximum(y_pred_death, y_death_prev)
 X_case = np.hstack([lag_cases[:, -num_lag_case:], geo_ids])
 X_case = lag_cases[:, -num_lag_case:]
y_case = target_cases
y_case_prev = lag_cases[:, -1:]
if fit:
lr_case.fit(X_case, y_case)
 y_pred_case = lr_case.predict(X_case)
 y_pred_case = np.maximum(y_pred_case, y_case_prev)
 if score:
death_score = val_score(y_death, y_pred_death)
 case_score = val_score(y_case, y_pred_case)
 else:
death_score = 0
case_score = 0
return death_score, case_score, y_pred_death, y_pred_case
def train_model(train, valid, start_lag_death, end_lag_death, num_lag_case, num_val, score=True):
alpha = 3
lr_death = Ridge(alpha=alpha, fit_intercept=False)
 lr_case = Ridge(alpha=alpha, fit_intercept=True)
(train_death_score, train_case_score, train_pred_death, train_pred_case, 
)= fit_eval(lr_death, lr_case, train, start_lag_death, end_lag_death, num_lag_case, fit=True, score=score)
 death_scores = []
case_scores = []
death_pred = []
case_pred = []
for i in range(num_val): 
(valid_death_score, valid_case_score, valid_pred_death, valid_pred_case, 
)= fit_eval(lr_death, lr_case, valid, start_lag_death, end_lag_death, num_lag_case, fit=False, score=score)
 death_scores.append(valid_death_score)
 case_scores.append(valid_case_score)
 death_pred.append(valid_pred_death)
 case_pred.append(valid_pred_case)
 if 0:
print('val death: %0.3f' % valid_death_score,
'val case: %0.3f' % valid_case_score,
'val : %0.3f' % np.mean([valid_death_score, valid_case_score]),
flush=True)
 valid = update_valid_dataset(valid, valid_pred_death, valid_pred_case)
 if score:
death_scores = np.sqrt(np.mean([s**2 for s in death_scores]))
case_scores = np.sqrt(np.mean([s**2 for s in case_scores]))
if 0:
print('train death: %0.3f' % train_death_score,
'train case: %0.3f' % train_case_score,
'val death: %0.3f' % death_scores,
'val case: %0.3f' % case_scores,
'val : %0.3f' %(( death_scores + case_scores)/ 2),
flush=True)
 else:
print('%0.4f' % case_scores,
', %0.4f' % death_scores,
'= %0.4f' %(( death_scores + case_scores)/ 2),
flush=True)
 death_pred = np.hstack(death_pred)
 case_pred = np.hstack(case_pred)
 return death_scores, case_scores, death_pred, case_pred
countries = [g.split('_')[0] for g in geo_data.index]
countries = pd.factorize(countries)[0]
country_ids_base = countries.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 country_ids_base = 0.2 * ohe.fit_transform(country_ids_base)
 country_ids_base.shape
geo_ids_base = np.arange(num_geo ).reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 geo_ids_base = 0.1 * ohe.fit_transform(geo_ids_base)
 geo_ids_base.shape
def val_score(true, pred):
pred = np.log1p(np.round(np.expm1(pred)- 0.2))
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def val_score(true, pred):
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
start_lag_death, end_lag_death = 14, 6,
num_train = 5
num_lag_case = 14
lag_period = max(start_lag_death, num_lag_case)
 def get_oof(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[start_val], start_val, num_val)
 train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = train[['Date', 'Id', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[sub.day >= start_val]
sub = sub[['Id', 'ConfirmedCases', 'Fatalities']].copy()
return sub
if save_oof:
for start_val_delta, date in zip(range(3, -8, -3),
['2020-03-22', '2020-03-19', '2020-03-16', '2020-03-13']):
print(date, end=' ')
 oof = get_oof(start_val_delta)
 oof.to_csv('.. /submissions/cpmp-%s.csv' % date, index=None)
 def get_sub(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = 14
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub.fillna(0)
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
return sub
return sub
known_test = train[['geo', 'day', 'ConfirmedCases', 'Fatalities']
].merge(test[['geo', 'day', 'ForecastId']], how='left', on=['geo', 'day'])
 known_test = known_test[['ForecastId', 'ConfirmedCases', 'Fatalities']][known_test.ForecastId.notnull() ].copy()
known_test
unknow_test = test[test.day > max_test_val_day]
unknow_test
def get_final_sub() :
start_val = max_test_val_day + 1
last_train = start_val - 1
num_val = max_test_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = num_val + 3
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
(_, _, val_death_preds, val_case_preds 
)= train_model(train_data, valid_data, start_lag_death, end_lag_death, num_lag_case, num_val, score=False)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
print(unknow_test.shape, pred_deaths.shape, pred_cases.shape)
 sub = unknow_test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
sub = pd.concat([known_test, sub])
 return sub
if save_public_test:
sub = get_sub()
else:
sub = get_final_sub()
return sub
sub1 = get_cpmp_sub()
sub1['ForecastId'] = sub1['ForecastId'].astype('int')
 sub1.sort_values("ForecastId", inplace=True)
 train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 dacheckv='2020-04-01'
train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train=train[train.Date>dacheckv]
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test1=test.merge(train,on=['Country_Region','Province_State','Date'],how='inner')
 print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['ConfirmedCases'].values,test1['ConfirmedCases'].values))
print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['Fatalities'].values,test1['Fatalities'].values))
BAGS = 3
SEED = 123
SET_FRAC = 0.01
TRUNCATED = False
DROPS = True
PRIVATE = True
USE_PRIORS = False
SUP_DROP = 0.0
ACTIONS_DROP = 0.0
PLACE_FRACTION = 1.0
LT_DECAY_MAX = 0.3
LT_DECAY_MIN = -0.4
SINGLE_MODEL = False
MODEL_Y = 'agg_dff'
pd.options.display.float_format = '{:.8}'.format
plt.rcParams["figure.figsize"] =(12, 4.75)
 InteractiveShell.ast_node_interactivity = "all"
pd.options.display.max_rows = 999
path = '/kaggle/input/c19week3'
input_path = '/kaggle/input/covid19-global-forecasting-week-4'
train = pd.read_csv(input_path + '/train.csv')
 if testing_result :train=train[train.Date<=dateck]
test = pd.read_csv(input_path + '/test.csv')
 sub = pd.read_csv(input_path + '/submission.csv')
 tt = pd.merge(train, test, on=['Country_Region', 'Province_State', 'Date'],
how='right', validate="1:1")\ 
.fillna(method = 'ffill')
 public = tt[['ForecastId', 'ConfirmedCases', 'Fatalities']]
train.Date.max()
test_dates = test.Date.unique()
test_dates
pp = 'public'
if PRIVATE:
test = test[ pd.to_datetime(test.Date)> train.Date.max() ]
pp = 'private'
test.Date.unique()
revised = pd.read_csv(path + '/outside_data' +
'/covid19_train_data_us_states_before_march_09_new.csv')
 revised = revised[['Province_State', 'Country_Region', 'Date', 'ConfirmedCases', 'Fatalities']]
train.tail()
revised.head()
train.Date = pd.to_datetime(train.Date)
 revised.Date = pd.to_datetime(revised.Date)
 rev_train = pd.merge(train, revised, on=['Province_State', 'Country_Region', 'Date'],
suffixes =('', '_r'), how='left')
 rev_train[~rev_train.ConfirmedCases_r.isnull() ].head()
rev_train.ConfirmedCases = np.where(( rev_train.ConfirmedCases == 0)&(( rev_train.ConfirmedCases_r > 0)) & 
(rev_train.Country_Region == 'US'),
rev_train.ConfirmedCases_r,
rev_train.ConfirmedCases)
 rev_train.Fatalities = np.where(~rev_train.Fatalities_r.isnull() & 
(rev_train.Fatalities == 0)&(( rev_train.Fatalities_r > 0)) & 
(rev_train.Country_Region == 'US')
 ,
rev_train.Fatalities_r,
rev_train.Fatalities)
 rev_train.drop(columns = ['ConfirmedCases_r', 'Fatalities_r'], inplace=True)
 train = rev_train
contain_data = pd.read_csv(path + '/outside_data' +
'/OxCGRT_Download_070420_160027_Full.csv')
 contain_data = contain_data[[c for c in contain_data.columns if
not any(z in c for z in ['_Notes','Unnamed', 'Confirmed',
'CountryCode',
'S8', 'S9', 'S10','S11',
'StringencyIndexForDisplay'])] ]\
contain_data.rename(columns = {'CountryName': "Country"}, inplace=True)
 contain_data.Date = contain_data.Date.astype(str ).apply(datetime.datetime.strptime, args=('%Y%m%d',))
contain_data_orig = contain_data.copy()
contain_data.columns
cds = []
for country in contain_data.Country.unique() :
cd = contain_data[contain_data.Country==country]
cd = cd.fillna(method = 'ffill' ).fillna(0)
 cd.StringencyIndex = cd.StringencyIndex.cummax()
col_count = cd.shape[1]
for col in [c for c in contain_data.columns if 'S' in c]:
col_diff = cd[col].diff()
cd[col+"_chg_5d_ewm"] = col_diff.ewm(span = 5 ).mean()
cd[col+"_chg_20_ewm"] = col_diff.ewm(span = 20 ).mean()
cd['StringencyIndex_5d_ewm'] = cd.StringencyIndex.ewm(span = 5 ).mean()
cd['StringencyIndex_20d_ewm'] = cd.StringencyIndex.ewm(span = 20 ).mean()
cd['S_data_days'] =(cd.Date - cd.Date.min() ).dt.days
for s in [1, 10, 20, 30, 50, ]:
cd['days_since_Stringency_{}'.format(s)] = np.clip(( cd.Date - cd[(cd.StringencyIndex > s)].Date.min() ).dt.days, 0, None)
 cds.append(cd.fillna(0)[['Country', 'Date'] + cd.columns.to_list() [col_count:]])
 contain_data = pd.concat(cds)
 contain_data.Date.max()
contain_data.columns
contain_data[contain_data.Country == 'Australia']
contain_data.shape
contain_data.Country.replace({ 'United States': "US",
'South Korea': "Korea, South",
'Taiwan': "Taiwan*",
'Myanmar': "Burma", 'Slovak Republic': "Slovakia",
'Czech Republic': 'Czechia',
}, inplace=True)
 set(contain_data.Country)- set(test.Country_Region)
 sup_data = pd.read_excel(path + '/outside_data' +
'/Data Join - Copy1.xlsx')
 sup_data.columns = [c.replace(' ', '_')for c in sup_data.columns.to_list() ]
sup_data.drop(columns = [c for c in sup_data.columns.to_list() if 'Unnamed:' in c], inplace=True)
 sup_data.drop(columns = [ 'Date', 'ConfirmedCases',
'Fatalities', 'log-cases', 'log-fatalities', 'continent'], inplace=True)
 sup_data['Migrants_in'] = np.clip(sup_data.Migrants, 0, None)
 sup_data['Migrants_out'] = -np.clip(sup_data.Migrants, None, 0)
 sup_data.drop(columns = 'Migrants', inplace=True)
 sup_data.head()
sup_data.shape
train.Date = pd.to_datetime(train.Date)
 test.Date = pd.to_datetime(test.Date)
 train.rename(columns={'Country_Region': 'Country'}, inplace=True)
 test.rename(columns={'Country_Region': 'Country'}, inplace=True)
 sup_data.rename(columns={'Country_Region': 'Country'}, inplace=True)
 train['Place'] = train.Country + train.Province_State.fillna("")
 test['Place'] = test.Country + test.Province_State.fillna("")
 sup_data['Place'] = sup_data.Country + sup_data.Province_State.fillna("")
 len(train.Place.unique())
 sup_data = sup_data[
sup_data.columns.to_list() [2:]]
sup_data = sup_data.replace('N.A.', np.nan ).fillna(-0.5)
 for c in sup_data.columns[:-1]:
m = sup_data[c].max()
if m > 300 and c!='TRUE_POPULATION':
print(c)
 sup_data[c] = np.log(sup_data[c] + 1)
 assert sup_data[c].min() > -1
for c in sup_data.columns[:-1]:
m = sup_data[c].max()
if m > 300:
print(c)
 DEATHS = 'Fatalities'
len(train.Place.unique())
 train.ConfirmedCases = np.where(
(train.ConfirmedCases.shift(1)> train.ConfirmedCases)& 
(train.ConfirmedCases.shift(1)> 0)&(train.ConfirmedCases.shift(-1)> 0)& 
(train.Place == train.Place.shift(1)) &(train.Place == train.Place.shift(-1)) &
~train.ConfirmedCases.shift(-1 ).isnull() ,
np.sqrt(train.ConfirmedCases.shift(1)* train.ConfirmedCases.shift(-1)) ,
train.ConfirmedCases)
 train.Fatalities = np.where(
(train.Fatalities.shift(1)> train.Fatalities)& 
(train.Fatalities.shift(1)> 0)&(train.Fatalities.shift(-1)> 0)& 
(train.Place == train.Place.shift(1)) &(train.Place == train.Place.shift(-1)) &
~train.Fatalities.shift(-1 ).isnull() ,
np.sqrt(train.Fatalities.shift(1)* train.Fatalities.shift(-1)) ,
train.Fatalities)
 for i in [0, -1]:
train.ConfirmedCases = np.where(
(train.ConfirmedCases.shift(2+ i)> train.ConfirmedCases)& 
(train.ConfirmedCases.shift(2+ i)> 0)&(train.ConfirmedCases.shift(-1+ i)> 0)& 
(train.Place == train.Place.shift(2+ i)) &(train.Place == train.Place.shift(-1+ i)) &
~train.ConfirmedCases.shift(-1+ i ).isnull() ,
np.sqrt(train.ConfirmedCases.shift(2+ i)* train.ConfirmedCases.shift(-1+ i)) ,
train.ConfirmedCases)
 train[train.Place=='USVirgin Islands'][-10:]
train[(train.ConfirmedCases.shift(2)> 2* train.ConfirmedCases)& 
(train.Place == train.Place.shift(2)) &(train.ConfirmedCases < 100000)]
train[(train.Fatalities.shift(1)> train.Fatalities)& 
(train.Place == train.Place.shift(1)) &(train.Fatalities < 10000)]
train_bk = train.copy()
full_train = train.copy()
train_c = train[train.Country == 'China']
train_nc = train[train.Country != 'China']
train_us = train[train.Country == 'US']
def lplot(data, minDate = datetime.datetime(2000, 1, 1),
columns = ['ConfirmedCases', 'Fatalities']):
return
REAL = datetime.datetime(2020, 2, 10)
 dataset = train.copy()
if TRUNCATED:
dataset = dataset[dataset.Country.isin(
 ['Italy', 'Spain', 'Germany', 'Portugal', 'Belgium', 'Austria', 'Switzerland' ])]
dataset.head()
def rollDates(df, i, preserve=False):
df = df.copy()
if preserve:
df['Date_i'] = df.Date
df.Date = df.Date + datetime.timedelta(i)
 return df
WINDOWS = [1, 2, 4, 7, 12, 20, 30]
for window in WINDOWS:
csuffix = '_{}d_prior_value'.format(window)
 base = rollDates(dataset, window)
 dataset = pd.merge(dataset, base[['Date', 'Place',
'ConfirmedCases', 'Fatalities']], on = ['Date', 'Place'],
suffixes =('', csuffix), how='left')
 for c in ['ConfirmedCases', 'Fatalities']:
dataset[c+ csuffix].fillna(0, inplace=True)
 dataset[c+ csuffix] = np.log(dataset[c + csuffix] + 1)
 dataset[c+ '_{}d_prior_slope'.format(window)] =(np.log(dataset[c] + 1)- dataset[c+ csuffix])/ window
dataset[c+ '_{}d_ago_zero'.format(window)] = 1.0*(dataset[c+ csuffix] == 0)
 for window1 in WINDOWS:
for window2 in WINDOWS:
for c in ['ConfirmedCases', 'Fatalities']:
if window1 * 1.3 < window2 and window1 * 5 > window2:
dataset[ c +'_{}d_{}d_prior_slope_chg'.format(window1, window2)] = dataset[c+ '_{}d_prior_slope'.format(window1)] - dataset[c+ '_{}d_prior_slope'.format(window2)]
first_case = dataset[dataset.ConfirmedCases >= 1].groupby('Place' ).min()
tenth_case = dataset[dataset.ConfirmedCases >= 10].groupby('Place' ).min()
hundredth_case = dataset[dataset.ConfirmedCases >= 100].groupby('Place' ).min()
thousandth_case = dataset[dataset.ConfirmedCases >= 1000].groupby('Place' ).min()
first_fatality = dataset[dataset.Fatalities >= 1].groupby('Place' ).min()
tenth_fatality = dataset[dataset.Fatalities >= 10].groupby('Place' ).min()
hundredth_fatality = dataset[dataset.Fatalities >= 100].groupby('Place' ).min()
thousandth_fatality = dataset[dataset.Fatalities >= 1000].groupby('Place' ).min()
dataset['days_since_first_case'] = np.clip(( dataset.Date - first_case.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_tenth_case'] = np.clip(( dataset.Date - tenth_case.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_hundredth_case'] = np.clip(( dataset.Date - hundredth_case.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_thousandth_case'] = np.clip(( dataset.Date - thousandth_case.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_first_fatality'] = np.clip(( dataset.Date - first_fatality.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_tenth_fatality'] = np.clip(( dataset.Date - tenth_fatality.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_hundredth_fatality'] = np.clip(( dataset.Date - hundredth_fatality.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['days_since_thousandth_fatality'] = np.clip(( dataset.Date - thousandth_fatality.loc[dataset.Place].Date.values ).dt.days.fillna(-1), -1, None)
 dataset['case_rate_since_first_case'] = np.clip(( np.log(dataset.ConfirmedCases + 1)- np.log(first_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['case_rate_since_tenth_case'] = np.clip(( np.log(dataset.ConfirmedCases + 1)- np.log(tenth_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) /(dataset.days_since_tenth_case+0.01), 0, 1)
 dataset['case_rate_since_hundredth_case'] = np.clip(( np.log(dataset.ConfirmedCases + 1)- np.log(hundredth_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['case_rate_since_thousandth_case'] = np.clip(( np.log(dataset.ConfirmedCases + 1)- np.log(thousandth_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_first_case'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(first_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_tenth_case'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(tenth_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_hundredth_case'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(hundredth_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_thousandth_case'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(thousandth_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_first_fatality'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(first_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_first_fatality+0.01), 0, 1)
 dataset['fatality_rate_since_tenth_fatality'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(tenth_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_tenth_fatality+0.01), 0, 1)
 dataset['fatality_rate_since_hundredth_fatality'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(hundredth_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_hundredth_fatality+0.01), 0, 1)
 dataset['fatality_rate_since_thousandth_fatality'] = np.clip(( np.log(dataset.Fatalities + 1)- np.log(thousandth_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) /(dataset.days_since_thousandth_fatality+0.01), 0, 1)
 dataset['first_case_ConfirmedCases'] = np.log(first_case.loc[dataset.Place].ConfirmedCases.values + 1)
 dataset['first_case_Fatalities'] = np.log(first_case.loc[dataset.Place].Fatalities.values + 1)
 dataset['first_fatality_ConfirmedCases'] = np.log(first_fatality.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)*(dataset.days_since_first_fatality >= 0)
 dataset['first_fatality_Fatalities'] = np.log(first_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)*(dataset.days_since_first_fatality >= 0)
 dataset['first_fatality_cfr'] = np.where(dataset.days_since_first_fatality < 0,
-8, 
(dataset.first_fatality_Fatalities)- 
(dataset.first_fatality_ConfirmedCases))
dataset['first_fatality_lag_vs_first_case'] = np.where(dataset.days_since_first_fatality >= 0,
dataset.days_since_first_case - dataset.days_since_first_fatality , -1)
 dataset['case_chg'] = np.clip(np.log(dataset.ConfirmedCases + 1)- np.log(dataset.ConfirmedCases.shift(1)+1), 0, None ).fillna(0)
 dataset['case_chg_ema_3d'] = dataset.case_chg.ewm(span = 3 ).mean() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1)
 dataset['case_chg_ema_10d'] = dataset.case_chg.ewm(span = 10 ).mean() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1)
 dataset['case_chg_stdev_5d'] = dataset.case_chg.rolling(5 ).std() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/5, 0, 1)
 dataset['case_chg_stdev_15d'] = dataset.case_chg.rolling(15 ).std() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/15, 0, 1)
 dataset['case_update_pct_3d_ewm'] =(dataset.case_chg > 0 ).ewm(span = 3 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1), 2)
 dataset['case_update_pct_10d_ewm'] =(dataset.case_chg > 0 ).ewm(span = 10 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1), 2)
 dataset['case_update_pct_30d_ewm'] =(dataset.case_chg > 0 ).ewm(span = 30 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/30, 0, 1), 2)
 dataset['fatality_chg'] = np.clip(np.log(dataset.Fatalities + 1)- np.log(dataset.Fatalities.shift(1)+1), 0, None ).fillna(0)
 dataset['fatality_chg_ema_3d'] = dataset.fatality_chg.ewm(span = 3 ).mean() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/33, 0, 1)
 dataset['fatality_chg_ema_10d'] = dataset.fatality_chg.ewm(span = 10 ).mean() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1)
 dataset['fatality_chg_stdev_5d'] = dataset.fatality_chg.rolling(5 ).std() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/5, 0, 1)
 dataset['fatality_chg_stdev_15d'] = dataset.fatality_chg.rolling(15 ).std() * np.clip(( dataset.Date - dataset.Date.min() ).dt.days/15, 0, 1)
 dataset['fatality_update_pct_3d_ewm'] =(dataset.fatality_chg > 0 ).ewm(span = 3 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1), 2)
 dataset['fatality_update_pct_10d_ewm'] =(dataset.fatality_chg > 0 ).ewm(span = 10 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1), 2)
 dataset['fatality_update_pct_30d_ewm'] =(dataset.fatality_chg > 0 ).ewm(span = 30 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/30, 0, 1), 2)
 dataset.tail()
contain_data.Date = contain_data.Date + datetime.timedelta(7)
 contain_data.Date.max()
assert set(dataset.Place.unique())== set(dataset.Place.unique())
 dataset = pd.merge(dataset, sup_data, on='Place', how='left', validate='m:1')
 dataset = pd.merge(dataset, contain_data, on = ['Country', 'Date'], how='left', validate='m:1')
 dataset['log_true_population'] = np.log(dataset.TRUE_POPULATION + 1)
 dataset['ConfirmedCases_percapita'] = np.log(dataset.ConfirmedCases + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['Fatalities_percapita'] = np.log(dataset.Fatalities + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['log_cfr'] = np.log(( dataset.Fatalities + np.clip(0.015 * dataset.ConfirmedCases, 0, 0.3)) /(dataset.ConfirmedCases + 0.1))
def cfr(case, fatality):
cfr_calc = np.log(( fatality + np.clip(0.015 * case, 0, 0.3)) /(case + 0.1))
return np.where(np.isnan(cfr_calc)| np.isinf(cfr_calc),
BLCFR, cfr_calc)
 BLCFR = np.median(dataset[dataset.ConfirmedCases==1].log_cfr[::10])
 dataset.log_cfr.fillna(BLCFR, inplace=True)
 dataset.log_cfr = np.where(dataset.log_cfr.isnull() | np.isinf(dataset.log_cfr),
BLCFR, dataset.log_cfr)
 BLCFR
dataset['log_cfr_3d_ewm'] = BLCFR +(dataset.log_cfr - BLCFR ).ewm(span = 3 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1), 2)
 dataset['log_cfr_8d_ewm'] = BLCFR +(dataset.log_cfr - BLCFR ).ewm(span = 8 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/8, 0, 1), 2)
 dataset['log_cfr_20d_ewm'] = BLCFR +(dataset.log_cfr - BLCFR ).ewm(span = 20 ).mean() * np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/20, 0, 1), 2)
 dataset['log_cfr_3d_20d_ewm_crossover'] = dataset.log_cfr_3d_ewm - dataset.log_cfr_20d_ewm
dataset.drop(columns = 'log_cfr', inplace=True)
 date_totals = dataset.groupby('Date' ).sum()
mean_7d_c_slope = dataset.groupby('Date')[['ConfirmedCases_7d_prior_slope']].apply(lambda x:
np.mean(x[x > 0])).ewm(span = 3 ).mean()
mean_7d_f_slope = dataset.groupby('Date')[['Fatalities_7d_prior_slope']].apply(lambda x:
np.mean(x[x > 0])).ewm(span = 7 ).mean()
dataset['ConfirmedCases_percapita_vs_world'] = np.log(dataset.ConfirmedCases + 1)- np.log(dataset.TRUE_POPULATION + 1)-(
 np.log(date_totals.loc[dataset.Date].ConfirmedCases + 1)
 -np.log(date_totals.loc[dataset.Date].TRUE_POPULATION + 1)
 ).values
dataset['Fatalities_percapita_vs_world'] = np.log(dataset.Fatalities + 1)- np.log(dataset.TRUE_POPULATION + 1)-(
 np.log(date_totals.loc[dataset.Date].Fatalities + 1)
 -np.log(date_totals.loc[dataset.Date].TRUE_POPULATION + 1)
 ).values
dataset['cfr_vs_world'] = dataset.log_cfr_3d_ewm - np.log(date_totals.loc[dataset.Date].Fatalities / date_totals.loc[dataset.Date].ConfirmedCases ).values
cont_date_totals = dataset.groupby(['Date', 'continent_generosity'] ).sum()
len(dataset)
 dataset['ConfirmedCases_percapita_vs_continent_mean'] = 0
dataset['Fatalities_percapita_vs_continent_mean'] = 0
dataset['ConfirmedCases_percapita_vs_continent_median'] = 0
dataset['Fatalities_percapita_vs_continent_median'] = 0
for cg in dataset.continent_generosity.unique() :
ps = dataset.groupby("Place" ).last()
tp = ps[ps.continent_generosity==cg].TRUE_POPULATION.sum()
print(tp / 1e9)
 for Date in dataset.Date.unique() :
cd = dataset[(dataset.Date == Date)& 
(dataset.continent_generosity == cg)]\
[['ConfirmedCases', 'Fatalities', 'TRUE_POPULATION']]
cmedian = np.median(np.log(cd.ConfirmedCases + 1)- np.log(cd.TRUE_POPULATION+1))
cmean = np.log(cd.ConfirmedCases.sum() + 1)- np.log(tp + 1)
 fmedian = np.median(np.log(cd.Fatalities + 1)- np.log(cd.TRUE_POPULATION+1))
fmean = np.log(cd.Fatalities.sum() + 1)- np.log(tp + 1)
 cfrmean = cfr(cd.ConfirmedCases.sum() , cd.Fatalities.sum())
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'ConfirmedCases_percapita_vs_continent_mean'] = \
dataset['ConfirmedCases_percapita'] \
-(cmean)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'ConfirmedCases_percapita_vs_continent_median'] = \
dataset['ConfirmedCases_percapita'] \
-(cmedian)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'Fatalities_percapita_vs_continent_mean'] = \
dataset['Fatalities_percapita']\
-(fmean)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'Fatalities_percapita_vs_continent_median'] = \
dataset['Fatalities_percapita']\
-(fmedian)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'cfr_vs_continent'] = \
dataset.log_cfr_3d_ewm \
- cfrmean
all_places = dataset[['Place', 'latitude', 'longitude']].drop_duplicates().set_index('Place',
drop=True)
 all_places.head()
def surroundingPlaces(place, d = 10):
dist =(all_places.latitude - all_places.loc[place].latitude)**2 +(all_places.longitude - all_places.loc[place].longitude)** 2
return all_places[dist < d**2][1:n+1]
def nearestPlaces(place, n = 10):
dist =(all_places.latitude - all_places.loc[place].latitude)**2 +(all_places.longitude - all_places.loc[place].longitude)** 2
ranked = np.argsort(dist)
 return all_places.iloc[ranked][1:n+1]
dgp = dataset.groupby('Place' ).last()
for n in [5, 10, 20]:
for place in dataset.Place.unique() :
nps = nearestPlaces(place, n)
 tp = dgp.loc[nps.index].TRUE_POPULATION.sum()
dataset.loc[dataset.Place==place,
'ratio_population_vs_nearest{}'.format(n)] = \
np.log(dataset.loc[dataset.Place==place].TRUE_POPULATION.mean() + 1)\
- np.log(tp+1)
 nbps = dataset[(dataset.Place.isin(nps.index)) ].groupby('Date')[['ConfirmedCases', 'Fatalities']].sum()
nppc =(np.log(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).ConfirmedCases + 1)- np.log(tp + 1))
nppf =(np.log(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).Fatalities + 1)- np.log(tp + 1))
npp_cfr = cfr(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).ConfirmedCases,
nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).Fatalities)
 dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_percapita_vs_nearest{}'.format(n)] = \
dataset[(dataset.Place == place)].ConfirmedCases_percapita \
- nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_percapita_vs_nearest{}'.format(n)] = \
dataset[(dataset.Place == place)].Fatalities_percapita \
- nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_vs_nearest{}'.format(n)] = \
dataset[(dataset.Place == place)].log_cfr_3d_ewm \
- npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_nearest{}_percapita'.format(n)] = nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_nearest{}_percapita'.format(n)] = nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_nearest{}'.format(n)] = npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_nearest{}_10d_slope'.format(n)] = \ 
(nppc.ewm(span = 1 ).mean() - nppc.ewm(span = 10 ).mean() ).values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_nearest{}_10d_slope'.format(n)] = \ 
(nppf.ewm(span = 1 ).mean() - nppf.ewm(span = 10 ).mean() ).values
npp_cfr_s = pd.Series(npp_cfr)
 dataset.loc[ 
(dataset.Place == place),
'cfr_nearest{}_10d_slope'.format(n)] = \ 
(npp_cfr_s.ewm(span = 1 ).mean() \
- npp_cfr_s.ewm(span = 10 ).mean() ).values
dgp = dataset.groupby('Place' ).last()
for d in [5, 10, 20]:
for place in dataset.Place.unique() :
nps = surroundingPlaces(place, d)
 dataset.loc[dataset.Place==place, 'num_surrounding_places_{}_degrees'.format(d)] = len(nps)
 tp = dgp.loc[nps.index].TRUE_POPULATION.sum()
dataset.loc[dataset.Place==place,
'ratio_population_vs_surrounding_places_{}_degrees'.format(d)] = \
np.log(dataset.loc[dataset.Place==place].TRUE_POPULATION.mean() + 1)\
- np.log(tp+1)
 if len(nps)==0:
continue;
nbps = dataset[(dataset.Place.isin(nps.index)) ].groupby('Date')[['ConfirmedCases', 'Fatalities']].sum()
nppc =(np.log(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).ConfirmedCases + 1)- np.log(tp + 1))
nppf =(np.log(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).Fatalities + 1)- np.log(tp + 1))
npp_cfr = cfr(nbps.loc[dataset[dataset.Place==place].Date].fillna(0 ).ConfirmedCases,
nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).Fatalities)
 dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_percapita_vs_surrounding_places_{}_degrees'.format(d)] = \
dataset[(dataset.Place == place)].ConfirmedCases_percapita \
- nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_percapita_vs_surrounding_places_{}_degrees'.format(d)] = \
dataset[(dataset.Place == place)].Fatalities_percapita \
- nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_vs_surrounding_places_{}_degrees'.format(d)] = \
dataset[(dataset.Place == place)].log_cfr_3d_ewm \
- npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_surrounding_places_{}_degrees_percapita'.format(d)] = nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_surrounding_places_{}_degrees_percapita'.format(d)] = nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_surrounding_places_{}_degrees'.format(d)] = npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_surrounding_places_{}_degrees_10d_slope'.format(d)] = \ 
(nppc.ewm(span = 1 ).mean() - nppc.ewm(span = 10 ).mean() ).values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_surrounding_places_{}_degrees_10d_slope'.format(d)] = \ 
(nppf.ewm(span = 1 ).mean() - nppf.ewm(span = 10 ).mean() ).values
npp_cfr_s = pd.Series(npp_cfr)
 dataset.loc[ 
(dataset.Place == place),
'cfr_surrounding_places_{}_degrees_10d_slope'.format(d)] = \ 
(npp_cfr_s.ewm(span = 1 ).mean() \
- npp_cfr_s.ewm(span = 10 ).mean() ).values
for col in [c for c in dataset.columns if 'surrounding_places' in c and 'num_sur' not in c]:
dataset[col] = dataset[col].fillna(0)
 n_col = 'num_surrounding_places_{}_degrees'.format(col.split('degrees')[0].split('_')[-2])
 print(col)
 dataset[col + "_times_num_places"] = dataset[col] * np.sqrt(dataset[n_col])
 dataset[dataset.Country=='US'][['Place', 'Date'] + [c for c in dataset.columns if 'ratio_p' in c]] [::50]
dataset.TRUE_POPULATION
dataset.TRUE_POPULATION.sum()
dataset.groupby('Date' ).sum().TRUE_POPULATION
dataset['first_case_ConfirmedCases_percapita'] = np.log(dataset.first_case_ConfirmedCases + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['first_case_Fatalities_percapita'] = np.log(dataset.first_case_Fatalities + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['first_fatality_Fatalities_percapita'] = np.log(dataset.first_fatality_Fatalities + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['first_fatality_ConfirmedCases_percapita'] = np.log(dataset.first_fatality_ConfirmedCases + 1)- np.log(dataset.TRUE_POPULATION + 1)
 dataset['days_to_saturation_ConfirmedCases_4d'] =(- np.log(dataset.ConfirmedCases + 1)+ np.log(dataset.TRUE_POPULATION + 1)) / dataset.ConfirmedCases_4d_prior_slope
dataset['days_to_saturation_ConfirmedCases_7d'] =(- np.log(dataset.ConfirmedCases + 1)+ np.log(dataset.TRUE_POPULATION + 1)) / dataset.ConfirmedCases_7d_prior_slope
dataset['days_to_saturation_Fatalities_20d_cases'] =(- np.log(dataset.Fatalities + 1)+ np.log(dataset.TRUE_POPULATION + 1)) / dataset.ConfirmedCases_20d_prior_slope
dataset['days_to_saturation_Fatalities_12d_cases'] =(- np.log(dataset.Fatalities + 1)+ np.log(dataset.TRUE_POPULATION + 1)) / dataset.ConfirmedCases_12d_prior_slope
dataset['days_to_3pct_ConfirmedCases_4d'] =(- np.log(dataset.ConfirmedCases + 1)+ np.log(dataset.TRUE_POPULATION + 1)- 3.5)/ dataset.ConfirmedCases_4d_prior_slope
dataset['days_to_3pct_ConfirmedCases_7d'] =(- np.log(dataset.ConfirmedCases + 1)+ np.log(dataset.TRUE_POPULATION + 1)- 3.5)/ dataset.ConfirmedCases_7d_prior_slope
dataset['days_to_0.3pct_Fatalities_20d_cases'] =(- np.log(dataset.Fatalities + 1)+ np.log(dataset.TRUE_POPULATION + 1)- 5.8)/ dataset.ConfirmedCases_20d_prior_slope
dataset['days_to_0.3pct_Fatalities_12d_cases'] =(- np.log(dataset.Fatalities + 1)+ np.log(dataset.TRUE_POPULATION + 1)- 5.8)/ dataset.ConfirmedCases_12d_prior_slope
dataset.tail()
dataset = dataset[dataset.ConfirmedCases > 0]
len(dataset)
 datas = []
for window in range(1, 35):
base = rollDates(dataset, window, True)
 datas.append(pd.merge(dataset[['Date', 'Place',
'ConfirmedCases', 'Fatalities']], base, on = ['Date', 'Place'],
how = 'right',
suffixes =('_f', '')))
 data = pd.concat(datas, axis =0 ).astype(np.float32, errors ='ignore')
 len(data)
 data['Date_f'] = data.Date
data.Date = data.Date_i
data['elapsed'] =(data.Date_f - data.Date_i ).dt.days
data['CaseChgRate'] =(np.log(data.ConfirmedCases_f + 1)- np.log(data.ConfirmedCases + 1)) / data.elapsed;
data['FatalityChgRate'] =(np.log(data.Fatalities_f + 1)- np.log(data.Fatalities + 1)) / data.elapsed;
data.elapsed
falloff_hash = {}
def true_agg(rate_i, elapsed, bend_rate):
elapsed = int(elapsed)
 if(bend_rate, elapsed)not in falloff_hash:
falloff_hash[(bend_rate, elapsed)] = np.sum([ np.power(bend_rate, e)for e in range(1, elapsed+1)])
 return falloff_hash[(bend_rate, elapsed)] * rate_i
true_agg(0.3, 30, 0.9)
 slope_cols = [c for c in data.columns if
any(z in c for z in ['prior_slope', 'chg', 'rate'])
 and not any(z in c for z in ['bend', 'prior_slope_chg', 'Country', 'ewm',
])]
print(slope_cols)
 bend_rates = [1, 0.95, 0.90]
for bend_rate in bend_rates:
bend_agg = data[['elapsed']].apply(lambda x: true_agg(1, *x, bend_rate), axis=1)
 for sc in slope_cols:
if bend_rate < 1:
data[sc+"_slope_bend_{}".format(bend_rate)] = data[sc] * np.power(( bend_rate + 1)/2, data.elapsed)
 data[sc+"_true_slope_bend_{}".format(bend_rate)] = bend_agg * data[sc] / data.elapsed
data[sc+"_agg_bend_{}".format(bend_rate)] = data[sc] * data.elapsed * np.power(( bend_rate + 1)/2, data.elapsed)
 data[sc+"_true_agg_bend_{}".format(bend_rate)] = bend_agg * data[sc]
slope_cols[:5]
for col in [c for c in data.columns if any(z in c for z in
['vs_continent', 'nearest', 'vs_world', 'surrounding_places'])]:
data[col + '_times_days'] = data[col] * data.elapsed
data['saturation_slope_ConfirmedCases'] =(- np.log(data.ConfirmedCases + 1)+ np.log(data.TRUE_POPULATION + 1)) / data.elapsed
data['saturation_slope_Fatalities'] =(- np.log(data.Fatalities + 1)+ np.log(data.TRUE_POPULATION + 1)) / data.elapsed
data['dist_to_ConfirmedCases_saturation_times_days'] =(- np.log(data.ConfirmedCases + 1)+ np.log(data.TRUE_POPULATION + 1)) * data.elapsed
data['dist_to_Fatalities_saturation_times_days'] =(- np.log(data.Fatalities + 1)+ np.log(data.TRUE_POPULATION + 1)) * data.elapsed
data['slope_to_1pct_ConfirmedCases'] =(- np.log(data.ConfirmedCases + 1)+ np.log(data.TRUE_POPULATION + 1)- 4.6)/ data.elapsed
data['slope_to_0.1pct_Fatalities'] =(- np.log(data.Fatalities + 1)+ np.log(data.TRUE_POPULATION + 1)- 6.9)/ data.elapsed
data['dist_to_1pct_ConfirmedCases_times_days'] =(- np.log(data.ConfirmedCases + 1)+ np.log(data.TRUE_POPULATION + 1)- 4.6)* data.elapsed
data['dist_to_0.1pct_Fatalities_times_days'] =(- np.log(data.Fatalities + 1)+ np.log(data.TRUE_POPULATION + 1)- 6.9)* data.elapsed
data['trendline_per_capita_ConfirmedCases_4d_slope'] =(np.log(data.ConfirmedCases + 1)- np.log(data.TRUE_POPULATION + 1)) +(data.ConfirmedCases_4d_prior_slope * data.elapsed)
 data['trendline_per_capita_ConfirmedCases_7d_slope'] =(np.log(data.ConfirmedCases + 1)- np.log(data.TRUE_POPULATION + 1)) +(data.ConfirmedCases_7d_prior_slope * data.elapsed)
 data['trendline_per_capita_Fatalities_12d_slope'] =(np.log(data.Fatalities + 1)- np.log(data.TRUE_POPULATION + 1)) +(data.ConfirmedCases_12d_prior_slope * data.elapsed)
 data['trendline_per_capita_Fatalities_20d_slope'] =(np.log(data.Fatalities + 1)- np.log(data.TRUE_POPULATION + 1)) +(data.ConfirmedCases_20d_prior_slope * data.elapsed)
 len(data)
 data.groupby('Place' ).last()
def logHist(x, b = 150):
return
data['log_fatalities'] = np.log(data.Fatalities + 1)
 data['log_cases'] = np.log(data.ConfirmedCases + 1)
 data['is_China'] =(data.Country=='China')&(~data.Place.isin(['Hong Kong', 'Macau']))
for col in [c for c in data.columns if 'd_ewm' in c]:
data[col] += np.random.normal(0, 1, len(data)) * np.std(data[col])* 0.2
data['is_province'] = 1.0*(~data.Province_State.isnull())
 data['log_elapsed'] = np.log(data.elapsed + 1)
 data.columns
data.columns[::19]
data.shape
logHist(data.ConfirmedCases)
 data.drop(columns = ['TRUE_POPULATION'], inplace=True)
 data['final_day_of_week'] = data.Date_f.apply(datetime.datetime.weekday)
 data['base_date_day_of_week'] = data.Date.apply(datetime.datetime.weekday)
 data['date_difference_modulo_7_days'] =(data.Date_f - data.Date ).dt.days % 7
for c in data.columns.to_list() :
if 'days_to' in c:
data[c] = data[c].where(~np.isinf(data[c]), 1e3)
 data[c] = np.clip(data[c], 0, 365)
 data[c] = np.sqrt(data[c])
 new_places = train[(train.Date == test.Date.min() - datetime.timedelta(1)) & 
(train.ConfirmedCases == 0)
 ].Place
model_data = data[(( len(test)==0)|(data.Date_f < test.Date.min()))
& 
(data.ConfirmedCases > 0)& 
(~data.ConfirmedCases_f.isnull())].copy()
test.Date.min()
model_data.Date_f.max()
model_data.Date_f.max()
model_data.Date.max()
model_data.Date_f.min()
model_data = model_data[~(
(np.random.rand(len(model_data)) < 0.8)& 
(model_data.Country == 'China')& 
(model_data.Date < datetime.datetime(2020, 2, 15)))]
x_dates = model_data[['Date_i', 'Date_f', 'Place']]
x = model_data[
model_data.columns.to_list() [
model_data.columns.to_list().index('ConfirmedCases_1d_prior_value'):]]\ 
.drop(columns = ['Date_i', 'Date_f', 'CaseChgRate', 'FatalityChgRate'])
 test.Date
if PRIVATE:
data_test = data[(data.Date_i == train.Date.max())& 
(data.Date_f.isin(test.Date.unique())) ].copy()
else:
data_test = data[(data.Date_i == test.Date.min() - datetime.timedelta(1)) & 
(data.Date_f.isin(test.Date.unique())) ].copy()
data_test.Date.unique()
test.Date.unique()
x_test = data_test[x.columns].copy()
train.Date.max()
test.Date.max()
if MODEL_Y is 'slope':
y_cases = model_data.CaseChgRate
y_fatalities = model_data.FatalityChgRate
else:
y_cases = model_data.CaseChgRate * model_data.elapsed
y_fatalities = model_data.FatalityChgRate * model_data.elapsed
y_cfr = np.log(( model_data.Fatalities_f + np.clip(0.015 * model_data.ConfirmedCases_f, 0, 0.3)) /(model_data.ConfirmedCases_f + 0.1))
groups = model_data.Country
places = model_data.Place
np.random.seed(SEED)
 enet_params = { 'alpha': [ 3e-6, 1e-5, 3e-5, 1e-4, 3e-4, 1e-3, ],
'l1_ratio': [ 0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 0.8, 0.9, 0.97, 0.99 ]}
et_params = { 'n_estimators': [50, 70, 100, 140],
'max_depth': [3, 5, 7, 8, 9, 10],
'min_samples_leaf': [30, 50, 70, 100, 130, 165, 200, 300, 600],
'max_features': [0.4, 0.5, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85],
'min_impurity_decrease': [0, 1e-5 ],
'bootstrap': [ True, False],
}
lgb_params = {
'max_depth': [5, 12],
'n_estimators': [ 100, 200, 300, 500],
'min_split_gain': [0, 0, 1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2],
'min_child_samples': [ 7, 10, 14, 20, 30, 40, 70, 100, 200, 400, 700, 1000, 2000],
'min_child_weight': [0],
'num_leaves': [5, 10, 20, 30],
'learning_rate': [0.05, 0.07, 0.1],
'colsample_bytree': [0.1, 0.2, 0.33, 0.5, 0.65, 0.8, 0.9],
'colsample_bynode':[0.1, 0.2, 0.33, 0.5, 0.65, 0.81],
'reg_lambda': [1e-5, 3e-5, 1e-4, 1e-3, 1e-2, 0.1, 1, 10, 100, 1000, ],
'reg_alpha': [1e-5, 1e-4, 1e-3, 1e-2, 0.1, 1, 30, 1000,],
'subsample': [ 0.8, 0.9, 1],
'subsample_freq': [1],
'max_bin': [ 7, 15, 31, 63, 127, 255],
}
MSE = 'neg_mean_squared_error'
MAE = 'neg_mean_absolute_error'
def trainENet(x, y, groups, cv = 0, **kwargs):
return trainModel(x, y, groups,
clf = ElasticNet(normalize = True, selection = 'random',
max_iter = 3000),
params = enet_params,
cv = cv, **kwargs)
 def trainETR(x, y, groups, cv = 0, n_jobs = 5, **kwargs):
clf = ExtraTreesRegressor(n_jobs = 1)
 params = et_params
return trainModel(x, y, groups, clf, params, cv, n_jobs, **kwargs)
 def trainLGB(x, y, groups, cv = 0, n_jobs = 4, **kwargs):
clf = lgb.LGBMRegressor(verbosity=-1, hist_pool_size = 1000, 
)
 params = lgb_params
return trainModel(x, y, groups, clf, params, cv, n_jobs, **kwargs)
 def trainModel(x, y, groups, clf, params, cv = 0, n_jobs = None,
verbose=0, splits=None, **kwargs):
if n_jobs is None:
n_jobs = 4
if np.random.rand() < 0.8:
folds = GroupShuffleSplit(n_splits=4,
test_size= 0.2 + 0.10 * np.random.rand())
 else:
folds = GroupKFold(4)
 clf = RandomizedSearchCV(clf, params,
cv= folds,
n_iter=12,
verbose = 0, n_jobs = n_jobs, scoring = MSE)
 f = clf.fit(x, y, groups)
 print(pd.DataFrame(clf.cv_results_['mean_test_score'])) ; print() ;
best = clf.best_estimator_; print(best)
 print("Best Score: {}".format(np.round(clf.best_score_,4)))
 return best
np.mean(y_cases)
 def getSparseColumns(x, verbose = 0):
sc = []
for c in x.columns.to_list() :
u = len(x[c].unique())
 if u > 10 and u < 0.01*len(x):
sc.append(c)
 if verbose > 0:
print("{}: {}".format(c, u))
return sc
def noisify(x, noise = 0.1):
x = x.copy()
cols = getSparseColumns(x)
 for c in cols:
u = len(x[c].unique())
 if u > 50:
x[c].values[:] = x[c].values + np.random.normal(0, noise, len(x)) * np.std(x[c])
 return x;
def getMaxOverlap(row, df):
df_place = df[df.Place == row.Place]
if len(df_place)==0:
return 0
overlap =(np.clip(df_place.Date_f, None, row.Date_f)- np.clip(df_place.Date_i, row.Date_i, None)).dt.days
overlap = np.clip(overlap, 0, None)
 length = np.clip(( df_place.Date_f - df_place.Date_i ).dt.days, 
(row.Date_f - row.Date_i ).days, None)
 return np.amax(overlap / length)
 def getSampleWeight(x, groups):
counter = Counter(groups)
 median_count = np.median([counter[group] for group in groups.unique() ])
 c_count = [counter[group] for group in groups]
e_decay = np.round(LT_DECAY_MIN + np.random.rand() *(LT_DECAY_MAX - LT_DECAY_MIN), 1)
 print("LT weight decay: {:.2f}".format(e_decay)) ;
ssr = np.power(1 / np.clip(c_count / median_count , 0.1, 30),
0.1 + np.random.rand() * 0.6)\
/ np.power(x.elapsed / 3, e_decay)\
* SET_FRAC * np.exp(- np.random.rand())
 group_drop = dict([(group, np.random.rand() < 0.15)for group in groups.unique() ])
 ssr = ssr *([ 1 -group_drop[group] for group in groups])
 return ssr;
def runBags(x, y, groups, cv, bags = 3, model_type = trainLGB,
noise = 0.1, splits = None, weights = None, **kwargs):
models = []
for bag in range(bags):
print("
BAG {}".format(bag+1))
x = x.copy()
if DROPS:
for col in [c for c in x.columns if any(z in c for z in ['bend', 'slope', 'prior'])]:
if np.random.rand() < np.sqrt(np.random.rand())* 0.7:
x[col].values[:] = 0
if DROPS and(np.random.rand() < 0.30):
print('dropping nearest features')
 for col in [c for c in x.columns if 'nearest' in c]:
x[col].values[:] = 0
if DROPS and(np.random.rand() < 0.25):
print('dropping 'surrounding places' features')
 for col in [c for c in x.columns if 'surrounding_places' in c]:
x[col].values[:] = 0
col_drop_frac = np.sqrt(np.random.rand())* 0.5
for col in [c for c in x.columns if 'elapsed' not in c ]:
if np.random.rand() < col_drop_frac:
x[col].values[:] = 0
x = noisify(x, noise)
 if DROPS and(np.random.rand() < SUP_DROP):
print("Dropping supplemental country data")
 for col in x[[c for c in x.columns if c in sup_data.columns]]:
x[col].values[:] = 0
if DROPS and(np.random.rand() < ACTIONS_DROP):
for col in x[[c for c in x.columns if c in contain_data.columns]]:
x[col].values[:] = 0
else:
print("*using containment data")
 if np.random.rand() < 0.6:
x.S_data_days = 0
ssr = getSampleWeight(x, groups)
 date_falloff = 0 +(1/30)* np.random.rand()
if weights is not None:
ssr = ssr * np.exp(-weights * date_falloff)
 ss =(np.random.rand(len(y)) < ssr)
 print("n={}".format(len(x[ss])))
 p1 =x.elapsed[ss].plot(kind='hist', bins = int(x.elapsed.max() - x.elapsed.min() + 1))
p1 = plt.figure() ;
print(( ss ).sum())
 models.append(model_type(x[ss], y[ss], groups[ss], cv, **kwargs))
return models
x = x.astype(np.float32)
 BAG_MULT = 1
x.shape
lgb_c_clfs = []; lgb_c_noise = []
date_weights = np.abs(( model_data.Date_f - test.Date.min() ).dt.days)
 for iteration in range(0, int(math.ceil(1.1 * BAGS))):
for noise in [ 0.05, 0.1, 0.2, 0.3, 0.4 ]:
print("
---
Noise of {}".format(noise)) ;
num_bags = 1 * BAG_MULT;
if np.random.rand() < PLACE_FRACTION:
cv_group = places
print("CV by Place")
 else:
cv_group = groups
print("CV by Country")
 lgb_c_clfs.extend(runBags(x, y_cases,
cv_group,
MSE, num_bags, trainLGB, verbose = 0,
noise = noise, weights = date_weights 
))
lgb_c_noise.extend([noise] * num_bags)
 if SINGLE_MODEL:
break;
lgb_f_clfs = []; lgb_f_noise = []
for iteration in range(0, int(np.ceil(np.sqrt(BAGS)))) :
for noise in [ 0.5, 1, 2, 3, ]:
print("
---
Noise of {}".format(noise)) ;
num_bags = 1 * int(np.ceil(np.sqrt(BAG_MULT)))
 if np.random.rand() < PLACE_FRACTION :
cv_group = places
print("CV by Place")
 else:
cv_group = groups
print("CV by Country")
 lgb_f_clfs.extend(runBags(x, y_fatalities,
cv_group,
MSE, num_bags, trainLGB,
verbose = 0, noise = noise,
weights = date_weights 
))
lgb_f_noise.extend([noise] * num_bags)
 if SINGLE_MODEL:
break;
lgb_cfr_clfs = []; lgb_cfr_noise = [];
for iteration in range(0, int(np.ceil(np.sqrt(BAGS)))) :
for noise in [ 0.4, 1, 2, 3]:
print("
---
Noise of {}".format(noise)) ;
num_bags = 1 * BAG_MULT;
if np.random.rand() < 0.5 * PLACE_FRACTION :
cv_group = places
print("CV by Place")
 else:
cv_group = groups
print("CV by Country")
 lgb_cfr_clfs.extend(runBags(x, y_cfr,
cv_group,
MSE, num_bags, trainLGB, verbose = 0,
noise = noise,
weights = date_weights 
))
lgb_cfr_noise.extend([noise] * num_bags)
 if SINGLE_MODEL:
break;
lgb_cfr_clfs[0].predict(x_test)
 def show_FI(model, featNames, featCount):
fis = model.feature_importances_
fig, ax = plt.subplots(figsize=(6, 5))
indices = np.argsort(fis)[::-1][:featCount]
g = sns.barplot(y=featNames[indices][:featCount],
x = fis[indices][:featCount] , orient='h')
 g.set_xlabel("Relative importance")
 g.set_ylabel("Features")
 g.tick_params(labelsize=12)
 g.set_title(" feature importance")
 def avg_FI(all_clfs, featNames, featCount):
clfs = []
for clf_set in all_clfs:
for clf in clf_set:
clfs.append(clf);
print("{} classifiers".format(len(clfs)))
 fi = np.zeros(( len(clfs), len(clfs[0].feature_importances_)))
 for idx, clf in enumerate(clfs):
fi[idx, :] = clf.feature_importances_
avg_fi = np.mean(fi, axis = 0)
 fis = avg_fi
fig, ax = plt.subplots(figsize=(6, 5))
indices = np.argsort(fis)[::-1]
g = sns.barplot(y=featNames[indices][:featCount],
x = fis[indices][:featCount] , orient='h')
 g.set_xlabel("Relative importance")
 g.set_ylabel("Features")
 g.tick_params(labelsize=12)
 g.set_title(" feature importance")
 return pd.Series(fis[indices], featNames[indices])
 def linear_FI_plot(fi, featNames, featCount):
fig, ax = plt.subplots(figsize=(6, 5))
indices = np.argsort(np.absolute(fi)) [::-1]
g = sns.barplot(y=featNames[indices][:featCount],
x = fi[indices][:featCount] , orient='h')
 g.set_xlabel("Relative importance")
 g.set_ylabel("Features")
 g.tick_params(labelsize=12)
 g.set_title(" feature importance")
 return pd.Series(fi[indices], featNames[indices])
 f = avg_FI([lgb_c_clfs], x.columns, 25)
 for feat in ['bend', 'capita', 'cfr', 'slope', 'since', 'chg', 'ersonal',
'world', 'continent', 'nearest', 'surrounding']:
print("{}: {:.2f}".format(feat, f.filter(like=feat ).sum() / f.sum()))
f[:100:3]
print("{}: {:.2f}".format('sup_data',
f[[c for c in f.index if c in sup_data.columns]].sum() / f.sum()))
print("{}: {:.2f}".format('contain_data',
f[[c for c in f.index if c in contain_data.columns]].sum() / f.sum()))
f = avg_FI([lgb_f_clfs], x.columns, 25)
 for feat in ['bend', 'capita', 'cfr', 'slope', 'since', 'chg', 'ersonal',
'world', 'continent', 'nearest', 'surrounding']:
print("{}: {:.2f}".format(feat, f.filter(like=feat ).sum() / f.sum()))
print("{}: {:.2f}".format('sup_data',
f[[c for c in f.index if c in sup_data.columns]].sum() / f.sum()))
print("{}: {:.2f}".format('contain_data',
f[[c for c in f.index if c in contain_data.columns]].sum() / f.sum()))
f = avg_FI([lgb_cfr_clfs], x.columns, 25)
 for feat in ['bend', 'capita', 'cfr', 'slope', 'since', 'chg', 'ersonal',
'world', 'continent', 'nearest', 'surrounding']:
print("{}: {:.2f}".format(feat, f.filter(like=feat ).sum() / f.sum()))
print("{}: {:.2f}".format('sup_data',
f[[c for c in f.index if c in sup_data.columns]].sum() / f.sum()))
print("{}: {:.2f}".format('contain_data',
f[[c for c in f.index if c in contain_data.columns]].sum() / f.sum()))
all_c_clfs = [lgb_c_clfs, ]
all_f_clfs = [lgb_f_clfs]
all_cfr_clfs = [lgb_cfr_clfs]
all_c_noise = [lgb_c_noise]
all_f_noise = [lgb_f_noise]
all_cfr_noise = [lgb_cfr_noise]
NUM_TEST_RUNS = 1
c_preds = np.zeros(( NUM_TEST_RUNS * sum([len(x)for x in all_c_clfs]), len(x_test)))
 f_preds = np.zeros(( NUM_TEST_RUNS * sum([len(x)for x in all_f_clfs]), len(x_test)))
 cfr_preds = np.zeros(( NUM_TEST_RUNS * sum([len(x)for x in all_cfr_clfs]), len(x_test)))
 def avg(x):
return(np.mean(x, axis=0)+ np.median(x, axis=0)) /2
count = 0
for idx, clf in enumerate(lgb_c_clfs):
for i in range(0, NUM_TEST_RUNS):
noise = lgb_c_noise[idx]
c_preds[count,:] = np.clip(clf.predict(noisify(x_test, noise)) , -1 , 10)
 count += 1
count = 0
for idx, clf in enumerate(lgb_f_clfs):
for i in range(0, NUM_TEST_RUNS):
noise = lgb_f_noise[idx]
f_preds[count,:] = np.clip(clf.predict(noisify(x_test, noise)) , -1 , 10)
 count += 1
count = 0
for idx, clf in enumerate(lgb_cfr_clfs):
for i in range(0, NUM_TEST_RUNS):
noise = lgb_cfr_noise[idx]
cfr_preds[count,:] = np.clip(clf.predict(noisify(x_test, noise)) , -10 , 10)
 count += 1
def qPred(preds, pctile, simple=False):
q = np.percentile(preds, pctile, axis = 0)
 if simple:
return q;
resid = preds - q
resid_wtg = 2/100/len(preds)*(np.clip(resid, 0, None)*(pctile)+ np.clip(resid, None, 0)*(100- pctile))
adj = np.sum(resid_wtg, axis = 0)
 return q + adj
q = 50
y_cases_pred_blended_full = qPred(c_preds, q)
 y_fatalities_pred_blended_full = qPred(f_preds, q)
 y_cfr_pred_blended_full = qPred(cfr_preds, q)
 print(np.mean(np.corrcoef(c_preds[::NUM_TEST_RUNS]),axis=0))
print(np.mean(np.corrcoef(f_preds[::NUM_TEST_RUNS]), axis=0))
print(np.mean(np.corrcoef(cfr_preds[::NUM_TEST_RUNS]), axis = 0))
pd.Series(np.std(c_preds, axis = 0)).plot(kind='hist', bins = 50)
 pd.Series(np.std(f_preds, axis = 0)).plot(kind='hist', bins = 50)
 pd.Series(np.std(cfr_preds, axis = 0)).plot(kind='hist', bins = 50)
 y_cfr 
(groups == 'Sierra Leone' ).sum()
pred = pd.DataFrame(np.hstack(( np.transpose(c_preds),
np.transpose(f_preds))), index=x_test.index)
 pred['Place'] = data_test.Place
pred['Date'] = data_test.Date
pred['Date_f'] = data_test.Date_f
pred[(pred.Date == pred.Date.max())&(pred.Date_f == pred.Date_f.max())][30: 60] 
(pred.Place=='Sierra Leone' ).sum()
np.round(pred[(pred.Date == pred.Date.max())&(pred.Date_f == pred.Date_f.max())], 2)[190:220:]
np.round(pred[(pred.Date == pred.Date.max())&(pred.Date_f == pred.Date_f.max())][220:-20],2)
 c_preds.shape
x_test.shape
data_wp = data_test.copy()
if MODEL_Y is 'slope':
data_wp['case_slope'] = y_cases_pred_blended_full
data_wp['fatality_slope'] = y_fatalities_pred_blended_full
else:
data_wp['case_slope'] = y_cases_pred_blended_full / x_test.elapsed
data_wp['fatality_slope'] = y_fatalities_pred_blended_full / x_test.elapsed
data_wp['cfr_pred'] = y_cfr_pred_blended_full
train.Date.max()
test.Date.min()
if len(test)> 0:
base_date = test.Date.min() - datetime.timedelta(1)
 else:
base_date = train.Date.max()
base_date
data_wp_ss = data_wp[data_wp.Date == base_date]
data_wp_ss = data_wp_ss.drop(columns='Date' ).rename(columns = {'Date_f': 'Date'})
 test_wp = pd.merge(test, data_wp_ss[['Date', 'Place', 'case_slope', 'fatality_slope', 'cfr_pred',
'elapsed']],
how='left', on = ['Date', 'Place'])
 first_c_slope = test_wp[~test_wp.case_slope.isnull() ].groupby('Place' ).first()
last_c_slope = test_wp[~test_wp.case_slope.isnull() ].groupby('Place' ).last()
first_f_slope = test_wp[~test_wp.fatality_slope.isnull() ].groupby('Place' ).first()
last_f_slope = test_wp[~test_wp.fatality_slope.isnull() ].groupby('Place' ).last()
first_cfr_pred = test_wp[~test_wp.cfr_pred.isnull() ].groupby('Place' ).first()
last_cfr_pred = test_wp[~test_wp.cfr_pred.isnull() ].groupby('Place' ).last()
test_wp.case_slope = np.where(test_wp.case_slope.isnull() & 
(test_wp.Date < first_c_slope.loc[test_wp.Place].Date.values),
first_c_slope.loc[test_wp.Place].case_slope.values,
test_wp.case_slope 
)
 test_wp.case_slope = np.where(test_wp.case_slope.isnull() & 
(test_wp.Date > last_c_slope.loc[test_wp.Place].Date.values),
last_c_slope.loc[test_wp.Place].case_slope.values,
test_wp.case_slope 
)
 test_wp.fatality_slope = np.where(test_wp.fatality_slope.isnull() & 
(test_wp.Date < first_f_slope.loc[test_wp.Place].Date.values),
first_f_slope.loc[test_wp.Place].fatality_slope.values,
test_wp.fatality_slope 
)
 test_wp.fatality_slope = np.where(test_wp.fatality_slope.isnull() & 
(test_wp.Date > last_f_slope.loc[test_wp.Place].Date.values),
last_f_slope.loc[test_wp.Place].fatality_slope.values,
test_wp.fatality_slope 
)
 test_wp.cfr_pred = np.where(test_wp.cfr_pred.isnull() & 
(test_wp.Date < first_cfr_pred.loc[test_wp.Place].Date.values),
first_cfr_pred.loc[test_wp.Place].cfr_pred.values,
test_wp.cfr_pred 
)
 test_wp.cfr_pred = np.where(test_wp.cfr_pred.isnull() & 
(test_wp.Date > last_cfr_pred.loc[test_wp.Place].Date.values),
last_cfr_pred.loc[test_wp.Place].cfr_pred.values,
test_wp.cfr_pred 
)
 test_wp.case_slope = test_wp.case_slope.interpolate('linear')
 test_wp.fatality_slope = test_wp.fatality_slope.interpolate('linear')
 test_wp.cfr_pred = test_wp.cfr_pred.interpolate('linear')
 test_wp.case_slope = test_wp.case_slope.fillna(0)
 test_wp.fatality_slope = test_wp.fatality_slope.fillna(0)
 LAST_DATE = test.Date.min() - datetime.timedelta(1)
 final = train_bk[train_bk.Date == LAST_DATE ]
test_wp = pd.merge(test_wp, final[['Place', 'ConfirmedCases', 'Fatalities']], on='Place',
how ='left', validate='m:1')
 LAST_DATE
test_wp.ConfirmedCases = np.exp(
 np.log(test_wp.ConfirmedCases + 1)\
+ test_wp.case_slope * 
(test_wp.Date - LAST_DATE ).dt.days)- 1
test_wp.Fatalities = np.exp(
 np.log(test_wp.Fatalities + 1)\
+ test_wp.fatality_slope * 
(test_wp.Date - LAST_DATE ).dt.days)-1
LAST_DATE
test_wp[(test_wp.Country == 'Italy')].groupby('Date' ).sum() [:10]
test_wp[(test_wp.Country == 'US')].groupby('Date' ).sum().iloc[-5:]
final = train_bk[train_bk.Date == test.Date.min() - datetime.timedelta(1)]
final.head()
test['elapsed'] =(test.Date - final.Date.max() ).dt.days
test.elapsed
full_bk = test_wp.copy()
full = test_wp.copy()
BASE_RATE = 0.01
CFR_CAP = 0.13
lplot(full_bk)
 lplot(full_bk, columns = ['case_slope', 'fatality_slope'])
 full['cfr_imputed_fatalities_low'] = full.ConfirmedCases * np.exp(full.cfr_pred)/ np.exp(0.5)
 full['cfr_imputed_fatalities_high'] = full.ConfirmedCases * np.exp(full.cfr_pred)* np.exp(0.5)
 full['cfr_imputed_fatalities'] = full.ConfirmedCases * np.exp(full.cfr_pred)
 full[(full.case_slope > 0.02)& 
(full.Fatalities < full.cfr_imputed_fatalities_low)& 
(full.cfr_imputed_fatalities_low > 0.3)& 
(full.Fatalities < 100000)& 
(full.Country!='China')& 
(full.Date == datetime.datetime(2020, 4,15)) ] \ 
.groupby('Place' ).last() \ 
.sort_values('Fatalities', ascending=False ).iloc[:, 9:] 
(np.log(full.Fatalities + 1)-np.log(full.cfr_imputed_fatalities)).plot(kind='hist', bins = 250)
 full[(full.case_slope > 0.02)& 
(full.Fatalities < full.cfr_imputed_fatalities_low)& 
(full.cfr_imputed_fatalities_low > 0.3)& 
(full.Fatalities < 100000)& 
(~full.Country.isin(['China', 'Korea, South'])) ][full.Date==train.Date.max() ]\ 
.groupby('Place' ).first() \ 
.sort_values('cfr_imputed_fatalities', ascending=False ).iloc[:, 9:]
full.Fatalities = np.where(
(full.case_slope > 0.02)& 
(full.Fatalities <= full.cfr_imputed_fatalities_low)& 
(full.cfr_imputed_fatalities_low > 0.3)& 
(full.Fatalities < 100000)& 
(~full.Country.isin(['China', 'Korea, South'])) , 
(full.cfr_imputed_fatalities_high + full.cfr_imputed_fatalities)/2,
full.Fatalities)
 full['elapsed'] =(test_wp.Date - LAST_DATE ).dt.days
full[(full.case_slope > 0.02)& 
(np.log(full.Fatalities + 1)< np.log(full.ConfirmedCases * BASE_RATE + 1)- 0.5)& 
(full.Country != 'China')]\
[full.Date == datetime.datetime(2020, 4, 5)] \ 
.groupby('Place' ).last().sort_values('ConfirmedCases', ascending=False ).iloc[:,8:]
full.Fatalities = np.where(( full.case_slope > 0.02)& 
(full.Fatalities < full.ConfirmedCases * BASE_RATE)& 
(full.Country != 'China'),
np.exp(
 np.log(full.ConfirmedCases * BASE_RATE + 1)\
* np.clip(0.5*(full.elapsed - 1)/ 30, 0, 1)\
+ np.log(full.Fatalities +1)\
* np.clip(1 - 0.5*(full.elapsed - 1)/ 30, 0, 1)
)-1
,
full.Fatalities)
 full[(full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China')]\ 
.groupby('Place' ).count() \ 
.sort_values('ConfirmedCases', ascending=False ).iloc[:, 8:]
full[(full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high * 2)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China')]\ 
.groupby('Place' ).last() \ 
.sort_values('ConfirmedCases', ascending=False ).iloc[:, 8:]
full[(full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high * 1.5)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China')][full.Date==train.Date.max() ]\ 
.groupby('Place' ).first() \ 
.sort_values('ConfirmedCases', ascending=False ).iloc[:, 8:]
full.Fatalities = np.where(( full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high * 2)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China'),
full.cfr_imputed_fatalities,
full.Fatalities)
 full.Fatalities = np.where(( full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China'),
np.exp(
 0.6667 * np.log(full.Fatalities + 1)\
+ 0.3333 * np.log(full.cfr_imputed_fatalities + 1)
)- 1,
full.Fatalities)
 full[(full.Fatalities > full.ConfirmedCases * CFR_CAP)& 
(full.ConfirmedCases > 1000)
 ].groupby('Place' ).last().sort_values('Fatalities', ascending=False)
(np.log(full.Fatalities + 1)-np.log(full.cfr_imputed_fatalities)).plot(kind='hist', bins = 250)
 assert len(pd.merge(full, final, on='Place', suffixes =('', '_i'), validate='m:1')) == len(full)
 ffm = pd.merge(full, final, on='Place', suffixes =('', '_i'), validate='m:1')
 ffm['fatality_slope'] =(np.log(ffm.Fatalities + 1)- np.log(ffm.Fatalities_i + 1)) / ffm.elapsed
ffm['case_slope'] =(np.log(ffm.ConfirmedCases + 1)- np.log(ffm.ConfirmedCases_i + 1)) / ffm.elapsed
ffm[np.log(ffm.Fatalities+1)< np.log(ffm.Fatalities_i+1)- 0.2] [['Place', 'Date', 'elapsed', 'Fatalities', 'Fatalities_i']]
ffm[np.log(ffm.ConfirmedCases + 1)< np.log(ffm.ConfirmedCases_i+1)- 0.2] [['Place', 'elapsed', 'ConfirmedCases', 'ConfirmedCases_i']]
full_bk[(full_bk.Date == test.Date.max())& 
(~full_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)
 lplot(ffm[~ffm.Place.isin(new_places)])
 lplot(ffm[~ffm.Place.isin(new_places)], columns = ['case_slope', 'fatality_slope'])
 ffm.fatality_slope = np.clip(ffm.fatality_slope, None, 0.5)
 for lr in [0.2, 0.14, 0.1, 0.07, 0.05, 0.03, 0.01 ]:
ffm.loc[(ffm.Place==ffm.Place.shift(4))
&(ffm.Place==ffm.Place.shift(-4)) , 'fatality_slope'] = \ 
(ffm.fatality_slope.shift(-2)* 0.25 \
+ ffm.fatality_slope.shift(-1)* 0.5 \
+ ffm.fatality_slope \
+ ffm.fatality_slope.shift(1)* 0.5 \
+ ffm.fatality_slope.shift(2)* 0.25)/ 2.5
ffm.ConfirmedCases = np.exp(
 np.log(ffm.ConfirmedCases_i + 1)\
+ ffm.case_slope *
ffm.elapsed)- 1
ffm.Fatalities = np.exp(
 np.log(ffm.Fatalities_i + 1)\
+ ffm.fatality_slope *
ffm.elapsed)- 1
lplot(ffm[~ffm.Place.isin(new_places)])
 lplot(ffm[~ffm.Place.isin(new_places)], columns = ['case_slope', 'fatality_slope'])
 ffm[(ffm.Date == test.Date.max())& 
(~ffm.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)
 ffm_bk = ffm.copy()
ffm = ffm_bk.copy()
counter = Counter(data.Place)
 median_count = np.median([ counter[group] for group in ffm.Place])
 c_count = [ np.clip(
 np.power(counter[group] / median_count, -1.5), None, 2.5)for group in ffm.Place]
RATE_MULT = 0.00
RATE_ADD = 0.003
LAG_FALLOFF = 15
ma_factor = np.clip(( ffm.elapsed - 14)/ 14 , 0, 1)
 ffm.case_slope = np.where(ffm.elapsed > 0,
0.7 * ffm.case_slope *(1+ ma_factor * RATE_MULT)\
+ 0.3 *(ffm.case_slope.ewm(span=LAG_FALLOFF ).mean() \
* np.clip(ma_factor, 0, 1)
 + ffm.case_slope * np.clip(1 - ma_factor, 0, 1))
+ RATE_ADD * ma_factor * c_count,
ffm.case_slope)
 RATE_MULT = 0
RATE_ADD = 0.015
LAG_FALLOFF = 15
ma_factor = np.clip(( ffm.elapsed - 10)/ 14 , 0, 1)
 ffm.fatality_slope = np.where(ffm.elapsed > 0,
0.3 * ffm.fatality_slope *(1+ ma_factor * RATE_MULT)\
+ 0.7*(ffm.fatality_slope.ewm(span=LAG_FALLOFF ).mean() \
* np.clip(ma_factor, 0, 1)
 + ffm.fatality_slope * np.clip(1 - ma_factor, 0, 1))
+ RATE_ADD * ma_factor * c_count \
*(ffm.Country != 'China')
 ,
ffm.case_slope)
 ffm.ConfirmedCases = np.exp(
 np.log(ffm.ConfirmedCases_i + 1)\
+ ffm.case_slope *
ffm.elapsed)- 1
ffm.Fatalities = np.exp(
 np.log(ffm.Fatalities_i + 1)\
+ ffm.fatality_slope *
ffm.elapsed)- 1
lplot(ffm[~ffm.Place.isin(new_places)])
 lplot(ffm[~ffm.Place.isin(new_places)], columns = ['case_slope', 'fatality_slope'])
 ffm_bk[(ffm_bk.Date == test.Date.max())& 
(~ffm_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[:15]
ffm[(ffm.Date == test.Date.max())& 
(~ffm.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[:15]
ffm_bk[(ffm_bk.Date == test.Date.max())& 
(~ffm_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[-50:]
ffm[(ffm.Date == test.Date.max())& 
(~ffm.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).loc[ffm_bk[(ffm_bk.Date == test.Date.max())& 
(~ffm_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[-50:].index]
NUM_TEST_DATES = len(test.Date.unique())
 base = np.zeros(( 2, NUM_TEST_DATES))
base2 = np.zeros(( 2, NUM_TEST_DATES))
for idx, c in enumerate(['ConfirmedCases', 'Fatalities']):
for n in range(0, NUM_TEST_DATES):
base[idx,n] = np.mean(
 np.log(train[(( train.Date < test.Date.min())) & 
(train.ConfirmedCases > 0)].groupby('Country' ).nth(n)[c]+1))
base = np.pad(base,(( 0,0),(6,0)) , mode='constant', constant_values = 0)
 for n in range(0, base2.shape[1]):
base2[:, n] = np.mean(base[:, n+0: n+7], axis = 1)
 new_places = train[(train.Date == test.Date.min() - datetime.timedelta(1)) & 
(train.ConfirmedCases == 0)
 ].Place
ffm.ConfirmedCases = np.where(ffm.Place.isin(new_places),
base2[ 0,(ffm.Date - test.Date.min() ).dt.days],
ffm.ConfirmedCases)
 ffm.Fatalities = np.where(ffm.Place.isin(new_places),
base2[ 1,(ffm.Date - test.Date.min() ).dt.days],
ffm.Fatalities)
 ffm[ffm.Country=='US'].groupby('Date' ).agg(
 {'ForecastId': 'count',
'case_slope': 'mean',
'fatality_slope': 'mean',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
})
 sub = pd.read_csv(input_path + '/submission.csv')
 scl = sub.columns.to_list()
print(full_bk.groupby('Place' ).last() [['Date', 'ConfirmedCases', 'Fatalities']])
 print(ffm.groupby('Place' ).last() [['Date', 'ConfirmedCases', 'Fatalities']])
 if ffm[scl].isnull().sum().sum() == 0:
out = full_bk[scl] * 0.7 + ffm[scl] * 0.3
else:
print('using full-bk')
 out = full_bk[scl]
out = out[sub.columns.to_list() ]
out.ForecastId = np.round(out.ForecastId, 0 ).astype(int)
 out = np.round(out, 2)
 private = out
full_pred = pd.concat(( private, public[~public.ForecastId.isin(private.ForecastId)]),
ignore_index=True ).sort_values('ForecastId')
 sub1.sort_values("ForecastId", inplace=True)
 sub4=full_pred
sub4.sort_values("ForecastId", inplace=True)
 train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 dacheckv='2020-04-01'
train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train=train[train.Date>dacheckv]
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test1=test.merge(train,on=['Country_Region','Province_State','Date'],how='inner')
 print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['ConfirmedCases'].values,test1['ConfirmedCases'].values))
print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['Fatalities'].values,test1['Fatalities'].values))
print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['ConfirmedCases'].values,test1['ConfirmedCases'].values))
print(RMSLE(sub1[sub1.ForecastId.isin(test1.ForecastId.values)]['Fatalities'].values,test1['Fatalities'].values))
print(RMSLE(sub4[sub4.ForecastId.isin(test1.ForecastId.values)]['ConfirmedCases'].values,test1['ConfirmedCases'].values))
print(RMSLE(sub4[sub4.ForecastId.isin(test1.ForecastId.values)]['Fatalities'].values,test1['Fatalities'].values))
sub1=sub1.reset_index(drop=True)
 sub4=sub4.reset_index(drop=True)
 sub_df = sub1.copy()
TARGETS = ["ConfirmedCases", "Fatalities"]
for t in TARGETS:
if t=='ConfirmedCases':
sub_df[t] = sub1[t]*0.01 + sub4[t]*0.99
else:
sub_df[t] = sub1[t]*0.01 + sub4[t]*0.99
sub_df.to_csv("submission.csv", index=False)
 print(RMSLE(sub_df[sub_df.ForecastId.isin(test1.ForecastId.values)]['ConfirmedCases'].values,test1['ConfirmedCases'].values))
print(RMSLE(sub_df[sub_df.ForecastId.isin(test1.ForecastId.values)]['Fatalities'].values,test1['Fatalities'].values)) 
<compute_test_metric>
test_labels = pd.read_csv(path/'sample_submission.csv')
 test = ImageList.from_df(test_labels, path = path_test, suffix = '.png' )
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
def fix_target(frame, key, target, new_target_name="target"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if current_value<previous_value:
current_value=previous_value
target[i] =current_value
target[i] =max(0,target[i])
 frame[new_target_name] = np.array(target)
 def rate(frame, key, target, new_target_name="rate"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[1.0 for k in range(len(target)) ]
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if previous_value!=0.0:
rate[i]=current_value/previous_value
rate[i] =max(1,rate[i])
 frame[new_target_name] = np.array(rate)
 def get_data_by_key(dataframe, key, key_value, fields=None):
mini_frame=dataframe[dataframe[key]==key_value]
if not fields is None:
mini_frame=mini_frame[fields].values
return mini_frame
directory="/kaggle/input/covid19-global-forecasting-week-4/"
model_directory="/kaggle/input/modelv2-dir/model"
geo_dir=None
extra_stable_columns=None
group_by_columns=None
if not group_by_columns is None and "continent" in group_by_columns:
assert not geo_dir is None
train=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 test=pd.read_csv('/kaggle/input/datatestadd2/test_new.csv', parse_dates=["Date"], engine="python")
 train["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 test["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 if not geo_dir is None:
region_metadata=pd.read_csv(geo_dir+"region_metadata.csv")
 train=pd.merge(train,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 test=pd.merge(test,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 train.to_csv(directory + "train_plus_geo.csv", index=False)
 group_names=None
if not group_by_columns is None and len(group_by_columns)>0:
group_names=[]
for group in group_by_columns:
groupss=train[["Date", group,target1,target2]]
grp=groupss.groupby(["Date", group], as_index=False ).sum()
grp.columns=["Date", group,group +"_" + target1,group +"_" + target2 ]
train=pd.merge(train,grp, how="left", left_on=["Date", group,group], right_on=["Date", group,group])
 for gr in [group +"_" + target1,group +"_" + target2]:
rate(train, key, gr, new_target_name="rate_" +gr)
 group_names+=["rate_" +gr]
train.to_csv(directory + "train_plus_groups.csv", index=False)
 max_train_date=train["Date"].max()
max_test_date=test["Date"].max()
horizon=(max_test_date-max_train_date ).days
print("horizon", int(horizon))
print("max_train_date",(max_train_date))
print("max_test_date",(max_test_date))
target1="ConfirmedCases"
target2="Fatalities"
key="key" 
<categorify>
src =(ImageList.from_df(labels, path = path_train, suffix = '.png')
.split_by_rand_pct(seed = 2019)
.label_from_df(cols = 'diagnosis')
.add_test(test))
fix_target(train, key, target1, new_target_name=target1)
 fix_target(train, key, target2, new_target_name=target2)
 rate(train, key, target1, new_target_name="rate_" +target1)
 rate(train, key, target2, new_target_name="rate_" +target2)
 unique_keys=train[key].unique()
print(len(unique_keys))
train<categorify>
data =(
 src.transform(
 tfms,
size = 446,
resize_method=ResizeMethod.SQUISH,
padding_mode='zeros' 
)
.databunch(bs=16)
.normalize(aptos19_stats))
def get_lags(rate_array, current_index, size=20):
lag_confirmed_rate=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
lag_confirmed_rate[j]=rate_array[current_index-j]
else :
break
return lag_confirmed_rate
def days_ago_thresold_hit(full_array, indx, thresold):
days_ago_confirmed_count_10=-1
if full_array[indx]>thresold:
for j in range(indx,-1,-1):
entered=False
if full_array[j]<=thresold:
days_ago_confirmed_count_10=abs(j-indx)
 entered=True
break
if entered==False:
days_ago_confirmed_count_10=100
return days_ago_confirmed_count_10
def ewma_vectorized(data, alpha):
sums=sum([(alpha**(k+1)) *data[k] for k in range(len(data)) ])
 counts=sum([(alpha**(k+1)) for k in range(len(data)) ])
 return sums/counts
def generate_ma_std_window(rate_array, current_index, size=20, window=3):
ma_rate_confirmed=[-1 for k in range(size)]
std_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ma_rate_confirmed[j]=np.mean(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 std_rate_confirmed[j]=np.std(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 else :
break
return ma_rate_confirmed, std_rate_confirmed
def generate_ewma_window(rate_array, current_index, size=20, window=3, alpha=0.05):
ewma_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ewma_rate_confirmed[j]=ewma_vectorized(rate_array[max(0,current_index-j-window+1):current_index-j+1, ], alpha)
 else :
break
return ewma_rate_confirmed
def get_target(rate_col, indx, horizon=33, average=3, use_hard_rule=False):
target_values=[-1 for k in range(horizon)]
cou=0
for j in range(indx+1, indx+1+horizon):
if j<len(rate_col):
if average==1:
target_values[cou]=rate_col[j]
else :
if use_hard_rule and j +average <=len(rate_col):
target_values[cou]=np.mean(rate_col[j:j +average])
 else :
target_values[cou]=np.mean(rate_col[j:min(len(rate_col),j +average)])
 cou+=1
else :
break
return target_values
def dereive_features(frame, confirmed, fatalities, rate_confirmed, rate_fatalities,
horizon ,size=20, windows=[3,7], days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10],
extra_data=None, groups_data=None, windows_group=[3,7], size_group=20,
days_back_confimed_group=[1,10,100]):
targets=[]
if not extra_data is None:
assert len(extra_stable_columns)==extra_data.shape[1]
if not groups_data is None:
assert len(group_names)==groups_data.shape[1]
names=["lag_confirmed_rate" + str(k+1)for k in range(size)]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["lag_fatalities_rate" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_data is None:
names+=[k for k in extra_stable_columns]
if not groups_data is None:
for gg in range(groups_data.shape[1]):
names+=["lag_rate_group_"+ str(gg+1)+ "_" + str(k+1)for k in range(size_group)]
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)]
names+=["std_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)]
names+=["confirmed_plus" + str(k+1)for k in range(horizon)]
names+=["fatalities_plus" + str(k+1)for k in range(horizon)]
features=[]
for i in range(len(confirmed)) :
row_features=[]
lag_confirmed_rate=get_lags(rate_confirmed, i, size=size)
 row_features+=lag_confirmed_rate
for day in days_back_confimed:
days_ago_confirmed_count_10=days_ago_thresold_hit(confirmed, i, day)
 row_features+=[days_ago_confirmed_count_10]
for window in windows:
ma3_rate_confirmed,std3_rate_confirmed= generate_ma_std_window(rate_confirmed, i, size=size, window=window)
 row_features+= ma3_rate_confirmed
row_features+= std3_rate_confirmed
ewma3_rate_confirmed=generate_ewma_window(rate_confirmed, i, size=size, window=window, alpha=0.05)
 row_features+= ewma3_rate_confirmed
lag_fatalities_rate=get_lags(rate_fatalities, i, size=size)
 row_features+=lag_fatalities_rate
for day in days_back_fatalities:
days_ago_fatalitiescount_2=days_ago_thresold_hit(fatalities, i, day)
 row_features+=[days_ago_fatalitiescount_2]
for window in windows:
ma3_rate_fatalities,std3_rate_fatalities= generate_ma_std_window(rate_fatalities, i, size=size, window=window)
 row_features+= ma3_rate_fatalities
row_features+= std3_rate_fatalities
ewma3_rate_fatalities=generate_ewma_window(rate_fatalities, i, size=size, window=window, alpha=0.05)
 row_features+= ewma3_rate_fatalities
confirmed_level=0

 confirmed_level= confirmed[i]
row_features+=[confirmed_level]
fatalities_is_level=0

 fatalities_is_level= fatalities[i]
row_features+=[fatalities_is_level]
if not extra_data is None:
row_features+=extra_data[i].tolist()
if not groups_data is None:
for gg in range(groups_data.shape[1]):
this_group=groups_data[:,gg].tolist()
lag_group_rate=get_lags(this_group, i, size=size_group)
 row_features+=lag_group_rate
for day in days_back_confimed_group:
days_ago_groupcount_2=days_ago_thresold_hit(this_group, i, day)
 row_features+=[days_ago_groupcount_2]
for window in windows_group:
ma3_rate_group,std3_rate_group= generate_ma_std_window(this_group, i, size=size_group, window=window)
 row_features+= ma3_rate_group
row_features+= std3_rate_group
confirmed_plus=get_target(rate_confirmed, i, horizon=horizon)
 row_features+= confirmed_plus
fatalities_plus=get_target(rate_fatalities, i, horizon=horizon)
 row_features+= fatalities_plus
features.append(row_features)
 new_frame=pd.DataFrame(data=features, columns=names ).reset_index(drop=True)
 frame=frame.reset_index(drop=True)
 frame=pd.concat([frame, new_frame], axis=1)
 return frame
def feature_engineering_for_single_key(frame, group, key, horizon=33, size=20, windows=[3,7],
days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10],
extra_stable_=None, group_nams=None,windows_group=[3,7],
size_group=20, days_back_confimed_group=[1,10,100]):
mini_frame=get_data_by_key(frame, group, key, fields=None)
 mini_frame_with_features=dereive_features(mini_frame, mini_frame["ConfirmedCases"].values,
mini_frame["Fatalities"].values, mini_frame["rate_ConfirmedCases"].values,
mini_frame["rate_Fatalities"].values, horizon ,size=size, windows=windows,
days_back_confimed=days_back_confimed, days_back_fatalities=days_back_fatalities,
extra_data=mini_frame[extra_stable_].values if not extra_stable_ is None else None,
groups_data=mini_frame[group_nams].values if not group_nams is None else None,
windows_group=windows_group, size_group=size_group,
days_back_confimed_group=days_back_confimed_group)
 return mini_frame_with_features<predict_on_test>
kappa = KappaScore("quadratic")
 kappa
def predict(xtest,input_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 model= joblib.load(input_name)
 preds=model.predict(xtest)
 baggedpred+=preds
return baggedpred 
<define_variables>
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )
names=["lag_confirmed_rate" + str(k+1)for k in range(size)]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["lag_fatalities_rate" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_stable_columns is None and len(extra_stable_columns)>0:
names+=[k for k in extra_stable_columns]
if not group_names is None:
for gg in range(len(group_names)) :
names+=["lag_rate_group_"+ str(gg+1)+ "_" + str(k+1)for k in range(size_group)]
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)]
names+=["std_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)] 
<categorify>
learn = cnn_learner(
 data,
models.resnet50,
metrics = [accuracy,quadratic_kappa],
model_dir = Path('.. /kaggle/working'),
path = Path("."),
pretrained=True
)
def decay_4_first_10_then_1_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
if j<10:
arr[j]=1.+(max(1,array[j])-1.) /4.
 else :
arr[j]=1.
 return arr
	
def decay_16_first_10_then_1_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
if j<10:
arr[j]=1.+(max(1,array[j])-1.) /16.
 else :
arr[j]=1.
 return arr	
def decay_2_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /2.
 return arr
def decay_4_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /4.
 return arr 	
	
def acceleratorx2_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) *2.
 return arr
def decay_1_5_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.5
return arr
def stay_same_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.
 return arr
def decay_2_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /2.
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def decay_4_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /4.
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def linear_last_12_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=max(1,array[j])
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
decay_4_first_10_then_1 =[ "Heilongjiang_China","Liaoning_China","Shanghai_China"]
decay_4_first_10_then_1_fatality=[]
decay_16_first_10_then_1 =["Beijing_China","Fujian_China","Guangdong_China","Shandong_China","Sichuan_China","Zhejiang_China"]
decay_16_first_10_then_1_fatality=[]
decay_4=["nan_Bhutan","nan_Burundi","nan_Cabo Verde","Prince Edward Island_Canada",
"nan_Central African Republic","Inner Mongolia_China","nan_Maldives"]
decay_4_fatality=["nan_Congo(Kinshasa)"]
decay_2 =["nan_Congo(Kinshasa)","Faroe Islands_Denmark","nan_Eritrea","French Guiana_France","nan_Korea, South","nan_MS Zaandam"]
decay_2_fatality=[]
stay_same=["nan_Diamond Princess","nan_Timor-Leste"]
		
stay_same_fatality=["Beijing_China","Fujian_China","Guangdong_China","Shandong_China",
"Sichuan_China","Zhejiang_China", "Heilongjiang_China","Liaoning_China","Shanghai_China"]
normal=[]
normal_fatality=["nan_Korea, South","New York_US"]
decay_4_last_12_linear_inter =[ "Greenland_Denmark","nan_Dominica","nan_Equatorial Guinea","nan_Eswatini","New Caledonia_France",
"Saint Barthelemy_France","St Martin_France","nan_Gambia","nan_Grenada","nan_Holy See","nan_Mauritania","nan_Namibia","nan_Nicaragua"
,"nan_Papua New Guinea","nan_Saint Lucia","nan_Saint Vincent and the Grenadines","nan_Seychelles","nan_Sierra Leone","nan_Somalia","nan_Suriname",
"Anguilla_United Kingdom","British Virgin Islands_United Kingdom","Montserrat_United Kingdom","Turks and Caicos Islands_United Kingdom","nan_Zimbabwe"
, "Hong Kong_China","Curacao_Netherlands"]
decay_4_last_12_linear_inter_fatality=[]
decay_2_last_12_linear_inter =[ "nan_Chad",
"nan_Congo(Brazzaville)","nan_Fiji","French Polynesia_France","nan_Gabon",
"nan_Guyana","nan_Laos","nan_Nepal","Sint Maarten_Netherlands",
"nan_Saint Kitts and Nevis","nan_Sudan","nan_Syria","nan_Tanzania",
"Bermuda_United Kingdom","Cayman Islands_United Kingdom","nan_Zambia","Northwest Territories_Canada","Yukon_Canada"
,"nan_Mongolia","nan_Uganda"]
decay_2_last_12_linear_inter_fatality=[]
acceleratorx2=[]
acceleratorx2_fatality=[]
warm_st=['nan_Angola','nan_Antigua and Barbuda','Northern Territory_Australia','nan_Bahamas',
'nan_Bangladesh','nan_Belize','nan_Benin','nan_Botswana','nan_Burundi','nan_Cabo Verde','nan_Cameroon',
'nan_Central African Republic','nan_Chad','Hong Kong_China',"nan_Cote d'Ivoire",'nan_Cuba','Greenland_Denmark',
'nan_Dominica','nan_Equatorial Guinea','nan_Eritrea','nan_Eswatini','nan_Fiji','French Polynesia_France','New Caledonia_France',
'Saint Barthelemy_France','St Martin_France','nan_Gabon','nan_Gambia','nan_Grenada','nan_Guyana','nan_Haiti','nan_Holy See',
'nan_Honduras','nan_Ireland','nan_Korea, South','nan_Laos','nan_Liberia','nan_Libya','nan_Maldives','nan_Mali',
'nan_Mauritania','nan_Mauritius','nan_Mongolia','nan_Mozambique','nan_Namibia','nan_Nepal','Aruba_Netherlands',
'nan_Nicaragua','nan_Niger','nan_Papua New Guinea','nan_Saint Kitts and Nevis','nan_Saint Lucia',
'nan_Saint Vincent and the Grenadines','nan_Seychelles','nan_Sierra Leone','nan_Somalia',
'nan_Spain','nan_Sudan','nan_Suriname','nan_Syria','nan_Tanzania','nan_Togo','nan_Uganda','Anguilla_United Kingdom',
'Bermuda_United Kingdom','British Virgin Islands_United Kingdom','Channel Islands_United Kingdom',
'Gibraltar_United Kingdom','Isle of Man_United Kingdom','Montserrat_United Kingdom','nan_United Kingdom',
'Turks and Caicos Islands_United Kingdom','nan_Uzbekistan','nan_Zimbabwe',
]
decay_1_5 =["nan_Angola" ,"nan_Antigua and Barbuda","Montana_US","Nebraska_US","nan_Bangladesh","Illinois_US"
,"Northern Territory_Australia","nan_Bahamas","nan_Bahrain","nan_Barbados" ,"nan_Belize","nan_Benin",
	"nan_Botswana","nan_Brunei","Manitoba_Canada","New Brunswick_Canada","Saskatchewan_Canada",
	"nan_Cote d'Ivoire","nan_France","nan_Guinea-Bissau","nan_Haiti","nan_Italy","nan_Libya","nan_Malta","nan_Mauritius",
	"Aruba_Netherlands","nan_Niger","nan_Spain","nan_Togo","Guam_US","Iowa_US","Idaho_US","Connecticut_US","California_US","New York_US","Virgin Islands_US",
	"Channel Islands_United Kingdom","Gibraltar_United Kingdom","Isle of Man_United Kingdom","nan_United Kingdom",'nan_Burma']
decay_1_5_fatality=["nan_Cameroon","nan_Mali","nan_Cuba","Delaware_US","District of Columbia_US",
"Kansas_US","Louisiana_US","Michigan_US","New Mexico_US","Ohio_US","Oklahoma_US","Pennsylvania_US","Puerto Rico_US","Rhode Island_US",
"South Dakota_US" ,"Tennessee_US","Texas_US","Vermont_US","Virginia_US","West Virginia_US","nan_Uzbekistan"]
linear_last_12=["nan_Honduras","nan_Ireland","Colorado_US","nan_Liberia","nan_Mozambique"]
linear_last_12_fatality=[]
tr_frame=train_frame
features_train=tr_frame[names].values
standard_confirmed_train=tr_frame["ConfirmedCases"].values
standard_fatalities_train=tr_frame["Fatalities"].values
current_confirmed_train=tr_frame["ConfirmedCases"].values
features_cv=[]
name_cv=[]
standard_confirmed_cv=[]
standard_fatalities_cv=[]
names_=tr_frame["key"].values
training_horizon=int(features_train.shape[0]/len(unique_keys))
print("training horizon = ",training_horizon)
 for dd in range(training_horizon-1,features_train.shape[0],training_horizon):
features_cv.append(features_train[dd])
 name_cv.append(names_[dd])
 standard_confirmed_cv.append(standard_confirmed_train[dd])
 standard_fatalities_cv.append(standard_fatalities_train[dd])
 print(name_cv[-1], standard_confirmed_cv[-1], standard_fatalities_cv[-1])
 features_cv=np.array(features_cv)
 preds_confirmed_cv=np.zeros(( features_cv.shape[0],horizon))
preds_confirmed_standard_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_standard_cv=np.zeros(( features_cv.shape[0],horizon))
overal_rmsle_metric_confirmed=0.0
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"confirmed"+ str(j%30))
preds_confirmed_cv[:,j]=preds
print(" modelling confirmed, case %d, , original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_confirmed_cv.shape[0]):
current_prediction=standard_confirmed_cv[ii]
if current_prediction==0 :
current_prediction=0.1
this_preds=preds_confirmed_cv[ii].tolist()
name=name_cv[ii]
reserve=this_preds[0]
	
if name in normal:
this_preds=this_preds	
	
elif name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1:
this_preds=decay_16_first_10_then_1_f(this_preds)		
		
elif name in decay_4_last_12_linear_inter:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_4:
this_preds=decay_4_f(this_preds)
 elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 if name in warm_st:
this_preds[0]=reserve
for j in range(preds_confirmed_cv.shape[1]):
current_prediction*=max(1,this_preds[j])
 preds_confirmed_standard_cv[ii][j]=current_prediction
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"fatal"+ str(j%30))
preds_fatalities_cv[:,j]=preds
print(" modelling fatalities, case %d, original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_fatalities_cv.shape[0]):
current_prediction=standard_fatalities_cv[ii]
if current_prediction==0 and standard_confirmed_cv[ii]>400:
current_prediction=0.1
this_preds=preds_fatalities_cv[ii].tolist()
name=name_cv[ii]
reserve=this_preds[0]
	
if name in normal_fatality:
this_preds=this_preds	 	
	
elif name in decay_4_first_10_then_1_fatality:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1_fatality:
this_preds=decay_16_first_10_then_1_f(this_preds)
 elif name in decay_4_last_12_linear_inter_fatality:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_4_fatality:
this_preds=decay_4_f(this_preds)		
		
elif name in decay_2_fatality:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter_fatality:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5_fatality:
this_preds=decay_1_5_f(this_preds)	
elif name in linear_last_12_fatality:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2_fatality:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same_fatality:
this_preds=stay_same_f(this_preds)
 elif name in normal:
this_preds=this_preds	
	
elif name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1:
this_preds=decay_16_first_10_then_1_f(this_preds)		
elif name in decay_4_last_12_linear_inter:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_4:
this_preds=decay_4_f(this_preds)
 		
elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 if name in warm_st:
this_preds[0]=reserve
for j in range(preds_fatalities_cv.shape[1]):
if current_prediction==0 and(preds_confirmed_standard_cv[ii][j]>400 or "Malta" in name or "Somalia" in name):
current_prediction=1.
 if j==0 and "nan_Antigua and Barbuda" in name:
current_prediction=2.
 if j==0 and 'nan_Burma' in name:
current_prediction=3.
 current_prediction*=max(1,this_preds[j])
 preds_fatalities_standard_cv[ii][j]=current_prediction<categorify>
learn.fit_one_cycle(5, slice(1e-2, 1e-1))
key_to_confirmed_rate={}
key_to_fatality_rate={}
key_to_confirmed={}
key_to_fatality={}
print(len(features_cv), len(name_cv),len(standard_confirmed_cv),len(standard_fatalities_cv))
print(preds_confirmed_cv.shape,preds_confirmed_standard_cv.shape,preds_fatalities_cv.shape,preds_fatalities_standard_cv.shape)
 for j in range(len(name_cv)) :
key_to_confirmed_rate[name_cv[j]]=preds_confirmed_cv[j,:].tolist()
key_to_fatality_rate[name_cv[j]]=preds_fatalities_cv[j,:].tolist()
key_to_confirmed[name_cv[j]] =preds_confirmed_standard_cv[j,:].tolist()
key_to_fatality[name_cv[j]]=preds_fatalities_standard_cv[j,:].tolist()<merge>
learn.save('resnet50-1' )
train_new=train[["Date","ConfirmedCases","Fatalities","key","rate_ConfirmedCases","rate_Fatalities"]]
test_new=pd.merge(test,train_new, how="left", left_on=["key","Date"], right_on=["key","Date"] ).reset_index(drop=True)
 test_new<categorify>
preds, _ = learn.get_preds(ds_type=DatasetType.Test )
def fillin_columns(frame,key_column, original_name, training_horizon, test_horizon, unique_values, key_to_values):
keys=frame[key_column].values
original_values=frame[original_name].values.tolist()
print(len(keys), len(original_values), training_horizon ,test_horizon,len(key_to_values))
for j in range(unique_values):
current_index=(j *(training_horizon +test_horizon)) +training_horizon
current_key=keys[current_index]
values=key_to_values[current_key]
co=0
for g in range(current_index, current_index + test_horizon):
original_values[g]=values[co]
co+=1
frame[original_name]=original_values
all_days=int(test_new.shape[0]/len(unique_keys))
tr_horizon=all_days-horizon
print(all_days,tr_horizon, horizon)
 fillin_columns(test_new,"key", 'ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed)
 fillin_columns(test_new,"key", 'Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality)
 submission=test_new[["ForecastId","ConfirmedCases","Fatalities"]]
submission<filter>
submission = pd.read_csv(path/'sample_submission.csv')
 submission.head()
train[train['Country_Region']=='Iran']<filter>
preds = np.array(preds.argmax(1)).astype(int ).tolist()
preds[:5]
test[test['Country_Region']=='Iran']<concatenate>
submission['diagnosis'] = preds
submission.head()
df = pd.concat([test, submission], axis=0)
 df.tail()<filter>
warnings.simplefilter(action='ignore', category=FutureWarning )
<save_to_csv><EOS> 
submission.to_csv('submission.csv', index = False )