Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<import_modules>
pd.set_option('display.max_columns', 100)
 warnings.filterwarnings('ignore')
 df_train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 print(df_train.shape)
 df_train.head()
df_test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 print(df_test.shape)
 df_test.head()
df_traintest = pd.concat([df_train, df_test])
 print(df_train.shape, df_test.shape, df_traintest.shape)
 df_traintest['Date'] = pd.to_datetime(df_traintest['Date'])
 df_traintest['day'] = df_traintest['Date'].apply(lambda x: x.dayofyear ).astype(np.int16)
 df_traintest.head()
day_before_valid = 104-7
day_before_public = 104
day_before_private = df_traintest['day'][pd.isna(df_traintest['ForecastId'])].max()
print(df_traintest['Date'][df_traintest['day']==day_before_valid].values[0])
 print(df_traintest['Date'][df_traintest['day']==day_before_public].values[0])
 print(df_traintest['Date'][df_traintest['day']==day_before_private].values[0])
 def func(x):
try:
x_new = x['Country_Region'] + "/" + x['Province_State']
except:
x_new = x['Country_Region']
return x_new
df_traintest['place_id'] = df_traintest.apply(lambda x: func(x), axis=1)
 df_traintest.head()
df_traintest[(df_traintest['day']>=day_before_public-3)&(df_traintest['place_id']=='China/Hubei')].head()
df_latlong = pd.read_csv(".. /input/smokingstats/df_Latlong.csv")
 df_latlong.head()
def func(x):
try:
x_new = x['Country/Region'] + "/" + x['Province/State']
except:
x_new = x['Country/Region']
return x_new
df_latlong['place_id'] = df_latlong.apply(lambda x: func(x), axis=1)
 df_latlong = df_latlong[df_latlong['place_id'].duplicated() ==False]
df_latlong.head()
df_traintest = pd.merge(df_traintest, df_latlong[['place_id', 'Lat', 'Long']], on='place_id', how='left')
 df_traintest.head()
print(pd.isna(df_traintest['Lat'] ).sum())
 df_traintest[pd.isna(df_traintest['Lat'])].head()
places = np.sort(df_traintest['place_id'].unique())
 print(len(places))
df_traintest2 = copy.deepcopy(df_traintest)
 df_traintest2['cases/day'] = 0
df_traintest2['fatal/day'] = 0
tmp_list = np.zeros(len(df_traintest2))
for place in places:
tmp = df_traintest2['ConfirmedCases'][df_traintest2['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest2['cases/day'][df_traintest2['place_id']==place] = tmp
tmp = df_traintest2['Fatalities'][df_traintest2['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest2['fatal/day'][df_traintest2['place_id']==place] = tmp
print(df_traintest2.shape)
 df_traintest2[df_traintest2['place_id']=='China/Hubei'].head()
def do_aggregation(df, col, mean_range):
df_new = copy.deepcopy(df)
 col_new = '{}_({}-{})'.format(col, mean_range[0], mean_range[1])
 df_new[col_new] = 0
tmp = df_new[col].rolling(mean_range[1]-mean_range[0]+1 ).mean()
df_new[col_new][mean_range[0]:] = tmp[:-(mean_range[0])]
df_new[col_new][pd.isna(df_new[col_new])] = 0
return df_new[[col_new]].reset_index(drop=True)
 def do_aggregations(df):
df = pd.concat([df, do_aggregation(df, 'cases/day', [1,1] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [1,7] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [8,14] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [15,21] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,1] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,7] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [8,14] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [15,21] ).reset_index(drop=True)], axis=1)
 for threshold in [1, 10, 100]:
days_under_threshold =(df['ConfirmedCases']<threshold ).sum()
tmp = df['day'].values - 22 - days_under_threshold
tmp[tmp<=0] = 0
df['days_since_{}cases'.format(threshold)] = tmp
for threshold in [1, 10, 100]:
days_under_threshold =(df['Fatalities']<threshold ).sum()
tmp = df['day'].values - 22 - days_under_threshold
tmp[tmp<=0] = 0
df['days_since_{}fatal'.format(threshold)] = tmp
if df['place_id'][0]=='China/Hubei':
df['days_since_1cases'] += 35
df['days_since_10cases'] += 35-13
df['days_since_100cases'] += 4
df['days_since_1fatal'] += 13
return df
df_traintest3 = []
for place in places[:]:
df_tmp = df_traintest2[df_traintest2['place_id']==place].reset_index(drop=True)
 df_tmp = do_aggregations(df_tmp)
 df_traintest3.append(df_tmp)
 df_traintest3 = pd.concat(df_traintest3 ).reset_index(drop=True)
 df_traintest3[df_traintest3['place_id']=='China/Hubei'].head()
df_smoking = pd.read_csv(".. /input/smokingstats/share-of-adults-who-smoke.csv")
 print(np.sort(df_smoking['Entity'].unique())[:10])
 df_smoking.head()
df_smoking_recent = df_smoking.sort_values('Year', ascending=False ).reset_index(drop=True)
 df_smoking_recent = df_smoking_recent[df_smoking_recent['Entity'].duplicated() ==False]
df_smoking_recent['Country_Region'] = df_smoking_recent['Entity']
df_smoking_recent['SmokingRate'] = df_smoking_recent['Smoking prevalence, total(ages 15+ )(% of adults)']
df_smoking_recent.head()
df_traintest4 = pd.merge(df_traintest3, df_smoking_recent[['Country_Region', 'SmokingRate']], on='Country_Region', how='left')
 print(df_traintest4.shape)
 df_traintest4.head()
SmokingRate = df_smoking_recent['SmokingRate'][df_smoking_recent['Entity']=='World'].values[0]
print("Smoking rate of the world: {:.6f}".format(SmokingRate))
df_traintest4['SmokingRate'][pd.isna(df_traintest4['SmokingRate'])] = SmokingRate
df_traintest4.head()
df_weo = pd.read_csv(".. /input/smokingstats/WEO.csv")
 df_weo.head()
print(df_weo['Subject Descriptor'].unique())
 subs = df_weo['Subject Descriptor'].unique() [:-1]
df_weo_agg = df_weo[['Country']][df_weo['Country'].duplicated() ==False].reset_index(drop=True)
 for sub in subs[:]:
df_tmp = df_weo[['Country', '2019']][df_weo['Subject Descriptor']==sub].reset_index(drop=True)
 df_tmp = df_tmp[df_tmp['Country'].duplicated() ==False].reset_index(drop=True)
 df_tmp.columns = ['Country', sub]
df_weo_agg = df_weo_agg.merge(df_tmp, on='Country', how='left')
 df_weo_agg.columns = ["".join(c if c.isalnum() else "_" for c in str(x)) for x in df_weo_agg.columns]
df_weo_agg.columns
df_weo_agg['Country_Region'] = df_weo_agg['Country']
df_weo_agg.head()
df_traintest5 = pd.merge(df_traintest4, df_weo_agg, on='Country_Region', how='left')
 print(df_traintest5.shape)
 df_traintest5.head()
df_life = pd.read_csv(".. /input/smokingstats/Life expectancy at birth.csv")
 tmp = df_life.iloc[:,1].values.tolist()
df_life = df_life[['Country', '2018']]
def func(x):
x_new = 0
try:
x_new = float(x.replace(",", ""))
except:
x_new = np.nan
return x_new
df_life['2018'] = df_life['2018'].apply(lambda x: func(x))
df_life.head()
df_life = df_life[['Country', '2018']]
df_life.columns = ['Country_Region', 'LifeExpectancy']
df_traintest6 = pd.merge(df_traintest5, df_life, on='Country_Region', how='left')
 print(len(df_traintest6))
df_traintest6.head()
df_country = pd.read_csv(".. /input/countryinfo/covid19countryinfo.csv")
 df_country.head()
df_country['Country_Region'] = df_country['country']
df_country = df_country[df_country['country'].duplicated() ==False]
print(df_country[df_country['country'].duplicated() ].shape)
 df_country[df_country['country'].duplicated() ]
df_traintest7 = pd.merge(df_traintest6,
df_country.drop(['tests', 'testpop', 'country'], axis=1),
on=['Country_Region',], how='left')
 print(df_traintest7.shape)
 df_traintest7.head()
def encode_label(df, col, freq_limit=0):
df[col][pd.isna(df[col])] = 'nan'
tmp = df[col].value_counts()
cols = tmp.index.values
freq = tmp.values
num_cols =(freq>=freq_limit ).sum()
print("col: {}, num_cat: {}, num_reduced: {}".format(col, len(cols), num_cols))
col_new = '{}_le'.format(col)
 df_new = pd.DataFrame(np.ones(len(df), np.int16)*(num_cols-1), columns=[col_new])
 for i, item in enumerate(cols[:num_cols]):
df_new[col_new][df[col]==item] = i
return df_new
def get_df_le(df, col_index, col_cat):
df_new = df[[col_index]]
for col in col_cat:
df_tmp = encode_label(df, col)
 df_new = pd.concat([df_new, df_tmp], axis=1)
 return df_new
df_traintest7['id'] = np.arange(len(df_traintest7))
df_le = get_df_le(df_traintest7, 'id', ['Country_Region', 'Province_State'])
 df_traintest8 = pd.merge(df_traintest7, df_le, on='id', how='left')
 df_traintest8['cases/day'] = df_traintest8['cases/day'].astype(np.float)
 df_traintest8['fatal/day'] = df_traintest8['fatal/day'].astype(np.float)
 def func(x):
x_new = 0
try:
x_new = float(x.replace(",", ""))
except:
x_new = np.nan
return x_new
cols = [
'Gross_domestic_product__constant_prices',
'Gross_domestic_product__current_prices',
'Gross_domestic_product__deflator',
'Gross_domestic_product_per_capita__constant_prices',
'Gross_domestic_product_per_capita__current_prices',
'Output_gap_in_percent_of_potential_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__valuation_of_country_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__per_capita_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__share_of_world_total',
'Implied_PPP_conversion_rate', 'Total_investment',
'Gross_national_savings', 'Inflation__average_consumer_prices',
'Inflation__end_of_period_consumer_prices',
'Six_month_London_interbank_offered_rate__LIBOR_',
'Volume_of_imports_of_goods_and_services',
'Volume_of_Imports_of_goods',
'Volume_of_exports_of_goods_and_services',
'Volume_of_exports_of_goods', 'Unemployment_rate', 'Employment', 'Population',
'General_government_revenue', 'General_government_total_expenditure',
'General_government_net_lending_borrowing', 'General_government_structural_balance',
'General_government_primary_net_lending_borrowing', 'General_government_net_debt',
'General_government_gross_debt', 'Gross_domestic_product_corresponding_to_fiscal_year__current_prices',
'Current_account_balance', 'pop'
]
for col in cols:
df_traintest8[col] = df_traintest8[col].apply(lambda x: func(x))
print(df_traintest8['pop'].dtype)
 df_traintest8[df_traintest8['place_id']=='China/Hubei'].head()
day_before_valid = 104-7
day_before_public = 104
day_before_launch = 104
def calc_score(y_true, y_pred):
y_true[y_true<0] = 0
score = metrics.mean_squared_error(np.log(y_true.clip(0, 1e10)+1), np.log(y_pred[:]+1)) **0.5
return score
SEED = 42
params = {'num_leaves': 8,
'min_data_in_leaf': 5,
'objective': 'regression',
'max_depth': 8,
'learning_rate': 0.02,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8201,
'bagging_seed': SEED,
'reg_alpha': 1,
'reg_lambda': 4.9847051755586085,
'random_state': SEED,
'metric': 'mse',
'verbosity': 100,
'min_gain_to_split': 0.02,
'min_child_weight': 5,
'num_threads': 6,
}
col_target = 'fatal/day'
col_var = [
'Lat', 'Long',
'cases/day_(1-1)',
'cases/day_(1-7)',
'fatal/day_(1-7)',
'fatal/day_(8-14)',
'fatal/day_(15-21)',
'SmokingRate',
'density',
]
col_cat = []
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_valid)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(day_before_valid<df_traintest8['day'])&(df_traintest8['day']<=day_before_public)]
df_test = df_traintest8[pd.isna(df_traintest8['ForecastId'])==False]
X_train = df_train[col_var]
X_valid = df_valid[col_var]
y_train = np.log(df_train[col_target].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 num_round = 15000
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 best_itr = model.best_iteration
y_true = df_valid['fatal/day'].values
y_pred = np.exp(model.predict(X_valid)) -1
score = calc_score(y_true, y_pred)
 print("{:.6f}".format(score))
tmp = pd.DataFrame()
tmp["feature"] = col_var
tmp["importance"] = model.feature_importance()
tmp = tmp.sort_values('importance', ascending=False)
 tmp
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_test = df_traintest8[pd.isna(df_traintest8['ForecastId'])==False]
X_train = df_train[col_var]
X_valid = df_valid[col_var]
y_train = np.log(df_train[col_target].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model = lgb.train(params, train_data, best_itr, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 col_target2 = 'cases/day'
col_var2 = [
'Lat', 'Long',
'days_since_10cases',
'cases/day_(1-1)',
'cases/day_(1-7)',
'cases/day_(8-14)',
'cases/day_(15-21)',
]
col_cat = []
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_valid)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(day_before_valid<df_traintest8['day'])&(df_traintest8['day']<=day_before_public)]
X_train = df_train[col_var2]
X_valid = df_valid[col_var2]
y_train = np.log(df_train[col_target2].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target2].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 best_itr2 = model2.best_iteration
y_true = df_valid['cases/day'].values
y_pred = np.exp(model2.predict(X_valid)) -1
score = calc_score(y_true, y_pred)
 print("{:.6f}".format(score))
tmp = pd.DataFrame()
tmp["feature"] = col_var2
tmp["importance"] = model2.feature_importance()
tmp = tmp.sort_values('importance', ascending=False)
 tmp
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
X_train = df_train[col_var2]
X_valid = df_valid[col_var2]
y_train = np.log(df_train[col_target2].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target2].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model2 = lgb.train(params, train_data, best_itr2, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(day_before_public<df_traintest8['day'])]
df_test = df_traintest8[pd.isna(df_traintest8['ForecastId'])==False]
X_train = df_train[col_var]
X_valid = df_valid[col_var]
y_train = np.log(df_train[col_target].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 num_round = 15000
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 best_itr = model.best_iteration
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) ]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) ]
X_train = df_train[col_var]
X_valid = df_valid[col_var]
y_train = np.log(df_train[col_target].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model_pri = lgb.train(params, train_data, best_itr, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(day_before_public<df_traintest8['day'])]
X_train = df_train[col_var2]
X_valid = df_valid[col_var2]
y_train = np.log(df_train[col_target2].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target2].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 best_itr2 = model2.best_iteration
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
X_train = df_train[col_var2]
X_valid = df_valid[col_var2]
y_train = np.log(df_train[col_target2].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target2].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model2_pri = lgb.train(params, train_data, best_itr2, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 df_tmp = df_traintest8[ 
(( df_traintest8['day']<=day_before_public)&(pd.isna(df_traintest8['ForecastId'])))
 |(( day_before_public<df_traintest8['day'])&(pd.isna(df_traintest8['ForecastId'])==False)) ].reset_index(drop=True)
 df_tmp = df_tmp.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_traintest9 = []
for i, place in enumerate(places[:]):
df_tmp2 = df_tmp[df_tmp['place_id']==place].reset_index(drop=True)
 df_tmp2 = do_aggregations(df_tmp2)
 df_traintest9.append(df_tmp2)
 df_traintest9 = pd.concat(df_traintest9 ).reset_index(drop=True)
 df_traintest9[df_traintest9['day']>day_before_public-2].head()
df_tmp = df_traintest8[ 
(( df_traintest8['day']<=day_before_private)&(pd.isna(df_traintest8['ForecastId'])))
 |(( day_before_private<df_traintest8['day'])&(pd.isna(df_traintest8['ForecastId'])==False)) ].reset_index(drop=True)
 df_tmp = df_tmp.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_traintest10 = []
for i, place in enumerate(places[:]):
df_tmp2 = df_tmp[df_tmp['place_id']==place].reset_index(drop=True)
 df_tmp2 = do_aggregations(df_tmp2)
 df_traintest10.append(df_tmp2)
 df_traintest10 = pd.concat(df_traintest10 ).reset_index(drop=True)
 df_traintest10[df_traintest10['day']>day_before_private-2].head()
df_preds = []
for i, place in enumerate(places[:]):
df_interest = copy.deepcopy(df_traintest9[df_traintest9['place_id']==place].reset_index(drop=True))
df_interest['cases/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
df_interest['fatal/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
len_known =(df_interest['day']<=day_before_public ).sum()
len_unknown =(day_before_public<df_interest['day'] ).sum()
for j in range(len_unknown):
X_valid = df_interest[col_var].iloc[j+len_known]
X_valid2 = df_interest[col_var2].iloc[j+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 pred_c =(np.exp(pred_c)-1 ).clip(0, 1e10)
 pred_f =(np.exp(pred_f)-1 ).clip(0, 1e10)
 df_interest['fatal/day'][j+len_known] = pred_f
df_interest['cases/day'][j+len_known] = pred_c
df_interest['Fatalities'][j+len_known] = df_interest['Fatalities'][j+len_known-1] + pred_f
df_interest['ConfirmedCases'][j+len_known] = df_interest['ConfirmedCases'][j+len_known-1] + pred_c
df_interest = df_interest.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_interest = do_aggregations(df_interest)
 if(i+1)%10==0:
print("{:3d}/{} {}, len known: {}, len unknown: {}".format(i+1, len(places), place, len_known, len_unknown), df_interest.shape)
 df_interest['fatal_pred'] = np.cumsum(df_interest['fatal/day'].values)
 df_interest['cases_pred'] = np.cumsum(df_interest['cases/day'].values)
 df_preds.append(df_interest)
 df_preds = pd.concat(df_preds)
 df_preds_pri = []
for i, place in enumerate(places[:]):
df_interest = copy.deepcopy(df_traintest10[df_traintest10['place_id']==place].reset_index(drop=True))
df_interest['cases/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
df_interest['fatal/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
len_known =(df_interest['day']<=day_before_private ).sum()
len_unknown =(day_before_private<df_interest['day'] ).sum()
for j in range(len_unknown):
X_valid = df_interest[col_var].iloc[j+len_known]
X_valid2 = df_interest[col_var2].iloc[j+len_known]
pred_f = model_pri.predict(X_valid)
 pred_c = model2_pri.predict(X_valid2)
 pred_c =(np.exp(pred_c)-1 ).clip(0, 1e10)
 pred_f =(np.exp(pred_f)-1 ).clip(0, 1e10)
 df_interest['fatal/day'][j+len_known] = pred_f
df_interest['cases/day'][j+len_known] = pred_c
df_interest['Fatalities'][j+len_known] = df_interest['Fatalities'][j+len_known-1] + pred_f
df_interest['ConfirmedCases'][j+len_known] = df_interest['ConfirmedCases'][j+len_known-1] + pred_c
df_interest = df_interest.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_interest = do_aggregations(df_interest)
 if(i+1)%10==0:
print("{:3d}/{} {}, len known: {}, len unknown: {}".format(i+1, len(places), place, len_known, len_unknown), df_interest.shape)
 df_interest['fatal_pred'] = np.cumsum(df_interest['fatal/day'].values)
 df_interest['cases_pred'] = np.cumsum(df_interest['cases/day'].values)
 df_preds_pri.append(df_interest)
 df_preds_pri = pd.concat(df_preds_pri)
 places_sort = df_traintest10[['place_id', 'ConfirmedCases']][df_traintest10['day']==day_before_private]
places_sort = places_sort.sort_values('ConfirmedCases', ascending=False ).reset_index(drop=True)['place_id'].values
print(len(places_sort))
places_sort[:5]
print("Fatalities / Public")
 plt.figure(figsize=(30,30))
for i in range(30):
plt.subplot(5,6,i+1)
 idx = i * 10
df_interest = df_preds[df_preds['place_id']==places_sort[idx]].reset_index(drop=True)
 tmp = df_interest['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest['day'], y=tmp, label='pred')
 df_interest2 = df_traintest10[(df_traintest10['place_id']==places_sort[idx])&(df_traintest10['day']<=day_before_private)].reset_index(drop=True)
 sns.lineplot(x=df_interest2['day'].values, y=df_interest2['Fatalities'].values, label='true')
 plt.title(places_sort[idx])
 plt.show()
print("Confirmed Cases / Public")
 plt.figure(figsize=(30,30))
for i in range(30):
plt.subplot(5,6,i+1)
 idx = i * 10
df_interest = df_preds[df_preds['place_id']==places_sort[idx]].reset_index(drop=True)
 tmp = df_interest['cases/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest['day'], y=tmp, label='pred')
 df_interest2 = df_traintest10[(df_traintest10['place_id']==places_sort[idx])&(df_traintest10['day']<=day_before_private)].reset_index(drop=True)
 sns.lineplot(x=df_interest2['day'].values, y=df_interest2['ConfirmedCases'].values, label='true')
 plt.title(places_sort[idx])
 plt.show()
print("Fatalities / Private")
 plt.figure(figsize=(30,30))
for i in range(30):
plt.subplot(5,6,i+1)
 idx = i * 10
df_interest = df_preds_pri[df_preds_pri['place_id']==places_sort[idx]].reset_index(drop=True)
 tmp = df_interest['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest['day'], y=tmp, label='pred')
 df_interest2 = df_traintest10[(df_traintest10['place_id']==places_sort[idx])&(df_traintest10['day']<=day_before_private)].reset_index(drop=True)
 sns.lineplot(x=df_interest2['day'].values, y=df_interest2['Fatalities'].values, label='true')
 plt.title(places_sort[idx])
 plt.show()
print("ConfirmedCases / Private")
 plt.figure(figsize=(30,30))
for i in range(30):
plt.subplot(5,6,i+1)
 idx = i * 10
df_interest = df_preds_pri[df_preds_pri['place_id']==places_sort[idx]].reset_index(drop=True)
 tmp = df_interest['cases/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest['day'], y=tmp, label='pred')
 df_interest2 = df_traintest10[(df_traintest10['place_id']==places_sort[idx])&(df_traintest10['day']<=day_before_private)].reset_index(drop=True)
 sns.lineplot(x=df_interest2['day'].values, y=df_interest2['ConfirmedCases'].values, label='true')
 plt.title(places_sort[idx])
 plt.show()
df_preds[df_preds['day']>day_before_private] = df_preds_pri[df_preds['day']>day_before_private]
df_preds.to_csv("df_preds.csv", index=None)
 df_sub = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv")
 print(len(df_sub))
df_sub.head()
df_sub = pd.merge(df_sub, df_traintest3[['ForecastId', 'place_id', 'day']])
 df_sub = pd.merge(df_sub, df_preds[['place_id', 'day', 'cases_pred', 'fatal_pred']], on=['place_id', 'day',], how='left')
 df_sub.head(10)
 df_sub['ConfirmedCases'] = df_sub['cases_pred']
df_sub['Fatalities'] = df_sub['fatal_pred']
df_sub = df_sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
df_sub2 = df_sub.copy()
df_sub.to_csv("submission5.csv", index=None)
 df_sub.head(10)
import numpy as np
import pandas as pd
from datetime import datetime<feature_engineering>
gc.collect()
init_time = datetime.now()<define_variables>
pd.set_option('display.max_columns', 100)
 warnings.filterwarnings('ignore')
 df_train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 print(df_train.shape)
 df_train.head()
df_test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 print(df_test.shape)
 df_test.head()
df_traintest = pd.concat([df_train, df_test])
 print(df_train.shape, df_test.shape, df_traintest.shape)
 df_traintest['Date'] = pd.to_datetime(df_traintest['Date'])
 df_traintest['day'] = df_traintest['Date'].apply(lambda x: x.dayofyear ).astype(np.int16)
 df_traintest.head()
day_before_valid = 104-7
day_before_public = 104
day_before_private = df_traintest['day'][pd.isna(df_traintest['ForecastId'])].max()
print(df_traintest['Date'][df_traintest['day']==day_before_valid].values[0])
 print(df_traintest['Date'][df_traintest['day']==day_before_public].values[0])
 print(df_traintest['Date'][df_traintest['day']==day_before_private].values[0])
 def func(x):
try:
x_new = x['Country_Region'] + "/" + x['Province_State']
except:
x_new = x['Country_Region']
return x_new
df_traintest['place_id'] = df_traintest.apply(lambda x: func(x), axis=1)
 df_traintest.head()
df_traintest[(df_traintest['day']>=day_before_public-3)&(df_traintest['place_id']=='China/Hubei')].head()
df_latlong = pd.read_csv(".. /input/smokingstats/df_Latlong.csv")
 df_latlong.head()
def func(x):
try:
x_new = x['Country/Region'] + "/" + x['Province/State']
except:
x_new = x['Country/Region']
return x_new
df_latlong['place_id'] = df_latlong.apply(lambda x: func(x), axis=1)
 df_latlong = df_latlong[df_latlong['place_id'].duplicated() ==False]
df_latlong.head()
df_traintest = pd.merge(df_traintest, df_latlong[['place_id', 'Lat', 'Long']], on='place_id', how='left')
 df_traintest.head()
tmp = np.sort(df_traintest['place_id'][pd.isna(df_traintest['Lat'])].unique())
 print(len(tmp))
tmp
places = np.sort(df_traintest['place_id'].unique())
 print(len(places))
df_traintest2 = copy.deepcopy(df_traintest)
 df_traintest2['cases/day'] = 0
df_traintest2['fatal/day'] = 0
tmp_list = np.zeros(len(df_traintest2))
for place in places:
tmp = df_traintest2['ConfirmedCases'][df_traintest2['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest2['cases/day'][df_traintest2['place_id']==place] = tmp
tmp = df_traintest2['Fatalities'][df_traintest2['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest2['fatal/day'][df_traintest2['place_id']==place] = tmp
print(df_traintest2.shape)
 df_traintest2[df_traintest2['place_id']=='China/Hubei'].head()
def do_aggregation(df, col, mean_range):
df_new = copy.deepcopy(df)
 col_new = '{}_({}-{})'.format(col, mean_range[0], mean_range[1])
 df_new[col_new] = 0
tmp = df_new[col].rolling(mean_range[1]-mean_range[0]+1 ).mean()
df_new[col_new][mean_range[0]:] = tmp[:-(mean_range[0])]
df_new[col_new][pd.isna(df_new[col_new])] = 0
return df_new[[col_new]].reset_index(drop=True)
 def do_aggregations(df):
df = pd.concat([df, do_aggregation(df, 'cases/day', [1,1] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [1,7] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [8,14] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [15,21] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,1] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,7] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [8,14] ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [15,21] ).reset_index(drop=True)], axis=1)
 for threshold in [1, 10, 100]:
days_under_threshold =(df['ConfirmedCases']<threshold ).sum()
tmp = df['day'].values - 22 - days_under_threshold
tmp[tmp<=0] = 0
df['days_since_{}cases'.format(threshold)] = tmp
for threshold in [1, 10, 100]:
days_under_threshold =(df['Fatalities']<threshold ).sum()
tmp = df['day'].values - 22 - days_under_threshold
tmp[tmp<=0] = 0
df['days_since_{}fatal'.format(threshold)] = tmp
if df['place_id'][0]=='China/Hubei':
df['days_since_1cases'] += 35
df['days_since_10cases'] += 35-13
df['days_since_100cases'] += 4
df['days_since_1fatal'] += 13
return df
df_traintest3 = []
for place in places[:]:
df_tmp = df_traintest2[df_traintest2['place_id']==place].reset_index(drop=True)
 df_tmp = do_aggregations(df_tmp)
 df_traintest3.append(df_tmp)
 df_traintest3 = pd.concat(df_traintest3 ).reset_index(drop=True)
 df_traintest3[df_traintest3['place_id']=='China/Hubei'].head()
df_smoking = pd.read_csv(".. /input/smokingstats/share-of-adults-who-smoke.csv")
 print(np.sort(df_smoking['Entity'].unique())[:10])
 df_smoking.head()
df_smoking_recent = df_smoking.sort_values('Year', ascending=False ).reset_index(drop=True)
 df_smoking_recent = df_smoking_recent[df_smoking_recent['Entity'].duplicated() ==False]
df_smoking_recent['Country_Region'] = df_smoking_recent['Entity']
df_smoking_recent['SmokingRate'] = df_smoking_recent['Smoking prevalence, total(ages 15+ )(% of adults)']
df_smoking_recent.head()
df_traintest4 = pd.merge(df_traintest3, df_smoking_recent[['Country_Region', 'SmokingRate']], on='Country_Region', how='left')
 print(df_traintest4.shape)
 df_traintest4.head()
SmokingRate = df_smoking_recent['SmokingRate'][df_smoking_recent['Entity']=='World'].values[0]
print("Smoking rate of the world: {:.6f}".format(SmokingRate))
df_traintest4['SmokingRate'][pd.isna(df_traintest4['SmokingRate'])] = SmokingRate
df_traintest4.head()
df_weo = pd.read_csv(".. /input/smokingstats/WEO.csv")
 df_weo.head()
print(df_weo['Subject Descriptor'].unique())
 subs = df_weo['Subject Descriptor'].unique() [:-1]
df_weo_agg = df_weo[['Country']][df_weo['Country'].duplicated() ==False].reset_index(drop=True)
 for sub in subs[:]:
df_tmp = df_weo[['Country', '2019']][df_weo['Subject Descriptor']==sub].reset_index(drop=True)
 df_tmp = df_tmp[df_tmp['Country'].duplicated() ==False].reset_index(drop=True)
 df_tmp.columns = ['Country', sub]
df_weo_agg = df_weo_agg.merge(df_tmp, on='Country', how='left')
 df_weo_agg.columns = ["".join(c if c.isalnum() else "_" for c in str(x)) for x in df_weo_agg.columns]
df_weo_agg.columns
df_weo_agg['Country_Region'] = df_weo_agg['Country']
df_weo_agg.head()
df_traintest5 = pd.merge(df_traintest4, df_weo_agg, on='Country_Region', how='left')
 print(df_traintest5.shape)
 df_traintest5.head()
df_life = pd.read_csv(".. /input/smokingstats/Life expectancy at birth.csv")
 tmp = df_life.iloc[:,1].values.tolist()
df_life = df_life[['Country', '2018']]
def func(x):
x_new = 0
try:
x_new = float(x.replace(",", ""))
except:
x_new = np.nan
return x_new
df_life['2018'] = df_life['2018'].apply(lambda x: func(x))
df_life.head()
df_life = df_life[['Country', '2018']]
df_life.columns = ['Country_Region', 'LifeExpectancy']
df_traintest6 = pd.merge(df_traintest5, df_life, on='Country_Region', how='left')
 print(len(df_traintest6))
df_traintest6.head()
df_country = pd.read_csv(".. /input/countryinfo/covid19countryinfo.csv")
 df_country.head()
df_country['Country_Region'] = df_country['country']
df_country = df_country[df_country['country'].duplicated() ==False]
print(df_country[df_country['country'].duplicated() ].shape)
 df_country[df_country['country'].duplicated() ]
df_traintest7 = pd.merge(df_traintest6,
df_country.drop(['tests', 'testpop', 'country'], axis=1),
on=['Country_Region',], how='left')
 print(df_traintest7.shape)
 df_traintest7.head()
def encode_label(df, col, freq_limit=0):
df[col][pd.isna(df[col])] = 'nan'
tmp = df[col].value_counts()
cols = tmp.index.values
freq = tmp.values
num_cols =(freq>=freq_limit ).sum()
print("col: {}, num_cat: {}, num_reduced: {}".format(col, len(cols), num_cols))
col_new = '{}_le'.format(col)
 df_new = pd.DataFrame(np.ones(len(df), np.int16)*(num_cols-1), columns=[col_new])
 for i, item in enumerate(cols[:num_cols]):
df_new[col_new][df[col]==item] = i
return df_new
def get_df_le(df, col_index, col_cat):
df_new = df[[col_index]]
for col in col_cat:
df_tmp = encode_label(df, col)
 df_new = pd.concat([df_new, df_tmp], axis=1)
 return df_new
df_traintest7['id'] = np.arange(len(df_traintest7))
df_le = get_df_le(df_traintest7, 'id', ['Country_Region', 'Province_State'])
 df_traintest8 = pd.merge(df_traintest7, df_le, on='id', how='left')
 df_traintest8['cases/day'] = df_traintest8['cases/day'].astype(np.float)
 df_traintest8['fatal/day'] = df_traintest8['fatal/day'].astype(np.float)
 def func(x):
x_new = 0
try:
x_new = float(x.replace(",", ""))
except:
x_new = np.nan
return x_new
cols = [
'Gross_domestic_product__constant_prices',
'Gross_domestic_product__current_prices',
'Gross_domestic_product__deflator',
'Gross_domestic_product_per_capita__constant_prices',
'Gross_domestic_product_per_capita__current_prices',
'Output_gap_in_percent_of_potential_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__valuation_of_country_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__per_capita_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__share_of_world_total',
'Implied_PPP_conversion_rate', 'Total_investment',
'Gross_national_savings', 'Inflation__average_consumer_prices',
'Inflation__end_of_period_consumer_prices',
'Six_month_London_interbank_offered_rate__LIBOR_',
'Volume_of_imports_of_goods_and_services',
'Volume_of_Imports_of_goods',
'Volume_of_exports_of_goods_and_services',
'Volume_of_exports_of_goods', 'Unemployment_rate', 'Employment', 'Population',
'General_government_revenue', 'General_government_total_expenditure',
'General_government_net_lending_borrowing', 'General_government_structural_balance',
'General_government_primary_net_lending_borrowing', 'General_government_net_debt',
'General_government_gross_debt', 'Gross_domestic_product_corresponding_to_fiscal_year__current_prices',
'Current_account_balance', 'pop'
]
for col in cols:
df_traintest8[col] = df_traintest8[col].apply(lambda x: func(x))
print(df_traintest8['pop'].dtype)
 df_traintest8[df_traintest8['place_id']=='China/Hubei'].head()
def calc_score(y_true, y_pred):
y_true[y_true<0] = 0
score = metrics.mean_squared_error(np.log(y_true.clip(0, 1e10)+1), np.log(y_pred[:]+1)) **0.5
return score
SEED = 42
params = {'num_leaves': 8,
'min_data_in_leaf': 5,
'objective': 'regression',
'max_depth': 8,
'learning_rate': 0.02,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8201,
'bagging_seed': SEED,
'reg_alpha': 1,
'reg_lambda': 4.9847051755586085,
'random_state': SEED,
'metric': 'mse',
'verbosity': 100,
'min_gain_to_split': 0.02,
'min_child_weight': 5,
'num_threads': 6,
}
col_target = 'fatal/day'
col_var = [
'Lat', 'Long',
'cases/day_(1-1)',
'cases/day_(1-7)',
'fatal/day_(1-7)',
'fatal/day_(8-14)',
'fatal/day_(15-21)',
'SmokingRate',
'density',
]
col_cat = []
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_valid)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(day_before_valid<df_traintest8['day'])&(df_traintest8['day']<=day_before_public)]
df_test = df_traintest8[pd.isna(df_traintest8['ForecastId'])==False]
X_train = df_train[col_var]
X_valid = df_valid[col_var]
y_train = np.log(df_train[col_target].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 num_round = 30000
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=300,)
 best_itr = model.best_iteration
y_true = df_valid['fatal/day'].values
y_pred = np.exp(model.predict(X_valid)) -1
score = calc_score(y_true, y_pred)
 print("{:.6f}".format(score))
tmp = pd.DataFrame()
tmp["feature"] = col_var
tmp["importance"] = model.feature_importance()
tmp = tmp.sort_values('importance', ascending=False)
 tmp
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_test = df_traintest8[pd.isna(df_traintest8['ForecastId'])==False]
X_train = df_train[col_var]
X_valid = df_valid[col_var]
y_train = np.log(df_train[col_target].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model = lgb.train(params, train_data, best_itr, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=300,)
 col_target2 = 'cases/day'
col_var2 = [
'Lat', 'Long',
'days_since_10cases',
'cases/day_(1-1)',
'cases/day_(1-7)',
'cases/day_(8-14)',
'cases/day_(15-21)',
]
col_cat = []
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_valid)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(day_before_valid<df_traintest8['day'])&(df_traintest8['day']<=day_before_public)]
X_train = df_train[col_var2]
X_valid = df_valid[col_var2]
y_train = np.log(df_train[col_target2].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target2].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=300,)
 best_itr2 = model2.best_iteration
y_true = df_valid['cases/day'].values
y_pred = np.exp(model2.predict(X_valid)) -1
score = calc_score(y_true, y_pred)
 print("{:.6f}".format(score))
tmp = pd.DataFrame()
tmp["feature"] = col_var2
tmp["importance"] = model2.feature_importance()
tmp = tmp.sort_values('importance', ascending=False)
 tmp
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
X_train = df_train[col_var2]
X_valid = df_valid[col_var2]
y_train = np.log(df_train[col_target2].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target2].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model2 = lgb.train(params, train_data, best_itr2, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=300,)
 df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(day_before_public<df_traintest8['day'])]
df_test = df_traintest8[pd.isna(df_traintest8['ForecastId'])==False]
X_train = df_train[col_var]
X_valid = df_valid[col_var]
y_train = np.log(df_train[col_target].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 num_round = 30000
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=300,)
 best_itr = model.best_iteration
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) ]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) ]
X_train = df_train[col_var]
X_valid = df_valid[col_var]
y_train = np.log(df_train[col_target].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model_pri = lgb.train(params, train_data, best_itr, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=300,)
 df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(day_before_public<df_traintest8['day'])]
X_train = df_train[col_var2]
X_valid = df_valid[col_var2]
y_train = np.log(df_train[col_target2].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target2].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=300,)
 best_itr2 = model2.best_iteration
df_train = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
df_valid = df_traintest8[(pd.isna(df_traintest8['ForecastId'])) &(df_traintest8['day']<=day_before_public)]
X_train = df_train[col_var2]
X_valid = df_valid[col_var2]
y_train = np.log(df_train[col_target2].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target2].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model2_pri = lgb.train(params, train_data, best_itr2, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=300,)
 df_tmp = df_traintest8[ 
(( df_traintest8['day']<=day_before_public)&(pd.isna(df_traintest8['ForecastId'])))
 |(( day_before_public<df_traintest8['day'])&(pd.isna(df_traintest8['ForecastId'])==False)) ].reset_index(drop=True)
 df_tmp = df_tmp.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_traintest9 = []
for i, place in enumerate(places[:]):
df_tmp2 = df_tmp[df_tmp['place_id']==place].reset_index(drop=True)
 df_tmp2 = do_aggregations(df_tmp2)
 df_traintest9.append(df_tmp2)
 df_traintest9 = pd.concat(df_traintest9 ).reset_index(drop=True)
 df_traintest9[df_traintest9['day']>day_before_public-2].head()
df_tmp = df_traintest8[ 
(( df_traintest8['day']<=day_before_private)&(pd.isna(df_traintest8['ForecastId'])))
 |(( day_before_private<df_traintest8['day'])&(pd.isna(df_traintest8['ForecastId'])==False)) ].reset_index(drop=True)
 df_tmp = df_tmp.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_traintest10 = []
for i, place in enumerate(places[:]):
df_tmp2 = df_tmp[df_tmp['place_id']==place].reset_index(drop=True)
 df_tmp2 = do_aggregations(df_tmp2)
 df_traintest10.append(df_tmp2)
 df_traintest10 = pd.concat(df_traintest10 ).reset_index(drop=True)
 df_traintest10[df_traintest10['day']>day_before_private-2].head()
df_preds = []
for i, place in enumerate(places[:]):
df_interest = copy.deepcopy(df_traintest9[df_traintest9['place_id']==place].reset_index(drop=True))
df_interest['cases/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
df_interest['fatal/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
len_known =(df_interest['day']<=day_before_public ).sum()
len_unknown =(day_before_public<df_interest['day'] ).sum()
for j in range(len_unknown):
X_valid = df_interest[col_var].iloc[j+len_known]
X_valid2 = df_interest[col_var2].iloc[j+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 pred_c =(np.exp(pred_c)-1 ).clip(0, 1e10)
 pred_f =(np.exp(pred_f)-1 ).clip(0, 1e10)
 df_interest['fatal/day'][j+len_known] = pred_f
df_interest['cases/day'][j+len_known] = pred_c
df_interest['Fatalities'][j+len_known] = df_interest['Fatalities'][j+len_known-1] + pred_f
df_interest['ConfirmedCases'][j+len_known] = df_interest['ConfirmedCases'][j+len_known-1] + pred_c
df_interest = df_interest.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_interest = do_aggregations(df_interest)
 if(i+1)%10==0:
print("{:3d}/{} {}, len known: {}, len unknown: {}".format(i+1, len(places), place, len_known, len_unknown), df_interest.shape)
 df_interest['fatal_pred'] = np.cumsum(df_interest['fatal/day'].values)
 df_interest['cases_pred'] = np.cumsum(df_interest['cases/day'].values)
 df_preds.append(df_interest)
 df_preds = pd.concat(df_preds)
 df_preds_pri = []
for i, place in enumerate(places[:]):
df_interest = copy.deepcopy(df_traintest10[df_traintest10['place_id']==place].reset_index(drop=True))
df_interest['cases/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
df_interest['fatal/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
len_known =(df_interest['day']<=day_before_private ).sum()
len_unknown =(day_before_private<df_interest['day'] ).sum()
for j in range(len_unknown):
X_valid = df_interest[col_var].iloc[j+len_known]
X_valid2 = df_interest[col_var2].iloc[j+len_known]
pred_f = model_pri.predict(X_valid)
 pred_c = model2_pri.predict(X_valid2)
 pred_c =(np.exp(pred_c)-1 ).clip(0, 1e10)
 pred_f =(np.exp(pred_f)-1 ).clip(0, 1e10)
 df_interest['fatal/day'][j+len_known] = pred_f
df_interest['cases/day'][j+len_known] = pred_c
df_interest['Fatalities'][j+len_known] = df_interest['Fatalities'][j+len_known-1] + pred_f
df_interest['ConfirmedCases'][j+len_known] = df_interest['ConfirmedCases'][j+len_known-1] + pred_c
df_interest = df_interest.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_interest = do_aggregations(df_interest)
 if(i+1)%10==0:
print("{:3d}/{} {}, len known: {}, len unknown: {}".format(i+1, len(places), place, len_known, len_unknown), df_interest.shape)
 df_interest['fatal_pred'] = np.cumsum(df_interest['fatal/day'].values)
 df_interest['cases_pred'] = np.cumsum(df_interest['cases/day'].values)
 df_preds_pri.append(df_interest)
 df_preds_pri = pd.concat(df_preds_pri)
 places_sort = df_traintest10[['place_id', 'ConfirmedCases']][df_traintest10['day']==day_before_private]
places_sort = places_sort.sort_values('ConfirmedCases', ascending=False ).reset_index(drop=True)['place_id'].values
print(len(places_sort))
places_sort[:5]
print("Fatalities / Public")
 plt.figure(figsize=(30,30))
for i in range(30):
plt.subplot(5,6,i+1)
 idx = i * 10
df_interest = df_preds[df_preds['place_id']==places_sort[idx]].reset_index(drop=True)
 tmp = df_interest['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest['day'], y=tmp, label='pred')
 df_interest2 = df_traintest10[(df_traintest10['place_id']==places_sort[idx])&(df_traintest10['day']<=day_before_private)].reset_index(drop=True)
 sns.lineplot(x=df_interest2['day'].values, y=df_interest2['Fatalities'].values, label='true')
 plt.title(places_sort[idx])
 plt.show()
print("Confirmed Cases / Public")
 plt.figure(figsize=(30,30))
for i in range(30):
plt.subplot(5,6,i+1)
 idx = i * 10
df_interest = df_preds[df_preds['place_id']==places_sort[idx]].reset_index(drop=True)
 tmp = df_interest['cases/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest['day'], y=tmp, label='pred')
 df_interest2 = df_traintest10[(df_traintest10['place_id']==places_sort[idx])&(df_traintest10['day']<=day_before_private)].reset_index(drop=True)
 sns.lineplot(x=df_interest2['day'].values, y=df_interest2['ConfirmedCases'].values, label='true')
 plt.title(places_sort[idx])
 plt.show()
print("Fatalities / Private")
 plt.figure(figsize=(30,30))
for i in range(30):
plt.subplot(5,6,i+1)
 idx = i * 10
df_interest = df_preds_pri[df_preds_pri['place_id']==places_sort[idx]].reset_index(drop=True)
 tmp = df_interest['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest['day'], y=tmp, label='pred')
 df_interest2 = df_traintest10[(df_traintest10['place_id']==places_sort[idx])&(df_traintest10['day']<=day_before_private)].reset_index(drop=True)
 sns.lineplot(x=df_interest2['day'].values, y=df_interest2['Fatalities'].values, label='true')
 plt.title(places_sort[idx])
 plt.show()
print("ConfirmedCases / Private")
 plt.figure(figsize=(30,30))
for i in range(30):
plt.subplot(5,6,i+1)
 idx = i * 10
df_interest = df_preds_pri[df_preds_pri['place_id']==places_sort[idx]].reset_index(drop=True)
 tmp = df_interest['cases/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest['day'], y=tmp, label='pred')
 df_interest2 = df_traintest10[(df_traintest10['place_id']==places_sort[idx])&(df_traintest10['day']<=day_before_private)].reset_index(drop=True)
 sns.lineplot(x=df_interest2['day'].values, y=df_interest2['ConfirmedCases'].values, label='true')
 plt.title(places_sort[idx])
 plt.show()
df_preds[df_preds['day']>day_before_private] = df_preds_pri[df_preds['day']>day_before_private]
df_preds.to_csv("df_preds.csv", index=None)
 df_sub = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv")
 print(len(df_sub))
df_sub.head()
df_sub = pd.merge(df_sub, df_traintest3[['ForecastId', 'place_id', 'day']])
 df_sub = pd.merge(df_sub, df_preds[['place_id', 'day', 'cases_pred', 'fatal_pred']], on=['place_id', 'day',], how='left')
 df_sub.head(10)
 df_sub['ConfirmedCases'] = df_sub['cases_pred']
df_sub['Fatalities'] = df_sub['fatal_pred']
df_sub = df_sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
df_sub1 = df_sub.copy()
df_sub.to_csv("submission1.csv", index=None)
 df_sub.head(10 )
IS_DEV = True
TREAT_OUTLIERS = False
if(IS_DEV == True):
TRAIN_FILE = '.. /input/train.csv'
TEST_FILE = '.. /input/test.csv'
COMPRESSION = None
SKIP_ROWS = range(1, 170000000)
 else:
TRAIN_FILE = '.. /input/train.csv.zip'
TEST_FILE = '.. /input/test.csv.zip'
COMPRESSION = 'zip'
TRAIN_SAMPLE_FILE = '.. /input/' + 'train_sample.csv'
LEARNING_RATE_INIT = 0.001
LEARNING_RATE_END = 0.0001
BATCH_SIZE = 20000
EPOCHS = 2
EMBEDDING_N = 50
DENSE_N = 1024
SPATIAL_DROPOUT_1D = 0.2
DROPOUT_1 = 0.2
DROPOUT_2 = 0.2<define_variables>
gc.collect()
dtypes = {
'ip': 'uint32',
'app': 'uint16',
'device': 'uint16',
'os': 'uint16',
'channel': 'uint16',
'is_attributed': 'uint8',
'click_id': 'uint32'
}<load_from_csv>
BAGS = 25
SEED = 1234
SET_FRAC = 0.01
TRUNCATED = False
DROPS = True
PRIVATE = True
USE_PRIORS = False
SUP_DROP = 0.0
ACTIONS_DROP = 0.0
PLACE_FRACTION = 1.0
LT_DECAY_MAX = 0.3
LT_DECAY_MIN = -0.4
SINGLE_MODEL = False
MODEL_Y = 'agg_dff'
pd.options.display.float_format = '{:.8}'.format
plt.rcParams["figure.figsize"] =(12, 4.75)
 InteractiveShell.ast_node_interactivity = "all"
pd.options.display.max_rows = 999
path = '.. /input/david1013orgdataset'
input_path = '/kaggle/input/covid19-global-forecasting-week-4'
train = pd.read_csv(input_path + '/train.csv')
 test = pd.read_csv(input_path + '/test.csv')
 sub = pd.read_csv(input_path + '/submission.csv')
 tt = pd.merge(train, test, on=['Country_Region', 'Province_State', 'Date'],
how='right', validate="1:1")\ 
.fillna(method = 'ffill')
 public = tt[['ForecastId', 'ConfirmedCases', 'Fatalities']]
train.Date.max()
test_dates = test.Date.unique()
test_dates
pp = 'public'
if PRIVATE:
test = test[ pd.to_datetime(test.Date)> train.Date.max() ]
pp = 'private'
test.Date.unique()
revised = pd.read_csv(path +
'/covid19_train_data_us_states_before_march_09_new.csv')
 revised = revised[['Province_State', 'Country_Region', 'Date', 'ConfirmedCases', 'Fatalities']]
train.tail()
revised.head()
train.Date = pd.to_datetime(train.Date)
 revised.Date = pd.to_datetime(revised.Date)
 rev_train = pd.merge(train, revised, on=['Province_State', 'Country_Region', 'Date'],
suffixes =('', '_r'), how='left')
 rev_train[~rev_train.ConfirmedCases_r.isnull() ].head()
rev_train.ConfirmedCases = \
np.where(( rev_train.ConfirmedCases == 0)&(( rev_train.ConfirmedCases_r > 0)) & 
(rev_train.Country_Region == 'US'),
rev_train.ConfirmedCases_r,
rev_train.ConfirmedCases)
 rev_train.Fatalities = \
np.where(~rev_train.Fatalities_r.isnull() & 
(rev_train.Fatalities == 0)&(( rev_train.Fatalities_r > 0)) & 
(rev_train.Country_Region == 'US')
 ,
rev_train.Fatalities_r,
rev_train.Fatalities)
 rev_train.drop(columns = ['ConfirmedCases_r', 'Fatalities_r'], inplace=True)
 train = rev_train
contain_data = pd.read_csv(path +
'/OxCGRT_Download_070420_160027_Full.csv')
 contain_data = contain_data[[c for c in contain_data.columns if
not any(z in c for z in ['_Notes','Unnamed', 'Confirmed',
'CountryCode',
'S8', 'S9', 'S10','S11',
'StringencyIndexForDisplay'])] ]\
contain_data.rename(columns = {'CountryName': "Country"}, inplace=True)
 contain_data.Date = contain_data.Date.astype(str)\ 
.apply(datetime.datetime.strptime, args=('%Y%m%d',))
contain_data_orig = contain_data.copy()
contain_data.columns
cds = []
for country in contain_data.Country.unique() :
cd = contain_data[contain_data.Country==country]
cd = cd.fillna(method = 'ffill' ).fillna(0)
 cd.StringencyIndex = cd.StringencyIndex.cummax()
col_count = cd.shape[1]
for col in [c for c in contain_data.columns if 'S' in c]:
col_diff = cd[col].diff()
cd[col+"_chg_5d_ewm"] = col_diff.ewm(span = 5 ).mean()
cd[col+"_chg_20_ewm"] = col_diff.ewm(span = 20 ).mean()
cd['StringencyIndex_5d_ewm'] = cd.StringencyIndex.ewm(span = 5 ).mean()
cd['StringencyIndex_20d_ewm'] = cd.StringencyIndex.ewm(span = 20 ).mean()
cd['S_data_days'] =(cd.Date - cd.Date.min() ).dt.days
for s in [1, 10, 20, 30, 50, ]:
cd['days_since_Stringency_{}'.format(s)] = \
np.clip(( cd.Date - cd[(cd.StringencyIndex > s)].Date.min() ).dt.days, 0, None)
 cds.append(cd.fillna(0)[['Country', 'Date'] + cd.columns.to_list() [col_count:]])
 contain_data = pd.concat(cds)
 contain_data.Date.max()
contain_data.columns
contain_data[contain_data.Country == 'Australia']
contain_data.shape
contain_data.Country.replace({ 'United States': "US",
'South Korea': "Korea, South",
'Taiwan': "Taiwan*",
'Myanmar': "Burma", 'Slovak Republic': "Slovakia",
'Czech Republic': 'Czechia',
}, inplace=True)
 set(contain_data.Country)- set(test.Country_Region)
 sup_data = pd.read_excel(path +
'/Data Join - Copy1.xlsx')
 sup_data.columns = [c.replace(' ', '_')for c in sup_data.columns.to_list() ]
sup_data.drop(columns = [c for c in sup_data.columns.to_list() if 'Unnamed:' in c], inplace=True)
 sup_data.drop(columns = [ 'Date', 'ConfirmedCases',
'Fatalities', 'log-cases', 'log-fatalities', 'continent'], inplace=True)
 sup_data['Migrants_in'] = np.clip(sup_data.Migrants, 0, None)
 sup_data['Migrants_out'] = -np.clip(sup_data.Migrants, None, 0)
 sup_data.drop(columns = 'Migrants', inplace=True)
 sup_data.head()
sup_data.shape
train.Date = pd.to_datetime(train.Date)
 test.Date = pd.to_datetime(test.Date)
 train.rename(columns={'Country_Region': 'Country'}, inplace=True)
 test.rename(columns={'Country_Region': 'Country'}, inplace=True)
 sup_data.rename(columns={'Country_Region': 'Country'}, inplace=True)
 train['Place'] = train.Country + train.Province_State.fillna("")
 test['Place'] = test.Country + test.Province_State.fillna("")
 sup_data['Place'] = sup_data.Country + sup_data.Province_State.fillna("")
 len(train.Place.unique())
 sup_data = sup_data[
sup_data.columns.to_list() [2:]]
sup_data = sup_data.replace('N.A.', np.nan ).fillna(-0.5)
 for c in sup_data.columns[:-1]:
m = sup_data[c].max()
if m > 300 and c!='TRUE_POPULATION':
print(c)
 sup_data[c] = np.log(sup_data[c] + 1)
 assert sup_data[c].min() > -1
for c in sup_data.columns[:-1]:
m = sup_data[c].max()
if m > 300:
print(c)
 DEATHS = 'Fatalities'
len(train.Place.unique())
 train.ConfirmedCases = \
np.where(
(train.ConfirmedCases.shift(1)> train.ConfirmedCases)& 
(train.ConfirmedCases.shift(1)> 0)&(train.ConfirmedCases.shift(-1)> 0)& 
(train.Place == train.Place.shift(1)) &(train.Place == train.Place.shift(-1)) &
~train.ConfirmedCases.shift(-1 ).isnull() ,
np.sqrt(train.ConfirmedCases.shift(1)* train.ConfirmedCases.shift(-1)) ,
train.ConfirmedCases)
 train.Fatalities = \
np.where(
(train.Fatalities.shift(1)> train.Fatalities)& 
(train.Fatalities.shift(1)> 0)&(train.Fatalities.shift(-1)> 0)& 
(train.Place == train.Place.shift(1)) &(train.Place == train.Place.shift(-1)) &
~train.Fatalities.shift(-1 ).isnull() ,
np.sqrt(train.Fatalities.shift(1)* train.Fatalities.shift(-1)) ,
train.Fatalities)
 for i in [0, -1]:
train.ConfirmedCases = \
np.where(
(train.ConfirmedCases.shift(2+ i)> train.ConfirmedCases)& 
(train.ConfirmedCases.shift(2+ i)> 0)&(train.ConfirmedCases.shift(-1+ i)> 0)& 
(train.Place == train.Place.shift(2+ i)) &(train.Place == train.Place.shift(-1+ i)) &
~train.ConfirmedCases.shift(-1+ i ).isnull() ,
np.sqrt(train.ConfirmedCases.shift(2+ i)* train.ConfirmedCases.shift(-1+ i)) ,
train.ConfirmedCases)
 train[train.Place=='USVirgin Islands'][-10:]
train[(train.ConfirmedCases.shift(2)> 2* train.ConfirmedCases)& 
(train.Place == train.Place.shift(2)) &(train.ConfirmedCases < 100000)]
train[(train.Fatalities.shift(1)> train.Fatalities)& 
(train.Place == train.Place.shift(1)) &(train.Fatalities < 10000)]
train_bk = train.copy()
full_train = train.copy()
train_c = train[train.Country == 'China']
train_nc = train[train.Country != 'China']
train_us = train[train.Country == 'US']
def lplot(data, minDate = datetime.datetime(2000, 1, 1),
columns = ['ConfirmedCases', 'Fatalities']):
return
REAL = datetime.datetime(2020, 2, 10)
 dataset = train.copy()
if TRUNCATED:
dataset = dataset[dataset.Country.isin(
 ['Italy', 'Spain', 'Germany', 'Portugal', 'Belgium', 'Austria', 'Switzerland' ])]
dataset.head()
def rollDates(df, i, preserve=False):
df = df.copy()
if preserve:
df['Date_i'] = df.Date
df.Date = df.Date + datetime.timedelta(i)
 return df
WINDOWS = [1, 2, 4, 7, 12, 20, 30]
for window in WINDOWS:
csuffix = '_{}d_prior_value'.format(window)
 base = rollDates(dataset, window)
 dataset = pd.merge(dataset, base[['Date', 'Place',
'ConfirmedCases', 'Fatalities']], on = ['Date', 'Place'],
suffixes =('', csuffix), how='left')
 for c in ['ConfirmedCases', 'Fatalities']:
dataset[c+ csuffix].fillna(0, inplace=True)
 dataset[c+ csuffix] = np.log(dataset[c + csuffix] + 1)
 dataset[c+ '_{}d_prior_slope'.format(window)] = \ 
(np.log(dataset[c] + 1)\
- dataset[c+ csuffix])/ window
dataset[c+ '_{}d_ago_zero'.format(window)] = 1.0*(dataset[c+ csuffix] == 0)
 for window1 in WINDOWS:
for window2 in WINDOWS:
for c in ['ConfirmedCases', 'Fatalities']:
if window1 * 1.3 < window2 and window1 * 5 > window2:
dataset[ c +'_{}d_{}d_prior_slope_chg'.format(window1, window2)] = \
dataset[c+ '_{}d_prior_slope'.format(window1)] \
- dataset[c+ '_{}d_prior_slope'.format(window2)]
first_case = dataset[dataset.ConfirmedCases >= 1].groupby('Place' ).min()
tenth_case = dataset[dataset.ConfirmedCases >= 10].groupby('Place' ).min()
hundredth_case = dataset[dataset.ConfirmedCases >= 100].groupby('Place' ).min()
thousandth_case = dataset[dataset.ConfirmedCases >= 1000].groupby('Place' ).min()
first_fatality = dataset[dataset.Fatalities >= 1].groupby('Place' ).min()
tenth_fatality = dataset[dataset.Fatalities >= 10].groupby('Place' ).min()
hundredth_fatality = dataset[dataset.Fatalities >= 100].groupby('Place' ).min()
thousandth_fatality = dataset[dataset.Fatalities >= 1000].groupby('Place' ).min()
dataset['days_since_first_case'] = \
np.clip(( dataset.Date - first_case.loc[dataset.Place].Date.values ).dt.days\ 
.fillna(-1), -1, None)
 dataset['days_since_tenth_case'] = \
np.clip(( dataset.Date - tenth_case.loc[dataset.Place].Date.values ).dt.days\ 
.fillna(-1), -1, None)
 dataset['days_since_hundredth_case'] = \
np.clip(( dataset.Date - hundredth_case.loc[dataset.Place].Date.values ).dt.days\ 
.fillna(-1), -1, None)
 dataset['days_since_thousandth_case'] = \
np.clip(( dataset.Date - thousandth_case.loc[dataset.Place].Date.values ).dt.days\ 
.fillna(-1), -1, None)
 dataset['days_since_first_fatality'] = \
np.clip(( dataset.Date - first_fatality.loc[dataset.Place].Date.values ).dt.days\ 
.fillna(-1), -1, None)
 dataset['days_since_tenth_fatality'] = \
np.clip(( dataset.Date - tenth_fatality.loc[dataset.Place].Date.values ).dt.days\ 
.fillna(-1), -1, None)
 dataset['days_since_hundredth_fatality'] = \
np.clip(( dataset.Date - hundredth_fatality.loc[dataset.Place].Date.values ).dt.days\ 
.fillna(-1), -1, None)
 dataset['days_since_thousandth_fatality'] = \
np.clip(( dataset.Date - thousandth_fatality.loc[dataset.Place].Date.values ).dt.days\ 
.fillna(-1), -1, None)
 dataset['case_rate_since_first_case'] = \
np.clip(( np.log(dataset.ConfirmedCases + 1)\
- np.log(first_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) \
/(dataset.days_since_first_case+0.01), 0, 1)
 dataset['case_rate_since_tenth_case'] = \
np.clip(( np.log(dataset.ConfirmedCases + 1)\
- np.log(tenth_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) \
/(dataset.days_since_tenth_case+0.01), 0, 1)
 dataset['case_rate_since_hundredth_case'] = \
np.clip(( np.log(dataset.ConfirmedCases + 1)\
- np.log(hundredth_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) \
/(dataset.days_since_first_case+0.01), 0, 1)
 dataset['case_rate_since_thousandth_case'] = \
np.clip(( np.log(dataset.ConfirmedCases + 1)\
- np.log(thousandth_case.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)) \
/(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_first_case'] = \
np.clip(( np.log(dataset.Fatalities + 1)\
- np.log(first_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) \
/(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_tenth_case'] = \
np.clip(( np.log(dataset.Fatalities + 1)\
- np.log(tenth_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) \
/(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_hundredth_case'] = \
np.clip(( np.log(dataset.Fatalities + 1)\
- np.log(hundredth_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) \
/(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_thousandth_case'] = \
np.clip(( np.log(dataset.Fatalities + 1)\
- np.log(thousandth_case.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) \
/(dataset.days_since_first_case+0.01), 0, 1)
 dataset['fatality_rate_since_first_fatality'] = \
np.clip(( np.log(dataset.Fatalities + 1)\
- np.log(first_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) \
/(dataset.days_since_first_fatality+0.01), 0, 1)
 dataset['fatality_rate_since_tenth_fatality'] = \
np.clip(( np.log(dataset.Fatalities + 1)\
- np.log(tenth_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) \
/(dataset.days_since_tenth_fatality+0.01), 0, 1)
 dataset['fatality_rate_since_hundredth_fatality'] = \
np.clip(( np.log(dataset.Fatalities + 1)\
- np.log(hundredth_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) \
/(dataset.days_since_hundredth_fatality+0.01), 0, 1)
 dataset['fatality_rate_since_thousandth_fatality'] = \
np.clip(( np.log(dataset.Fatalities + 1)\
- np.log(thousandth_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)) \
/(dataset.days_since_thousandth_fatality+0.01), 0, 1)
 dataset['first_case_ConfirmedCases'] = \
np.log(first_case.loc[dataset.Place].ConfirmedCases.values + 1)
 dataset['first_case_Fatalities'] = \
np.log(first_case.loc[dataset.Place].Fatalities.values + 1)
 dataset['first_fatality_ConfirmedCases'] = \
np.log(first_fatality.loc[dataset.Place].ConfirmedCases.fillna(0 ).values + 1)\
*(dataset.days_since_first_fatality >= 0)
 dataset['first_fatality_Fatalities'] = \
np.log(first_fatality.loc[dataset.Place].Fatalities.fillna(0 ).values + 1)\
*(dataset.days_since_first_fatality >= 0)
 dataset['first_fatality_cfr'] = \
np.where(dataset.days_since_first_fatality < 0,
-8, 
(dataset.first_fatality_Fatalities)- 
(dataset.first_fatality_ConfirmedCases))
dataset['first_fatality_lag_vs_first_case'] = \
np.where(dataset.days_since_first_fatality >= 0,
dataset.days_since_first_case - dataset.days_since_first_fatality , -1)
 dataset['case_chg'] = \
np.clip(np.log(dataset.ConfirmedCases + 1)\
- np.log(dataset.ConfirmedCases.shift(1)+1), 0, None ).fillna(0)
 dataset['case_chg_ema_3d'] = dataset.case_chg.ewm(span = 3 ).mean() \
* np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1)
 dataset['case_chg_ema_10d'] = dataset.case_chg.ewm(span = 10 ).mean() \
* np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1)
 dataset['case_chg_stdev_5d'] = dataset.case_chg.rolling(5 ).std() \
* np.clip(( dataset.Date - dataset.Date.min() ).dt.days/5, 0, 1)
 dataset['case_chg_stdev_15d'] = dataset.case_chg.rolling(15 ).std() \
* np.clip(( dataset.Date - dataset.Date.min() ).dt.days/15, 0, 1)
 dataset['case_update_pct_3d_ewm'] =(dataset.case_chg > 0 ).ewm(span = 3 ).mean() \
* np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1), 2)
 dataset['case_update_pct_10d_ewm'] =(dataset.case_chg > 0 ).ewm(span = 10 ).mean() \
* np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1), 2)
 dataset['case_update_pct_30d_ewm'] =(dataset.case_chg > 0 ).ewm(span = 30 ).mean() \
* np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/30, 0, 1), 2)
 dataset['fatality_chg'] = \
np.clip(np.log(dataset.Fatalities + 1)\
- np.log(dataset.Fatalities.shift(1)+1), 0, None ).fillna(0)
 dataset['fatality_chg_ema_3d'] = dataset.fatality_chg.ewm(span = 3 ).mean() \
* np.clip(( dataset.Date - dataset.Date.min() ).dt.days/33, 0, 1)
 dataset['fatality_chg_ema_10d'] = dataset.fatality_chg.ewm(span = 10 ).mean() \
* np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1)
 dataset['fatality_chg_stdev_5d'] = dataset.fatality_chg.rolling(5 ).std() \
* np.clip(( dataset.Date - dataset.Date.min() ).dt.days/5, 0, 1)
 dataset['fatality_chg_stdev_15d'] = dataset.fatality_chg.rolling(15 ).std() \
* np.clip(( dataset.Date - dataset.Date.min() ).dt.days/15, 0, 1)
 dataset['fatality_update_pct_3d_ewm'] =(dataset.fatality_chg > 0 ).ewm(span = 3 ).mean() \
* np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1), 2)
 dataset['fatality_update_pct_10d_ewm'] =(dataset.fatality_chg > 0 ).ewm(span = 10 ).mean() \
* np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/10, 0, 1), 2)
 dataset['fatality_update_pct_30d_ewm'] =(dataset.fatality_chg > 0 ).ewm(span = 30 ).mean() \
* np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/30, 0, 1), 2)
 dataset.tail()
contain_data.Date = contain_data.Date + datetime.timedelta(7)
 contain_data.Date.max()
assert set(dataset.Place.unique())== set(dataset.Place.unique())
 dataset = pd.merge(dataset, sup_data, on='Place', how='left', validate='m:1')
 dataset = pd.merge(dataset, contain_data, on = ['Country', 'Date'], how='left', validate='m:1')
 dataset['log_true_population'] = np.log(dataset.TRUE_POPULATION + 1)
 dataset['ConfirmedCases_percapita'] = np.log(dataset.ConfirmedCases + 1)\
- np.log(dataset.TRUE_POPULATION + 1)
 dataset['Fatalities_percapita'] = np.log(dataset.Fatalities + 1)\
- np.log(dataset.TRUE_POPULATION + 1)
 dataset['log_cfr'] = np.log(( dataset.Fatalities \
+ np.clip(0.015 * dataset.ConfirmedCases, 0, 0.3)) \
/(dataset.ConfirmedCases + 0.1))
def cfr(case, fatality):
cfr_calc = np.log(( fatality \
+ np.clip(0.015 * case, 0, 0.3)) \
/(case + 0.1))
return np.where(np.isnan(cfr_calc)| np.isinf(cfr_calc),
BLCFR, cfr_calc)
 BLCFR = np.median(dataset[dataset.ConfirmedCases==1].log_cfr[::10])
 dataset.log_cfr.fillna(BLCFR, inplace=True)
 dataset.log_cfr = np.where(dataset.log_cfr.isnull() | np.isinf(dataset.log_cfr),
BLCFR, dataset.log_cfr)
 BLCFR
dataset['log_cfr_3d_ewm'] = BLCFR + \ 
(dataset.log_cfr - BLCFR ).ewm(span = 3 ).mean() \
* np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/3, 0, 1), 2)
 dataset['log_cfr_8d_ewm'] = BLCFR + \ 
(dataset.log_cfr - BLCFR ).ewm(span = 8 ).mean() \
* np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/8, 0, 1), 2)
 dataset['log_cfr_20d_ewm'] = BLCFR + \ 
(dataset.log_cfr - BLCFR ).ewm(span = 20 ).mean() \
* np.power(np.clip(( dataset.Date - dataset.Date.min() ).dt.days/20, 0, 1), 2)
 dataset['log_cfr_3d_20d_ewm_crossover'] = dataset.log_cfr_3d_ewm - dataset.log_cfr_20d_ewm
dataset.drop(columns = 'log_cfr', inplace=True)
 date_totals = dataset.groupby('Date' ).sum()
mean_7d_c_slope = dataset.groupby('Date')[['ConfirmedCases_7d_prior_slope']].apply(lambda x:
np.mean(x[x > 0])).ewm(span = 3 ).mean()
mean_7d_f_slope = dataset.groupby('Date')[['Fatalities_7d_prior_slope']].apply(lambda x:
np.mean(x[x > 0])).ewm(span = 7 ).mean()
dataset['ConfirmedCases_percapita_vs_world'] = np.log(dataset.ConfirmedCases + 1)\
- np.log(dataset.TRUE_POPULATION + 1)\
-(
 np.log(date_totals.loc[dataset.Date].ConfirmedCases + 1)
 -np.log(date_totals.loc[dataset.Date].TRUE_POPULATION + 1)
 ).values
dataset['Fatalities_percapita_vs_world'] = np.log(dataset.Fatalities + 1)\
- np.log(dataset.TRUE_POPULATION + 1)\
-(
 np.log(date_totals.loc[dataset.Date].Fatalities + 1)
 -np.log(date_totals.loc[dataset.Date].TRUE_POPULATION + 1)
 ).values
dataset['cfr_vs_world'] = dataset.log_cfr_3d_ewm \
- np.log(date_totals.loc[dataset.Date].Fatalities \
/ date_totals.loc[dataset.Date].ConfirmedCases ).values
cont_date_totals = dataset.groupby(['Date', 'continent_generosity'] ).sum()
len(dataset)
 dataset['ConfirmedCases_percapita_vs_continent_mean'] = 0
dataset['Fatalities_percapita_vs_continent_mean'] = 0
dataset['ConfirmedCases_percapita_vs_continent_median'] = 0
dataset['Fatalities_percapita_vs_continent_median'] = 0
for cg in dataset.continent_generosity.unique() :
ps = dataset.groupby("Place" ).last()
tp = ps[ps.continent_generosity==cg].TRUE_POPULATION.sum()
print(tp / 1e9)
 for Date in dataset.Date.unique() :
cd = dataset[(dataset.Date == Date)& 
(dataset.continent_generosity == cg)]\
[['ConfirmedCases', 'Fatalities', 'TRUE_POPULATION']]
cmedian = np.median(np.log(cd.ConfirmedCases + 1)\
- np.log(cd.TRUE_POPULATION+1))
cmean = np.log(cd.ConfirmedCases.sum() + 1)- np.log(tp + 1)
 fmedian = np.median(np.log(cd.Fatalities + 1)\
- np.log(cd.TRUE_POPULATION+1))
fmean = np.log(cd.Fatalities.sum() + 1)- np.log(tp + 1)
 cfrmean = cfr(cd.ConfirmedCases.sum() , cd.Fatalities.sum())
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'ConfirmedCases_percapita_vs_continent_mean'] = \
dataset['ConfirmedCases_percapita'] \
-(cmean)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'ConfirmedCases_percapita_vs_continent_median'] = \
dataset['ConfirmedCases_percapita'] \
-(cmedian)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'Fatalities_percapita_vs_continent_mean'] = \
dataset['Fatalities_percapita']\
-(fmean)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'Fatalities_percapita_vs_continent_median'] = \
dataset['Fatalities_percapita']\
-(fmedian)
 dataset.loc[(dataset.Date == Date)& 
(dataset.continent_generosity == cg),
'cfr_vs_continent'] = \
dataset.log_cfr_3d_ewm \
- cfrmean
all_places = dataset[['Place', 'latitude', 'longitude']].drop_duplicates().set_index('Place',
drop=True)
 all_places.head()
def surroundingPlaces(place, d = 10):
dist =(all_places.latitude - all_places.loc[place].latitude)**2 \
+(all_places.longitude - all_places.loc[place].longitude)** 2
return all_places[dist < d**2][1:n+1]
def nearestPlaces(place, n = 10):
dist =(all_places.latitude - all_places.loc[place].latitude)**2 \
+(all_places.longitude - all_places.loc[place].longitude)** 2
ranked = np.argsort(dist)
 return all_places.iloc[ranked][1:n+1]
dgp = dataset.groupby('Place' ).last()
for n in [5, 10, 20]:
for place in dataset.Place.unique() :
nps = nearestPlaces(place, n)
 tp = dgp.loc[nps.index].TRUE_POPULATION.sum()
dataset.loc[dataset.Place==place,
'ratio_population_vs_nearest{}'.format(n)] = \
np.log(dataset.loc[dataset.Place==place].TRUE_POPULATION.mean() + 1)\
- np.log(tp+1)
 nbps = dataset[(dataset.Place.isin(nps.index)) ]\ 
.groupby('Date')[['ConfirmedCases', 'Fatalities']].sum()
nppc =(np.log(nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).ConfirmedCases + 1)- np.log(tp + 1))
nppf =(np.log(nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).Fatalities + 1)- np.log(tp + 1))
npp_cfr = cfr(nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).ConfirmedCases,
nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).Fatalities)
 dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_percapita_vs_nearest{}'.format(n)] = \
dataset[(dataset.Place == place)].ConfirmedCases_percapita \
- nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_percapita_vs_nearest{}'.format(n)] = \
dataset[(dataset.Place == place)].Fatalities_percapita \
- nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_vs_nearest{}'.format(n)] = \
dataset[(dataset.Place == place)].log_cfr_3d_ewm \
- npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_nearest{}_percapita'.format(n)] = nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_nearest{}_percapita'.format(n)] = nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_nearest{}'.format(n)] = npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_nearest{}_10d_slope'.format(n)] = \ 
(nppc.ewm(span = 1 ).mean() - nppc.ewm(span = 10 ).mean() ).values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_nearest{}_10d_slope'.format(n)] = \ 
(nppf.ewm(span = 1 ).mean() - nppf.ewm(span = 10 ).mean() ).values
npp_cfr_s = pd.Series(npp_cfr)
 dataset.loc[ 
(dataset.Place == place),
'cfr_nearest{}_10d_slope'.format(n)] = \ 
(npp_cfr_s.ewm(span = 1 ).mean() \
- npp_cfr_s.ewm(span = 10 ).mean() ).values
dgp = dataset.groupby('Place' ).last()
for d in [5, 10, 20]:
for place in dataset.Place.unique() :
nps = surroundingPlaces(place, d)
 dataset.loc[dataset.Place==place, 'num_surrounding_places_{}_degrees'.format(d)] = \
len(nps)
 tp = dgp.loc[nps.index].TRUE_POPULATION.sum()
dataset.loc[dataset.Place==place,
'ratio_population_vs_surrounding_places_{}_degrees'.format(d)] = \
np.log(dataset.loc[dataset.Place==place].TRUE_POPULATION.mean() + 1)\
- np.log(tp+1)
 if len(nps)==0:
continue;
nbps = dataset[(dataset.Place.isin(nps.index)) ]\ 
.groupby('Date')[['ConfirmedCases', 'Fatalities']].sum()
nppc =(np.log(nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).ConfirmedCases + 1)- np.log(tp + 1))
nppf =(np.log(nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).Fatalities + 1)- np.log(tp + 1))
npp_cfr = cfr(nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).ConfirmedCases,
nbps.loc[dataset[dataset.Place==place].Date]\ 
.fillna(0 ).Fatalities)
 dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_percapita_vs_surrounding_places_{}_degrees'.format(d)] = \
dataset[(dataset.Place == place)].ConfirmedCases_percapita \
- nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_percapita_vs_surrounding_places_{}_degrees'.format(d)] = \
dataset[(dataset.Place == place)].Fatalities_percapita \
- nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_vs_surrounding_places_{}_degrees'.format(d)] = \
dataset[(dataset.Place == place)].log_cfr_3d_ewm \
- npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_surrounding_places_{}_degrees_percapita'.format(d)] = nppc.values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_surrounding_places_{}_degrees_percapita'.format(d)] = nppf.values
dataset.loc[ 
(dataset.Place == place),
'cfr_surrounding_places_{}_degrees'.format(d)] = npp_cfr
dataset.loc[ 
(dataset.Place == place),
'ConfirmedCases_surrounding_places_{}_degrees_10d_slope'.format(d)] = \ 
(nppc.ewm(span = 1 ).mean() - nppc.ewm(span = 10 ).mean() ).values
dataset.loc[ 
(dataset.Place == place),
'Fatalities_surrounding_places_{}_degrees_10d_slope'.format(d)] = \ 
(nppf.ewm(span = 1 ).mean() - nppf.ewm(span = 10 ).mean() ).values
npp_cfr_s = pd.Series(npp_cfr)
 dataset.loc[ 
(dataset.Place == place),
'cfr_surrounding_places_{}_degrees_10d_slope'.format(d)] = \ 
(npp_cfr_s.ewm(span = 1 ).mean() \
- npp_cfr_s.ewm(span = 10 ).mean() ).values
for col in [c for c in dataset.columns if 'surrounding_places' in c and 'num_sur' not in c]:
dataset[col] = dataset[col].fillna(0)
 n_col = 'num_surrounding_places_{}_degrees'.format(col.split('degrees')[0]\ 
.split('_')[-2])
 print(col)
 dataset[col + "_times_num_places"] = dataset[col] * np.sqrt(dataset[n_col])
 dataset[dataset.Country=='US'][['Place', 'Date'] \
+ [c for c in dataset.columns if 'ratio_p' in c]]\
[::50]
dataset.TRUE_POPULATION
dataset.TRUE_POPULATION.sum()
dataset.groupby('Date' ).sum().TRUE_POPULATION
dataset['first_case_ConfirmedCases_percapita'] = \
np.log(dataset.first_case_ConfirmedCases + 1)\
- np.log(dataset.TRUE_POPULATION + 1)
 dataset['first_case_Fatalities_percapita'] = \
np.log(dataset.first_case_Fatalities + 1)\
- np.log(dataset.TRUE_POPULATION + 1)
 dataset['first_fatality_Fatalities_percapita'] = \
np.log(dataset.first_fatality_Fatalities + 1)\
- np.log(dataset.TRUE_POPULATION + 1)
 dataset['first_fatality_ConfirmedCases_percapita'] = \
np.log(dataset.first_fatality_ConfirmedCases + 1)\
- np.log(dataset.TRUE_POPULATION + 1)
 dataset['days_to_saturation_ConfirmedCases_4d'] = \ 
(- np.log(dataset.ConfirmedCases + 1)\
+ np.log(dataset.TRUE_POPULATION + 1)) \
/ dataset.ConfirmedCases_4d_prior_slope
dataset['days_to_saturation_ConfirmedCases_7d'] = \ 
(- np.log(dataset.ConfirmedCases + 1)\
+ np.log(dataset.TRUE_POPULATION + 1)) \
/ dataset.ConfirmedCases_7d_prior_slope
dataset['days_to_saturation_Fatalities_20d_cases'] = \ 
(- np.log(dataset.Fatalities + 1)\
+ np.log(dataset.TRUE_POPULATION + 1)) \
/ dataset.ConfirmedCases_20d_prior_slope
dataset['days_to_saturation_Fatalities_12d_cases'] = \ 
(- np.log(dataset.Fatalities + 1)\
+ np.log(dataset.TRUE_POPULATION + 1)) \
/ dataset.ConfirmedCases_12d_prior_slope
dataset['days_to_3pct_ConfirmedCases_4d'] = \ 
(- np.log(dataset.ConfirmedCases + 1)\
+ np.log(dataset.TRUE_POPULATION + 1)- 3.5)\
/ dataset.ConfirmedCases_4d_prior_slope
dataset['days_to_3pct_ConfirmedCases_7d'] = \ 
(- np.log(dataset.ConfirmedCases + 1)\
+ np.log(dataset.TRUE_POPULATION + 1)- 3.5)\
/ dataset.ConfirmedCases_7d_prior_slope
dataset['days_to_0.3pct_Fatalities_20d_cases'] = \ 
(- np.log(dataset.Fatalities + 1)\
+ np.log(dataset.TRUE_POPULATION + 1)- 5.8)\
/ dataset.ConfirmedCases_20d_prior_slope
dataset['days_to_0.3pct_Fatalities_12d_cases'] = \ 
(- np.log(dataset.Fatalities + 1)\
+ np.log(dataset.TRUE_POPULATION + 1)- 5.8)\
/ dataset.ConfirmedCases_12d_prior_slope
dataset.tail()
dataset = dataset[dataset.ConfirmedCases > 0]
len(dataset)
 datas = []
for window in range(1, 35):
base = rollDates(dataset, window, True)
 datas.append(pd.merge(dataset[['Date', 'Place',
'ConfirmedCases', 'Fatalities']], base, on = ['Date', 'Place'],
how = 'right',
suffixes =('_f', '')))
 data = pd.concat(datas, axis =0 ).astype(np.float32, errors ='ignore')
 len(data)
 data['Date_f'] = data.Date
data.Date = data.Date_i
data['elapsed'] =(data.Date_f - data.Date_i ).dt.days
data['CaseChgRate'] =(np.log(data.ConfirmedCases_f + 1)- np.log(data.ConfirmedCases + 1)) \
/ data.elapsed;
data['FatalityChgRate'] =(np.log(data.Fatalities_f + 1)- np.log(data.Fatalities + 1)) \
/ data.elapsed;
data.elapsed
falloff_hash = {}
def true_agg(rate_i, elapsed, bend_rate):
elapsed = int(elapsed)
 if(bend_rate, elapsed)not in falloff_hash:
falloff_hash[(bend_rate, elapsed)] = \
np.sum([ np.power(bend_rate, e)for e in range(1, elapsed+1)])
 return falloff_hash[(bend_rate, elapsed)] * rate_i
true_agg(0.3, 30, 0.9)
 slope_cols = [c for c in data.columns if
any(z in c for z in ['prior_slope', 'chg', 'rate'])
 and not any(z in c for z in ['bend', 'prior_slope_chg', 'Country', 'ewm',
])]
print(slope_cols)
 bend_rates = [1, 0.95, 0.90]
for bend_rate in bend_rates:
bend_agg = data[['elapsed']].apply(lambda x: true_agg(1, *x, bend_rate), axis=1)
 for sc in slope_cols:
if bend_rate < 1:
data[sc+"_slope_bend_{}".format(bend_rate)] = data[sc] \
* np.power(( bend_rate + 1)/2, data.elapsed)
 data[sc+"_true_slope_bend_{}".format(bend_rate)] = \
bend_agg * data[sc] / data.elapsed
data[sc+"_agg_bend_{}".format(bend_rate)] = data[sc] * data.elapsed \
* np.power(( bend_rate + 1)/2, data.elapsed)
 data[sc+"_true_agg_bend_{}".format(bend_rate)] = \
bend_agg * data[sc]
slope_cols[:5]
for col in [c for c in data.columns if any(z in c for z in
['vs_continent', 'nearest', 'vs_world', 'surrounding_places'])]:
data[col + '_times_days'] = data[col] * data.elapsed
data['saturation_slope_ConfirmedCases'] =(- np.log(data.ConfirmedCases + 1)\
+ np.log(data.TRUE_POPULATION + 1)) \
/ data.elapsed
data['saturation_slope_Fatalities'] =(- np.log(data.Fatalities + 1)\
+ np.log(data.TRUE_POPULATION + 1)) \
/ data.elapsed
data['dist_to_ConfirmedCases_saturation_times_days'] =(- np.log(data.ConfirmedCases + 1)\
+ np.log(data.TRUE_POPULATION + 1)) \
* data.elapsed
data['dist_to_Fatalities_saturation_times_days'] =(- np.log(data.Fatalities + 1)\
+ np.log(data.TRUE_POPULATION + 1)) \
* data.elapsed
data['slope_to_1pct_ConfirmedCases'] =(- np.log(data.ConfirmedCases + 1)\
+ np.log(data.TRUE_POPULATION + 1)- 4.6)\
/ data.elapsed
data['slope_to_0.1pct_Fatalities'] =(- np.log(data.Fatalities + 1)\
+ np.log(data.TRUE_POPULATION + 1)- 6.9)\
/ data.elapsed
data['dist_to_1pct_ConfirmedCases_times_days'] =(- np.log(data.ConfirmedCases + 1)\
+ np.log(data.TRUE_POPULATION + 1)- 4.6)\
* data.elapsed
data['dist_to_0.1pct_Fatalities_times_days'] =(- np.log(data.Fatalities + 1)\
+ np.log(data.TRUE_POPULATION + 1)- 6.9)\
* data.elapsed
data['trendline_per_capita_ConfirmedCases_4d_slope'] =(np.log(data.ConfirmedCases + 1)\
- np.log(data.TRUE_POPULATION + 1)) \
+(data.ConfirmedCases_4d_prior_slope * data.elapsed)
 data['trendline_per_capita_ConfirmedCases_7d_slope'] =(np.log(data.ConfirmedCases + 1)\
- np.log(data.TRUE_POPULATION + 1)) \
+(data.ConfirmedCases_7d_prior_slope * data.elapsed)
 data['trendline_per_capita_Fatalities_12d_slope'] =(np.log(data.Fatalities + 1)\
- np.log(data.TRUE_POPULATION + 1)) \
+(data.ConfirmedCases_12d_prior_slope * data.elapsed)
 data['trendline_per_capita_Fatalities_20d_slope'] =(np.log(data.Fatalities + 1)\
- np.log(data.TRUE_POPULATION + 1)) \
+(data.ConfirmedCases_20d_prior_slope * data.elapsed)
 len(data)
 data.groupby('Place' ).last()
def logHist(x, b = 150):
return
data['log_fatalities'] = np.log(data.Fatalities + 1)
 data['log_cases'] = np.log(data.ConfirmedCases + 1)
 data['is_China'] =(data.Country=='China')&(~data.Place.isin(['Hong Kong', 'Macau']))
for col in [c for c in data.columns if 'd_ewm' in c]:
data[col] += np.random.normal(0, 1, len(data)) * np.std(data[col])* 0.2
data['is_province'] = 1.0*(~data.Province_State.isnull())
 data['log_elapsed'] = np.log(data.elapsed + 1)
 data.columns
data.columns[::19]
data.shape
logHist(data.ConfirmedCases)
 data.drop(columns = ['TRUE_POPULATION'], inplace=True)
 data['final_day_of_week'] = data.Date_f.apply(datetime.datetime.weekday)
 data['base_date_day_of_week'] = data.Date.apply(datetime.datetime.weekday)
 data['date_difference_modulo_7_days'] =(data.Date_f - data.Date ).dt.days % 7
for c in data.columns.to_list() :
if 'days_to' in c:
data[c] = data[c].where(~np.isinf(data[c]), 1e3)
 data[c] = np.clip(data[c], 0, 365)
 data[c] = np.sqrt(data[c])
 new_places = train[(train.Date == test.Date.min() - datetime.timedelta(1)) & 
(train.ConfirmedCases == 0)
 ].Place
model_data = data[(( len(test)==0)|(data.Date_f < test.Date.min()))
& 
(data.ConfirmedCases > 0)& 
(~data.ConfirmedCases_f.isnull())].copy()
test.Date.min()
model_data.Date_f.max()
model_data.Date_f.max()
model_data.Date.max()
model_data.Date_f.min()
model_data = model_data[~(
(np.random.rand(len(model_data)) < 0.8)& 
(model_data.Country == 'China')& 
(model_data.Date < datetime.datetime(2020, 2, 15)))]
x_dates = model_data[['Date_i', 'Date_f', 'Place']]
x = model_data[
model_data.columns.to_list() [
model_data.columns.to_list().index('ConfirmedCases_1d_prior_value'):]]\ 
.drop(columns = ['Date_i', 'Date_f', 'CaseChgRate', 'FatalityChgRate'])
 test.Date
if PRIVATE:
data_test = data[(data.Date_i == train.Date.max())& 
(data.Date_f.isin(test.Date.unique())) ].copy()
else:
data_test = data[(data.Date_i == test.Date.min() - datetime.timedelta(1)) & 
(data.Date_f.isin(test.Date.unique())) ].copy()
data_test.Date.unique()
test.Date.unique()
x_test = data_test[x.columns].copy()
train.Date.max()
test.Date.max()
if MODEL_Y is 'slope':
y_cases = model_data.CaseChgRate
y_fatalities = model_data.FatalityChgRate
else:
y_cases = model_data.CaseChgRate * model_data.elapsed
y_fatalities = model_data.FatalityChgRate * model_data.elapsed
y_cfr = np.log(( model_data.Fatalities_f \
+ np.clip(0.015 * model_data.ConfirmedCases_f, 0, 0.3)) \
/(model_data.ConfirmedCases_f + 0.1))
groups = model_data.Country
places = model_data.Place
np.random.seed(SEED)
 enet_params = { 'alpha': [ 3e-6, 1e-5, 3e-5, 1e-4, 3e-4, 1e-3, ],
'l1_ratio': [ 0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 0.8, 0.9, 0.97, 0.99 ]}
et_params = { 'n_estimators': [50, 70, 100, 140],
'max_depth': [3, 5, 7, 8, 9, 10],
'min_samples_leaf': [30, 50, 70, 100, 130, 165, 200, 300, 600],
'max_features': [0.4, 0.5, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85],
'min_impurity_decrease': [0, 1e-5 ],
'bootstrap': [ True, False],
}
lgb_params = {
'max_depth': [5, 12],
'n_estimators': [ 100, 200, 300, 500],
'min_split_gain': [0, 0, 1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2],
'min_child_samples': [ 7, 10, 14, 20, 30, 40, 70, 100, 200, 400, 700, 1000, 2000],
'min_child_weight': [0],
'num_leaves': [5, 10, 20, 30],
'learning_rate': [0.05, 0.07, 0.1],
'colsample_bytree': [0.1, 0.2, 0.33, 0.5, 0.65, 0.8, 0.9],
'colsample_bynode':[0.1, 0.2, 0.33, 0.5, 0.65, 0.81],
'reg_lambda': [1e-5, 3e-5, 1e-4, 1e-3, 1e-2, 0.1, 1, 10, 100, 1000, ],
'reg_alpha': [1e-5, 1e-4, 1e-3, 1e-2, 0.1, 1, 30, 1000,],
'subsample': [ 0.8, 0.9, 1],
'subsample_freq': [1],
'max_bin': [ 7, 15, 31, 63, 127, 255],
}
MSE = 'neg_mean_squared_error'
MAE = 'neg_mean_absolute_error'
def trainENet(x, y, groups, cv = 0, **kwargs):
return trainModel(x, y, groups,
clf = ElasticNet(normalize = True, selection = 'random',
max_iter = 3000),
params = enet_params,
cv = cv, **kwargs)
 def trainETR(x, y, groups, cv = 0, n_jobs = 5, **kwargs):
clf = ExtraTreesRegressor(n_jobs = 1)
 params = et_params
return trainModel(x, y, groups, clf, params, cv, n_jobs, **kwargs)
 def trainLGB(x, y, groups, cv = 0, n_jobs = 4, **kwargs):
clf = lgb.LGBMRegressor(verbosity=-1, hist_pool_size = 1000, 
)
 params = lgb_params
return trainModel(x, y, groups, clf, params, cv, n_jobs, **kwargs)
 def trainModel(x, y, groups, clf, params, cv = 0, n_jobs = None,
verbose=0, splits=None, **kwargs):
if n_jobs is None:
n_jobs = 4
if np.random.rand() < 0.8:
folds = GroupShuffleSplit(n_splits=4,
test_size= 0.2 + 0.10 * np.random.rand())
 else:
folds = GroupKFold(4)
 clf = RandomizedSearchCV(clf, params,
cv= folds,
n_iter=12,
verbose = 0, n_jobs = n_jobs, scoring = MSE)
 f = clf.fit(x, y, groups)
 print(pd.DataFrame(clf.cv_results_['mean_test_score'])) ; print() ;
best = clf.best_estimator_; print(best)
 print("Best Score: {}".format(np.round(clf.best_score_,4)))
 return best
np.mean(y_cases)
 def getSparseColumns(x, verbose = 0):
sc = []
for c in x.columns.to_list() :
u = len(x[c].unique())
 if u > 10 and u < 0.01*len(x):
sc.append(c)
 if verbose > 0:
print("{}: {}".format(c, u))
return sc
def noisify(x, noise = 0.1):
x = x.copy()
cols = getSparseColumns(x)
 for c in cols:
u = len(x[c].unique())
 if u > 50:
x[c].values[:] = x[c].values + np.random.normal(0, noise, len(x)) * np.std(x[c])
 return x;
def getMaxOverlap(row, df):
df_place = df[df.Place == row.Place]
if len(df_place)==0:
return 0
overlap = \ 
(np.clip(df_place.Date_f, None, row.Date_f)\
- np.clip(df_place.Date_i, row.Date_i, None)).dt.days
overlap = np.clip(overlap, 0, None)
 length = np.clip(( df_place.Date_f - df_place.Date_i ).dt.days, 
(row.Date_f - row.Date_i ).days, None)
 return np.amax(overlap / length)
 def getSampleWeight(x, groups):
counter = Counter(groups)
 median_count = np.median([counter[group] for group in groups.unique() ])
 c_count = [counter[group] for group in groups]
e_decay = np.round(LT_DECAY_MIN + np.random.rand() *(LT_DECAY_MAX - LT_DECAY_MIN), 1)
 print("LT weight decay: {:.2f}".format(e_decay)) ;
ssr = np.power(1 / np.clip(c_count / median_count , 0.1, 30),
0.1 + np.random.rand() * 0.6)\
/ np.power(x.elapsed / 3, e_decay)\
* SET_FRAC * np.exp(- np.random.rand())
 group_drop = dict([(group, np.random.rand() < 0.15)for group in groups.unique() ])
 ssr = ssr *([ 1 -group_drop[group] for group in groups])
 return ssr;
def runBags(x, y, groups, cv, bags = 3, model_type = trainLGB,
noise = 0.1, splits = None, weights = None, **kwargs):
models = []
for bag in range(bags):
print("
BAG {}".format(bag+1))
x = x.copy()
if DROPS:
for col in [c for c in x.columns if any(z in c for z in ['bend', 'slope', 'prior'])]:
if np.random.rand() < np.sqrt(np.random.rand())* 0.7:
x[col].values[:] = 0
if DROPS and(np.random.rand() < 0.30):
print('dropping nearest features')
 for col in [c for c in x.columns if 'nearest' in c]:
x[col].values[:] = 0
if DROPS and(np.random.rand() < 0.25):
print('dropping 'surrounding places' features')
 for col in [c for c in x.columns if 'surrounding_places' in c]:
x[col].values[:] = 0
col_drop_frac = np.sqrt(np.random.rand())* 0.5
for col in [c for c in x.columns if 'elapsed' not in c ]:
if np.random.rand() < col_drop_frac:
x[col].values[:] = 0
x = noisify(x, noise)
 if DROPS and(np.random.rand() < SUP_DROP):
print("Dropping supplemental country data")
 for col in x[[c for c in x.columns if c in sup_data.columns]]:
x[col].values[:] = 0
if DROPS and(np.random.rand() < ACTIONS_DROP):
for col in x[[c for c in x.columns if c in contain_data.columns]]:
x[col].values[:] = 0
else:
print("*using containment data")
 if np.random.rand() < 0.6:
x.S_data_days = 0
ssr = getSampleWeight(x, groups)
 date_falloff = 0 +(1/30)* np.random.rand()
if weights is not None:
ssr = ssr * np.exp(-weights * date_falloff)
 ss =(np.random.rand(len(y)) < ssr)
 print("n={}".format(len(x[ss])))
 p1 =x.elapsed[ss].plot(kind='hist', bins = int(x.elapsed.max() - x.elapsed.min() + 1))
p1 = plt.figure() ;
print(( ss ).sum())
 models.append(model_type(x[ss], y[ss], groups[ss], cv, **kwargs))
return models
x = x.astype(np.float32)
 BAG_MULT = 1
x.shape
lgb_c_clfs = []; lgb_c_noise = []
date_weights = np.abs(( model_data.Date_f - test.Date.min() ).dt.days)
 for iteration in range(0, int(math.ceil(1.1 * BAGS))):
for noise in [ 0.05, 0.1, 0.2, 0.3, 0.4 ]:
print("
---
Noise of {}".format(noise)) ;
num_bags = 1 * BAG_MULT;
if np.random.rand() < PLACE_FRACTION:
cv_group = places
print("CV by Place")
 else:
cv_group = groups
print("CV by Country")
 lgb_c_clfs.extend(runBags(x, y_cases,
cv_group,
MSE, num_bags, trainLGB, verbose = 0,
noise = noise, weights = date_weights 
))
lgb_c_noise.extend([noise] * num_bags)
 if SINGLE_MODEL:
break;
lgb_f_clfs = []; lgb_f_noise = []
for iteration in range(0, int(np.ceil(np.sqrt(BAGS)))) :
for noise in [ 0.5, 1, 2, 3, ]:
print("
---
Noise of {}".format(noise)) ;
num_bags = 1 * int(np.ceil(np.sqrt(BAG_MULT)))
 if np.random.rand() < PLACE_FRACTION :
cv_group = places
print("CV by Place")
 else:
cv_group = groups
print("CV by Country")
 lgb_f_clfs.extend(runBags(x, y_fatalities,
cv_group,
MSE, num_bags, trainLGB,
verbose = 0, noise = noise,
weights = date_weights 
))
lgb_f_noise.extend([noise] * num_bags)
 if SINGLE_MODEL:
break;
lgb_cfr_clfs = []; lgb_cfr_noise = [];
for iteration in range(0, int(np.ceil(np.sqrt(BAGS)))) :
for noise in [ 0.4, 1, 2, 3]:
print("
---
Noise of {}".format(noise)) ;
num_bags = 1 * BAG_MULT;
if np.random.rand() < 0.5 * PLACE_FRACTION :
cv_group = places
print("CV by Place")
 else:
cv_group = groups
print("CV by Country")
 lgb_cfr_clfs.extend(runBags(x, y_cfr,
cv_group,
MSE, num_bags, trainLGB, verbose = 0,
noise = noise,
weights = date_weights 
))
lgb_cfr_noise.extend([noise] * num_bags)
 if SINGLE_MODEL:
break;
lgb_cfr_clfs[0].predict(x_test)
 def show_FI(model, featNames, featCount):
fis = model.feature_importances_
fig, ax = plt.subplots(figsize=(6, 5))
indices = np.argsort(fis)[::-1][:featCount]
g = sns.barplot(y=featNames[indices][:featCount],
x = fis[indices][:featCount] , orient='h')
 g.set_xlabel("Relative importance")
 g.set_ylabel("Features")
 g.tick_params(labelsize=12)
 g.set_title(" feature importance")
 def avg_FI(all_clfs, featNames, featCount):
clfs = []
for clf_set in all_clfs:
for clf in clf_set:
clfs.append(clf);
print("{} classifiers".format(len(clfs)))
 fi = np.zeros(( len(clfs), len(clfs[0].feature_importances_)))
 for idx, clf in enumerate(clfs):
fi[idx, :] = clf.feature_importances_
avg_fi = np.mean(fi, axis = 0)
 fis = avg_fi
fig, ax = plt.subplots(figsize=(6, 5))
indices = np.argsort(fis)[::-1]
g = sns.barplot(y=featNames[indices][:featCount],
x = fis[indices][:featCount] , orient='h')
 g.set_xlabel("Relative importance")
 g.set_ylabel("Features")
 g.tick_params(labelsize=12)
 g.set_title(" feature importance")
 return pd.Series(fis[indices], featNames[indices])
 def linear_FI_plot(fi, featNames, featCount):
fig, ax = plt.subplots(figsize=(6, 5))
indices = np.argsort(np.absolute(fi)) [::-1]
g = sns.barplot(y=featNames[indices][:featCount],
x = fi[indices][:featCount] , orient='h')
 g.set_xlabel("Relative importance")
 g.set_ylabel("Features")
 g.tick_params(labelsize=12)
 g.set_title(" feature importance")
 return pd.Series(fi[indices], featNames[indices])
 f = avg_FI([lgb_c_clfs], x.columns, 25)
 for feat in ['bend', 'capita', 'cfr', 'slope', 'since', 'chg', 'ersonal',
'world', 'continent', 'nearest', 'surrounding']:
print("{}: {:.2f}".format(feat, f.filter(like=feat ).sum() / f.sum()))
f[:100:3]
print("{}: {:.2f}".format('sup_data',
f[[c for c in f.index if c in sup_data.columns]].sum() / f.sum()))
print("{}: {:.2f}".format('contain_data',
f[[c for c in f.index if c in contain_data.columns]].sum() / f.sum()))
f = avg_FI([lgb_f_clfs], x.columns, 25)
 for feat in ['bend', 'capita', 'cfr', 'slope', 'since', 'chg', 'ersonal',
'world', 'continent', 'nearest', 'surrounding']:
print("{}: {:.2f}".format(feat, f.filter(like=feat ).sum() / f.sum()))
print("{}: {:.2f}".format('sup_data',
f[[c for c in f.index if c in sup_data.columns]].sum() / f.sum()))
print("{}: {:.2f}".format('contain_data',
f[[c for c in f.index if c in contain_data.columns]].sum() / f.sum()))
f = avg_FI([lgb_cfr_clfs], x.columns, 25)
 for feat in ['bend', 'capita', 'cfr', 'slope', 'since', 'chg', 'ersonal',
'world', 'continent', 'nearest', 'surrounding']:
print("{}: {:.2f}".format(feat, f.filter(like=feat ).sum() / f.sum()))
print("{}: {:.2f}".format('sup_data',
f[[c for c in f.index if c in sup_data.columns]].sum() / f.sum()))
print("{}: {:.2f}".format('contain_data',
f[[c for c in f.index if c in contain_data.columns]].sum() / f.sum()))
all_c_clfs = [lgb_c_clfs, ]
all_f_clfs = [lgb_f_clfs]
all_cfr_clfs = [lgb_cfr_clfs]
all_c_noise = [lgb_c_noise]
all_f_noise = [lgb_f_noise]
all_cfr_noise = [lgb_cfr_noise]
NUM_TEST_RUNS = 1
c_preds = np.zeros(( NUM_TEST_RUNS * sum([len(x)for x in all_c_clfs]), len(x_test)))
 f_preds = np.zeros(( NUM_TEST_RUNS * sum([len(x)for x in all_f_clfs]), len(x_test)))
 cfr_preds = np.zeros(( NUM_TEST_RUNS * sum([len(x)for x in all_cfr_clfs]), len(x_test)))
 def avg(x):
return(np.mean(x, axis=0)+ np.median(x, axis=0)) /2
count = 0
for idx, clf in enumerate(lgb_c_clfs):
for i in range(0, NUM_TEST_RUNS):
noise = lgb_c_noise[idx]
c_preds[count,:] = np.clip(clf.predict(noisify(x_test, noise)) , -1 , 10)
 count += 1
count = 0
for idx, clf in enumerate(lgb_f_clfs):
for i in range(0, NUM_TEST_RUNS):
noise = lgb_f_noise[idx]
f_preds[count,:] = np.clip(clf.predict(noisify(x_test, noise)) , -1 , 10)
 count += 1
count = 0
for idx, clf in enumerate(lgb_cfr_clfs):
for i in range(0, NUM_TEST_RUNS):
noise = lgb_cfr_noise[idx]
cfr_preds[count,:] = np.clip(clf.predict(noisify(x_test, noise)) , -10 , 10)
 count += 1
def qPred(preds, pctile, simple=False):
q = np.percentile(preds, pctile, axis = 0)
 if simple:
return q;
resid = preds - q
resid_wtg = 2/100/len(preds)*(np.clip(resid, 0, None)*(pctile)\
+ np.clip(resid, None, 0)*(100- pctile))
adj = np.sum(resid_wtg, axis = 0)
 return q + adj
q = 50
y_cases_pred_blended_full = qPred(c_preds, q)
 y_fatalities_pred_blended_full = qPred(f_preds, q)
 y_cfr_pred_blended_full = qPred(cfr_preds, q)
 print(np.mean(np.corrcoef(c_preds[::NUM_TEST_RUNS]),axis=0))
print(np.mean(np.corrcoef(f_preds[::NUM_TEST_RUNS]), axis=0))
print(np.mean(np.corrcoef(cfr_preds[::NUM_TEST_RUNS]), axis = 0))
pd.Series(np.std(c_preds, axis = 0)).plot(kind='hist', bins = 50)
 pd.Series(np.std(f_preds, axis = 0)).plot(kind='hist', bins = 50)
 pd.Series(np.std(cfr_preds, axis = 0)).plot(kind='hist', bins = 50)
 y_cfr 
(groups == 'Sierra Leone' ).sum()
pred = pd.DataFrame(np.hstack(( np.transpose(c_preds),
np.transpose(f_preds))), index=x_test.index)
 pred['Place'] = data_test.Place
pred['Date'] = data_test.Date
pred['Date_f'] = data_test.Date_f
pred[(pred.Date == pred.Date.max())&(pred.Date_f == pred.Date_f.max())][30: 60] 
(pred.Place=='Sierra Leone' ).sum()
np.round(pred[(pred.Date == pred.Date.max())&(pred.Date_f == pred.Date_f.max())], 2)[190:220:]
np.round(pred[(pred.Date == pred.Date.max())&(pred.Date_f == pred.Date_f.max())][220:-20],2)
 c_preds.shape
x_test.shape
data_wp = data_test.copy()
if MODEL_Y is 'slope':
data_wp['case_slope'] = y_cases_pred_blended_full
data_wp['fatality_slope'] = y_fatalities_pred_blended_full
else:
data_wp['case_slope'] = y_cases_pred_blended_full / x_test.elapsed
data_wp['fatality_slope'] = y_fatalities_pred_blended_full / x_test.elapsed
data_wp['cfr_pred'] = y_cfr_pred_blended_full
train.Date.max()
test.Date.min()
if len(test)> 0:
base_date = test.Date.min() - datetime.timedelta(1)
 else:
base_date = train.Date.max()
base_date
data_wp_ss = data_wp[data_wp.Date == base_date]
data_wp_ss = data_wp_ss.drop(columns='Date' ).rename(columns = {'Date_f': 'Date'})
 test_wp = pd.merge(test, data_wp_ss[['Date', 'Place', 'case_slope', 'fatality_slope', 'cfr_pred',
'elapsed']],
how='left', on = ['Date', 'Place'])
 first_c_slope = test_wp[~test_wp.case_slope.isnull() ].groupby('Place' ).first()
last_c_slope = test_wp[~test_wp.case_slope.isnull() ].groupby('Place' ).last()
first_f_slope = test_wp[~test_wp.fatality_slope.isnull() ].groupby('Place' ).first()
last_f_slope = test_wp[~test_wp.fatality_slope.isnull() ].groupby('Place' ).last()
first_cfr_pred = test_wp[~test_wp.cfr_pred.isnull() ].groupby('Place' ).first()
last_cfr_pred = test_wp[~test_wp.cfr_pred.isnull() ].groupby('Place' ).last()
test_wp.case_slope = np.where(test_wp.case_slope.isnull() & 
(test_wp.Date < first_c_slope.loc[test_wp.Place].Date.values),
first_c_slope.loc[test_wp.Place].case_slope.values,
test_wp.case_slope 
)
 test_wp.case_slope = np.where(test_wp.case_slope.isnull() & 
(test_wp.Date > last_c_slope.loc[test_wp.Place].Date.values),
last_c_slope.loc[test_wp.Place].case_slope.values,
test_wp.case_slope 
)
 test_wp.fatality_slope = np.where(test_wp.fatality_slope.isnull() & 
(test_wp.Date < first_f_slope.loc[test_wp.Place].Date.values),
first_f_slope.loc[test_wp.Place].fatality_slope.values,
test_wp.fatality_slope 
)
 test_wp.fatality_slope = np.where(test_wp.fatality_slope.isnull() & 
(test_wp.Date > last_f_slope.loc[test_wp.Place].Date.values),
last_f_slope.loc[test_wp.Place].fatality_slope.values,
test_wp.fatality_slope 
)
 test_wp.cfr_pred = np.where(test_wp.cfr_pred.isnull() & 
(test_wp.Date < first_cfr_pred.loc[test_wp.Place].Date.values),
first_cfr_pred.loc[test_wp.Place].cfr_pred.values,
test_wp.cfr_pred 
)
 test_wp.cfr_pred = np.where(test_wp.cfr_pred.isnull() & 
(test_wp.Date > last_cfr_pred.loc[test_wp.Place].Date.values),
last_cfr_pred.loc[test_wp.Place].cfr_pred.values,
test_wp.cfr_pred 
)
 test_wp.case_slope = test_wp.case_slope.interpolate('linear')
 test_wp.fatality_slope = test_wp.fatality_slope.interpolate('linear')
 test_wp.cfr_pred = test_wp.cfr_pred.interpolate('linear')
 test_wp.case_slope = test_wp.case_slope.fillna(0)
 test_wp.fatality_slope = test_wp.fatality_slope.fillna(0)
 LAST_DATE = test.Date.min() - datetime.timedelta(1)
 final = train_bk[train_bk.Date == LAST_DATE ]
test_wp = pd.merge(test_wp, final[['Place', 'ConfirmedCases', 'Fatalities']], on='Place',
how ='left', validate='m:1')
 LAST_DATE
test_wp.ConfirmedCases = np.exp(
 np.log(test_wp.ConfirmedCases + 1)\
+ test_wp.case_slope * 
(test_wp.Date - LAST_DATE ).dt.days)- 1
test_wp.Fatalities = np.exp(
 np.log(test_wp.Fatalities + 1)\
+ test_wp.fatality_slope * 
(test_wp.Date - LAST_DATE ).dt.days)-1
LAST_DATE
test_wp[(test_wp.Country == 'Italy')].groupby('Date' ).sum() [:10]
test_wp[(test_wp.Country == 'US')].groupby('Date' ).sum().iloc[-5:]
final = train_bk[train_bk.Date == test.Date.min() - datetime.timedelta(1)]
final.head()
test['elapsed'] =(test.Date - final.Date.max() ).dt.days
test.elapsed
full_bk = test_wp.copy()
full = test_wp.copy()
BASE_RATE = 0.01
CFR_CAP = 0.13
lplot(full_bk)
 lplot(full_bk, columns = ['case_slope', 'fatality_slope'])
 full['cfr_imputed_fatalities_low'] = full.ConfirmedCases * np.exp(full.cfr_pred)/ np.exp(0.5)
 full['cfr_imputed_fatalities_high'] = full.ConfirmedCases * np.exp(full.cfr_pred)* np.exp(0.5)
 full['cfr_imputed_fatalities'] = full.ConfirmedCases * np.exp(full.cfr_pred)
 full[(full.case_slope > 0.02)& 
(full.Fatalities < full.cfr_imputed_fatalities_low)& 
(full.cfr_imputed_fatalities_low > 0.3)& 
(full.Fatalities < 100000)& 
(full.Country!='China')& 
(full.Date == datetime.datetime(2020, 4,15)) ] \ 
.groupby('Place' ).last() \ 
.sort_values('Fatalities', ascending=False ).iloc[:, 9:] 
(np.log(full.Fatalities + 1)-np.log(full.cfr_imputed_fatalities)).plot(kind='hist', bins = 250)
 full[(full.case_slope > 0.02)& 
(full.Fatalities < full.cfr_imputed_fatalities_low)& 
(full.cfr_imputed_fatalities_low > 0.3)& 
(full.Fatalities < 100000)& 
(~full.Country.isin(['China', 'Korea, South'])) ][full.Date==train.Date.max() ]\ 
.groupby('Place' ).first() \ 
.sort_values('cfr_imputed_fatalities', ascending=False ).iloc[:, 9:]
full.Fatalities = np.where(
(full.case_slope > 0.02)& 
(full.Fatalities <= full.cfr_imputed_fatalities_low)& 
(full.cfr_imputed_fatalities_low > 0.3)& 
(full.Fatalities < 100000)& 
(~full.Country.isin(['China', 'Korea, South'])) , 
(full.cfr_imputed_fatalities_high + full.cfr_imputed_fatalities)/2,
full.Fatalities)
 full['elapsed'] =(test_wp.Date - LAST_DATE ).dt.days
full[(full.case_slope > 0.02)& 
(np.log(full.Fatalities + 1)< np.log(full.ConfirmedCases * BASE_RATE + 1)- 0.5)& 
(full.Country != 'China')]\
[full.Date == datetime.datetime(2020, 4, 5)] \ 
.groupby('Place' ).last().sort_values('ConfirmedCases', ascending=False ).iloc[:,8:]
full.Fatalities = np.where(( full.case_slope > 0.02)& 
(full.Fatalities < full.ConfirmedCases * BASE_RATE)& 
(full.Country != 'China'),
np.exp(
 np.log(full.ConfirmedCases * BASE_RATE + 1)\
* np.clip(0.5*(full.elapsed - 1)/ 30, 0, 1)\
+ np.log(full.Fatalities +1)\
* np.clip(1 - 0.5*(full.elapsed - 1)/ 30, 0, 1)
)-1
,
full.Fatalities)
 full[(full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China')]\ 
.groupby('Place' ).count() \ 
.sort_values('ConfirmedCases', ascending=False ).iloc[:, 8:]
full[(full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high * 2)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China')]\ 
.groupby('Place' ).last() \ 
.sort_values('ConfirmedCases', ascending=False ).iloc[:, 8:]
full[(full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high * 1.5)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China')][full.Date==train.Date.max() ]\ 
.groupby('Place' ).first() \ 
.sort_values('ConfirmedCases', ascending=False ).iloc[:, 8:]
full.Fatalities = np.where(( full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high * 2)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China'),
full.cfr_imputed_fatalities,
full.Fatalities)
 full.Fatalities = np.where(( full.case_slope > 0.02)& 
(full.Fatalities > full.cfr_imputed_fatalities_high)& 
(full.cfr_imputed_fatalities_low > 0.4)& 
(full.Country!='China'),
np.exp(
 0.6667 * np.log(full.Fatalities + 1)\
+ 0.3333 * np.log(full.cfr_imputed_fatalities + 1)
)- 1,
full.Fatalities)
 full[(full.Fatalities > full.ConfirmedCases * CFR_CAP)& 
(full.ConfirmedCases > 1000)
 ].groupby('Place' ).last().sort_values('Fatalities', ascending=False)
(np.log(full.Fatalities + 1)-np.log(full.cfr_imputed_fatalities)).plot(kind='hist', bins = 250)
 assert len(pd.merge(full, final, on='Place', suffixes =('', '_i'), validate='m:1')) == len(full)
 ffm = pd.merge(full, final, on='Place', suffixes =('', '_i'), validate='m:1')
 ffm['fatality_slope'] =(np.log(ffm.Fatalities + 1)\
- np.log(ffm.Fatalities_i + 1)) \
/ ffm.elapsed
ffm['case_slope'] =(np.log(ffm.ConfirmedCases + 1)\
- np.log(ffm.ConfirmedCases_i + 1)) \
/ ffm.elapsed
ffm[np.log(ffm.Fatalities+1)< np.log(ffm.Fatalities_i+1)- 0.2]\
[['Place', 'Date', 'elapsed', 'Fatalities', 'Fatalities_i']]
ffm[np.log(ffm.ConfirmedCases + 1)< np.log(ffm.ConfirmedCases_i+1)- 0.2]\
[['Place', 'elapsed', 'ConfirmedCases', 'ConfirmedCases_i']]
full_bk[(full_bk.Date == test.Date.max())& 
(~full_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)
 lplot(ffm[~ffm.Place.isin(new_places)])
 lplot(ffm[~ffm.Place.isin(new_places)], columns = ['case_slope', 'fatality_slope'])
 ffm.fatality_slope = np.clip(ffm.fatality_slope, None, 0.5)
 for lr in [0.2, 0.14, 0.1, 0.07, 0.05, 0.03, 0.01 ]:
ffm.loc[(ffm.Place==ffm.Place.shift(4))
&(ffm.Place==ffm.Place.shift(-4)) , 'fatality_slope'] = \ 
(ffm.fatality_slope.shift(-2)* 0.25 \
+ ffm.fatality_slope.shift(-1)* 0.5 \
+ ffm.fatality_slope \
+ ffm.fatality_slope.shift(1)* 0.5 \
+ ffm.fatality_slope.shift(2)* 0.25)/ 2.5
ffm.ConfirmedCases = np.exp(
 np.log(ffm.ConfirmedCases_i + 1)\
+ ffm.case_slope *
ffm.elapsed)- 1
ffm.Fatalities = np.exp(
 np.log(ffm.Fatalities_i + 1)\
+ ffm.fatality_slope *
ffm.elapsed)- 1
lplot(ffm[~ffm.Place.isin(new_places)])
 lplot(ffm[~ffm.Place.isin(new_places)], columns = ['case_slope', 'fatality_slope'])
 ffm[(ffm.Date == test.Date.max())& 
(~ffm.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)
 ffm_bk = ffm.copy()
ffm = ffm_bk.copy()
counter = Counter(data.Place)
 median_count = np.median([ counter[group] for group in ffm.Place])
 c_count = [ np.clip(
 np.power(counter[group] / median_count, -1.5), None, 2.5)for group in ffm.Place]
RATE_MULT = 0.00
RATE_ADD = 0.003
LAG_FALLOFF = 15
ma_factor = np.clip(( ffm.elapsed - 14)/ 14 , 0, 1)
 ffm.case_slope = np.where(ffm.elapsed > 0,
0.7 * ffm.case_slope *(1+ ma_factor * RATE_MULT)\
+ 0.3 *(ffm.case_slope.ewm(span=LAG_FALLOFF ).mean() \
* np.clip(ma_factor, 0, 1)
 + ffm.case_slope * np.clip(1 - ma_factor, 0, 1))
+ RATE_ADD * ma_factor * c_count,
ffm.case_slope)
 RATE_MULT = 0
RATE_ADD = 0.015
LAG_FALLOFF = 15
ma_factor = np.clip(( ffm.elapsed - 10)/ 14 , 0, 1)
 ffm.fatality_slope = np.where(ffm.elapsed > 0,
0.3 * ffm.fatality_slope *(1+ ma_factor * RATE_MULT)\
+ 0.7*(ffm.fatality_slope.ewm(span=LAG_FALLOFF ).mean() \
* np.clip(ma_factor, 0, 1)
 + ffm.fatality_slope * np.clip(1 - ma_factor, 0, 1))
+ RATE_ADD * ma_factor * c_count \
*(ffm.Country != 'China')
 ,
ffm.case_slope)
 ffm.ConfirmedCases = np.exp(
 np.log(ffm.ConfirmedCases_i + 1)\
+ ffm.case_slope *
ffm.elapsed)- 1
ffm.Fatalities = np.exp(
 np.log(ffm.Fatalities_i + 1)\
+ ffm.fatality_slope *
ffm.elapsed)- 1
lplot(ffm[~ffm.Place.isin(new_places)])
 lplot(ffm[~ffm.Place.isin(new_places)], columns = ['case_slope', 'fatality_slope'])
 ffm_bk[(ffm_bk.Date == test.Date.max())& 
(~ffm_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[:15]
ffm[(ffm.Date == test.Date.max())& 
(~ffm.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[:15]
ffm_bk[(ffm_bk.Date == test.Date.max())& 
(~ffm_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[-50:]
ffm[(ffm.Date == test.Date.max())& 
(~ffm.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).loc[ffm_bk[(ffm_bk.Date == test.Date.max())& 
(~ffm_bk.Place.isin(new_places)) ].groupby('Country' ).agg(
 {'ForecastId': 'count',
'case_slope': 'last',
'fatality_slope': 'last',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
}
).sort_values('ConfirmedCases', ascending=False)[-50:].index]
NUM_TEST_DATES = len(test.Date.unique())
 base = np.zeros(( 2, NUM_TEST_DATES))
base2 = np.zeros(( 2, NUM_TEST_DATES))
for idx, c in enumerate(['ConfirmedCases', 'Fatalities']):
for n in range(0, NUM_TEST_DATES):
base[idx,n] = np.mean(
 np.log(train[(( train.Date < test.Date.min())) & 
(train.ConfirmedCases > 0)].groupby('Country' ).nth(n)[c]+1))
base = np.pad(base,(( 0,0),(6,0)) , mode='constant', constant_values = 0)
 for n in range(0, base2.shape[1]):
base2[:, n] = np.mean(base[:, n+0: n+7], axis = 1)
 new_places = train[(train.Date == test.Date.min() - datetime.timedelta(1)) & 
(train.ConfirmedCases == 0)
 ].Place
ffm.ConfirmedCases = \
np.where(ffm.Place.isin(new_places),
base2[ 0,(ffm.Date - test.Date.min() ).dt.days],
ffm.ConfirmedCases)
 ffm.Fatalities = \
np.where(ffm.Place.isin(new_places),
base2[ 1,(ffm.Date - test.Date.min() ).dt.days],
ffm.Fatalities)
 ffm[ffm.Country=='US'].groupby('Date' ).agg(
 {'ForecastId': 'count',
'case_slope': 'mean',
'fatality_slope': 'mean',
'ConfirmedCases': 'sum',
'Fatalities': 'sum',
})
 sub = pd.read_csv(input_path + '/submission.csv')
 scl = sub.columns.to_list()
if ffm[scl].isnull().sum().sum() == 0:
out = full_bk[scl] * 0.3 + ffm_bk[scl] * 0.3 + full[scl] * 0.3 + ffm[scl] * 0.1
else:
print('using full-bk')
 out = full_bk[scl]
out.ForecastId = np.round(out.ForecastId, 0 ).astype(int)
 print(pd.merge(out, test[['ForecastId', 'Date', 'Place']], on='ForecastId')\ 
.sort_values('ForecastId')\ 
.groupby('Place' ).last() [['Date', 'ConfirmedCases', 'Fatalities']])
 out = np.round(out, 2)
 private = out[sub.columns.to_list() ]
full_pred = pd.concat(( private, public[~public.ForecastId.isin(private.ForecastId)]),
ignore_index=True ).sort_values('ForecastId')
 full_pred.to_csv('submission3.csv', index=False )
train_sample = pd.read_csv(
 TRAIN_SAMPLE_FILE,
dtype=dtypes,
usecols=['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed'],
compression = None,
header=0,
engine='c' 
)
 train_sample.head()<load_from_csv>
gc.collect()
print('Loading data...')
 start_time = datetime.now()
train = pd.read_csv(
 TRAIN_FILE,
dtype=dtypes,
usecols=['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed'],
compression = COMPRESSION,
header=0,
skiprows=SKIP_ROWS,
engine='c' 
)
 test = pd.read_csv(
 TEST_FILE,
dtype=dtypes,
usecols=['ip', 'app', 'device', 'os', 'channel', 'click_time', 'click_id'],
compression = COMPRESSION,
header=0,
engine='c' 
)
 print('End of Loading Data: {time_taken}'.format(time_taken=datetime.now() - start_time))<feature_engineering>
!pip install tensorflow_addons
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
def swishE(x):
beta = 1.75
return beta * x * tf.keras.backend.sigmoid(x)
 def swish(x):
return x * tf.keras.backend.sigmoid(x)
 def phrishII(x):
return x*tf.keras.backend.tanh(1.75 * x * tf.keras.backend.sigmoid(x))
def mish(x):
return x*tf.keras.backend.tanh(tf.keras.backend.softplus(x))
def gelu_new(x):

 cdf = 0.5 *(1.0 + tf.tanh(
(np.sqrt(2 / np.pi)*(x + 0.044715 * tf.pow(x, 3)))))
 return x * cdf
print(len(train))
class Outliers() :
data = None
def __init__(self, data):

 self.data = data
def treat_outliers_by_iqr(self, colName, what_sd=1.5):
mean_value = np.mean(self.data[colName].values)
 sd_value = np.std(self.data[colName].values)
 max_value = mean_value + what_sd * sd_value
min_value = mean_value - what_sd * sd_value
new_data = self.data[(self.data[colName] >= min_value)&(self.data[colName] <= max_value)]
outliers = self.data[(self.data[colName] > max_value)|(self.data[colName] < min_value)]
print('%Outliers: {outlier_percentage:<8} Mean: {mean_value:<8} SD: {sd_value:<8} IQR Min: {iqr_min:<8} IQR Max: {iqr_max:<8} Min: {mini:<8} Max: {maxi:<8} Outliers: {outliers:<8}
outlier_percentage=np.around(( len(outliers)/len(self.data)) *100, decimals=2),
mean_value=np.around(mean_value, decimals=2), sd_value=np.around(sd_value, decimals=2),
iqr_min=np.around(min_value, decimals=2),
iqr_max=np.around(max_value, decimals=2),
mini=np.around(min(self.data[colName].values), decimals=2),
maxi=np.around(max(self.data[colName].values), decimals=2),
outliers=len(outliers),
app_count=len(new_data.app.unique()),
size=len(new_data)
))
return outliers, new_data
if(TREAT_OUTLIERS == True):
print('Remove Outliers...')
 start_time = datetime.now()
outs = Outliers(train)
 outliers, train = outs.treat_outliers_by_iqr('is_attributed')
 print('End of Outlier Removal: {time_taken}'.format(time_taken=datetime.now() - start_time))<concatenate>
def get_ridgeCV_sub(save_oof=False, save_public_test=False):
train = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test
day_min = train['day'].min()
train['day'] -= day_min
test['day'] -= day_min
min_test_val_day = test.day.min()
max_test_val_day = train.day.max()
max_test_day = test.day.max()
num_days = max_test_day + 1
min_test_val_day, max_test_val_day, num_days
train['ForecastId'] = -1
test['Id'] = -1
test['ConfirmedCases'] = 0
test['Fatalities'] = 0
debug = False
data = pd.concat([train,
test[test.day > max_test_val_day][train.columns]
] ).reset_index(drop=True)
 if debug:
data = data[data['geo'] >= 'France_'].reset_index(drop=True)
 gc.collect()
dates = data[data['geo'] == 'France_'].Date.values
if 0:
gr = data.groupby('geo')
 data['ConfirmedCases'] = gr.ConfirmedCases.transform('cummax')
 data['Fatalities'] = gr.Fatalities.transform('cummax')
 geo_data = data.pivot(index='geo', columns='day', values='ForecastId')
 num_geo = geo_data.shape[0]
geo_data
geo_id = {}
for i,g in enumerate(geo_data.index):
geo_id[g] = i
ConfirmedCases = data.pivot(index='geo', columns='day', values='ConfirmedCases')
 Fatalities = data.pivot(index='geo', columns='day', values='Fatalities')
 if debug:
cases = ConfirmedCases.values
deaths = Fatalities.values
else:
cases = np.log1p(ConfirmedCases.values)
 deaths = np.log1p(Fatalities.values)
 def get_dataset(start_pred, num_train, lag_period):
days = np.arange(start_pred - num_train + 1, start_pred + 1)
 lag_cases = np.vstack([cases[:, d - lag_period : d] for d in days])
 lag_deaths = np.vstack([deaths[:, d - lag_period : d] for d in days])
 target_cases = np.vstack([cases[:, d : d + 1] for d in days])
 target_deaths = np.vstack([deaths[:, d : d + 1] for d in days])
 geo_ids = np.vstack([geo_ids_base for d in days])
 country_ids = np.vstack([country_ids_base for d in days])
 return lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days
def update_valid_dataset(data, pred_death, pred_case):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
day = days[-1] + 1
new_lag_cases = np.hstack([lag_cases[:, 1:], pred_case])
 new_lag_deaths = np.hstack([lag_deaths[:, 1:], pred_death])
 new_target_cases = cases[:, day:day+1]
new_target_deaths = deaths[:, day:day+1]
new_geo_ids = geo_ids
new_country_ids = country_ids
new_days = 1 + days
return new_lag_cases, new_lag_deaths, new_target_cases, new_target_deaths, new_geo_ids, new_country_ids, new_days
def fit_eval(lr_death, lr_case, data, start_lag_death, end_lag_death, num_lag_case, fit, score):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], country_ids])
 X_death = np.hstack([lag_deaths[:, -num_lag_case:], country_ids])
 X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], lag_deaths[:, -num_lag_case:], country_ids])
 y_death = target_deaths
y_death_prev = lag_deaths[:, -1:]
if fit:
if 0:
keep =(y_death > 0 ).ravel()
X_death = X_death[keep]
y_death = y_death[keep]
y_death_prev = y_death_prev[keep]
lr_death.fit(X_death, y_death)
 y_pred_death = lr_death.predict(X_death)
 y_pred_death = np.maximum(y_pred_death, y_death_prev)
 X_case = np.hstack([lag_cases[:, -num_lag_case:], geo_ids])
 X_case = lag_cases[:, -num_lag_case:]
y_case = target_cases
y_case_prev = lag_cases[:, -1:]
if fit:
lr_case.fit(X_case, y_case)
 y_pred_case = lr_case.predict(X_case)
 y_pred_case = np.maximum(y_pred_case, y_case_prev)
 if score:
death_score = val_score(y_death, y_pred_death)
 case_score = val_score(y_case, y_pred_case)
 else:
death_score = 0
case_score = 0
return death_score, case_score, y_pred_death, y_pred_case
def train_model(train, valid, start_lag_death, end_lag_death, num_lag_case, num_val, score=True):
alphas_alt = [3, 6, 7, 8, 10, 11, 14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
kfolds = KFold(n_splits=5, shuffle=True, random_state=42)
 lr_death = RidgeCV(alphas=alphas_alt, fit_intercept=False)
 lr_case = RidgeCV(alphas=alphas_alt, fit_intercept=True)
(train_death_score, train_case_score, train_pred_death, train_pred_case, 
)= fit_eval(lr_death, lr_case, train, start_lag_death, end_lag_death, num_lag_case, fit=True, score=score)
 death_scores = []
case_scores = []
death_pred = []
case_pred = []
for i in range(num_val): 
(valid_death_score, valid_case_score, valid_pred_death, valid_pred_case, 
)= fit_eval(lr_death, lr_case, valid, start_lag_death, end_lag_death, num_lag_case, fit=False, score=score)
 death_scores.append(valid_death_score)
 case_scores.append(valid_case_score)
 death_pred.append(valid_pred_death)
 case_pred.append(valid_pred_case)
 if 0:
print('val death: %0.3f' % valid_death_score,
'val case: %0.3f' % valid_case_score,
'val : %0.3f' % np.mean([valid_death_score, valid_case_score]),
flush=True)
 valid = update_valid_dataset(valid, valid_pred_death, valid_pred_case)
 if score:
death_scores = np.sqrt(np.mean([s**2 for s in death_scores]))
case_scores = np.sqrt(np.mean([s**2 for s in case_scores]))
if 0:
print('train death: %0.3f' % train_death_score,
'train case: %0.3f' % train_case_score,
'val death: %0.3f' % death_scores,
'val case: %0.3f' % case_scores,
'val : %0.3f' %(( death_scores + case_scores)/ 2),
flush=True)
 else:
print('%0.4f' % case_scores,
', %0.4f' % death_scores,
'= %0.4f' %(( death_scores + case_scores)/ 2),
flush=True)
 death_pred = np.hstack(death_pred)
 case_pred = np.hstack(case_pred)
 return death_scores, case_scores, death_pred, case_pred
countries = [g.split('_')[0] for g in geo_data.index]
countries = pd.factorize(countries)[0]
country_ids_base = countries.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 country_ids_base = 0.2 * ohe.fit_transform(country_ids_base)
 country_ids_base.shape
geo_ids_base = np.arange(num_geo ).reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 geo_ids_base = 0.1 * ohe.fit_transform(geo_ids_base)
 geo_ids_base.shape
def val_score(true, pred):
pred = np.log1p(np.round(np.expm1(pred)- 0.2))
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def val_score(true, pred):
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
start_lag_death, end_lag_death = 14, 6,
num_train = 5
num_lag_case = 14
lag_period = max(start_lag_death, num_lag_case)
 def get_oof(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[start_val], start_val, num_val)
 train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = train[['Date', 'Id', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[sub.day >= start_val]
sub = sub[['Id', 'ConfirmedCases', 'Fatalities']].copy()
return sub
if save_oof:
for start_val_delta, date in zip(range(3, -8, -3),
['2020-04-01', '2020-03-29', '2020-03-26', '2020-03-23']):
print(date, end=' ')
 oof = get_oof(start_val_delta)
 oof.to_csv('.. /submissions/cpmp-%s.csv' % date, index=None)
 def get_sub(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = 14
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub.fillna(0)
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
return sub
return sub
known_test = train[['geo', 'day', 'ConfirmedCases', 'Fatalities']
].merge(test[['geo', 'day', 'ForecastId']], how='left', on=['geo', 'day'])
 known_test = known_test[['ForecastId', 'ConfirmedCases', 'Fatalities']][known_test.ForecastId.notnull() ].copy()
known_test
unknow_test = test[test.day > max_test_val_day]
unknow_test
def get_final_sub() :
start_val = max_test_val_day + 1
last_train = start_val - 1
num_val = max_test_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = num_val + 3
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
(_, _, val_death_preds, val_case_preds 
)= train_model(train_data, valid_data, start_lag_death, end_lag_death, num_lag_case, num_val, score=False)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
print(unknow_test.shape, pred_deaths.shape, pred_cases.shape)
 sub = unknow_test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
sub = pd.concat([known_test, sub])
 return sub
if save_public_test:
sub = get_sub()
else:
sub = get_final_sub()
return sub
train_size = len(train)
 train = train.append(test)
 del test
gc.collect()<data_type_conversions>
gc.collect()
print('Making Hour, Day and Weekday Columns...')
 start_time = datetime.now()
date_time = pd.to_datetime(train.click_time)
 train['hour'] = date_time.dt.hour.astype('uint8')
 train['day'] = date_time.dt.day.astype('uint8')
 train['wday'] = date_time.dt.dayofweek.astype('uint8')
 print('End of Making HDWd: {time_taken}'.format(time_taken=datetime.now() - start_time))<merge>
def get_nn_sub() :
df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 sub_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 coo_df = pd.read_csv(".. /input/covid19week1/train.csv" ).rename(columns={"Country/Region": "Country_Region"})
 coo_df = coo_df.groupby("Country_Region")[["Lat", "Long"]].mean().reset_index()
coo_df = coo_df[coo_df["Country_Region"].notnull() ]
loc_group = ["Province_State", "Country_Region"]
def preprocess(df):
df["Date"] = df["Date"].astype("datetime64[ms]")
 df["days"] =(df["Date"] - pd.to_datetime("2020-01-01")).dt.days
df["weekend"] = df["Date"].dt.dayofweek//5
df = df.merge(coo_df, how="left", on="Country_Region")
 df["Lat"] =(df["Lat"] // 30 ).astype(np.float32 ).fillna(0)
 df["Long"] =(df["Long"] // 60 ).astype(np.float32 ).fillna(0)
 for col in loc_group:
df[col].fillna("none", inplace=True)
 return df
df = preprocess(df)
 sub_df = preprocess(sub_df)
 print(df.shape)
 TARGETS = ["ConfirmedCases", "Fatalities"]
for col in TARGETS:
df[col] = np.log1p(df[col])
 NUM_SHIFT = 5
features = ["Lat", "Long"]
for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
df["prev_{}_{}".format(col, s)] = df.groupby(loc_group)[col].shift(s)
 features.append("prev_{}_{}".format(col, s))
df = df[df["Date"] >= df["Date"].min() + timedelta(days=NUM_SHIFT)].copy()
TEST_FIRST = sub_df["Date"].min()
TEST_DAYS =(df["Date"].max() - TEST_FIRST ).days + 1
dev_df, test_df = df[df["Date"] < TEST_FIRST].copy() , df[df["Date"] >= TEST_FIRST].copy()
def nn_block(input_layer, size, dropout_rate, activation):
out_layer = KL.Dense(size, activation=None )(input_layer)
 out_layer = KL.Activation(activation )(out_layer)
 out_layer = KL.Dropout(dropout_rate )(out_layer)
 return out_layer
def get_model1() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swishE )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model2() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation='linear' )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model3() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model4() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, mish)
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, mish)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model5() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model6() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, phrishII)
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, phrishII)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model7() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
model1 = get_model1()
model2 = get_model2()
model3 = get_model3()
model4 = get_model4()
model5 = get_model5()
model6 = get_model6()
model7 = get_model7()
inp = KL.Input(shape=(len(features),))
y1 = model1(inp)
 y2 = model2(inp)
 y3 = model3(inp)
 y4 = model4(inp)
 y5 = model5(inp)
 y6 = model6(inp)
 y7 = model7(inp)
 outputs = tf.keras.layers.average([y1, y2, y3, y4, y5, y6, y7])
 ensemble_model = tf.keras.Model(inputs=inp, outputs=outputs)
 ensemble_model.summary()
plot_model(ensemble_model, to_file='model.png',show_shapes=True, show_layer_names=True, expand_nested=True)
 def get_input(df):
return [df[features]]
NUM_MODELS = 10
def train_models(df, save=False):
models = []
for i in range(NUM_MODELS):
model = ensemble_model
model.compile(loss="mean_squared_error", optimizer= tfa.optimizers.Lookahead(tfa.optimizers.AdamW(lr=0.005,weight_decay=6.924e-5), sync_period=15, slow_step_size=0.5))
if save:
model.load_weights(".. /input/nn10final/model_1_{}.h5".format(i))
else:
model.load_weights(".. /input/nn10final/model_1_dev_{}.h5".format(i))
models.append(model)
 return models
models = train_models(dev_df)
 prev_targets = ['prev_ConfirmedCases_1', 'prev_Fatalities_1']
def predict_one(df, models):
pred = np.zeros(( df.shape[0], 2))
for model in models:
pred += model.predict(get_input(df)) /len(models)
 pred = np.maximum(pred, df[prev_targets].values)
 pred[:, 0] = np.log1p(np.expm1(pred[:, 0])+ 0.1)
 pred[:, 1] = np.log1p(np.expm1(pred[:, 1])+ 0.01)
 return np.clip(pred, None, 15)
 print([mean_squared_error(dev_df[TARGETS[i]], predict_one(dev_df, models)[:, i])for i in range(len(TARGETS)) ])
 def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
def evaluate(df):
error = 0
for col in TARGETS:
error += rmse(df[col].values, df["pred_{}".format(col)].values)
 return np.round(error/len(TARGETS), 5)
 def predict(test_df, first_day, num_days, models, val=False):
temp_df = test_df.loc[test_df["Date"] == first_day].copy()
y_pred = predict_one(temp_df, models)
 for i, col in enumerate(TARGETS):
test_df["pred_{}".format(col)] = 0
test_df.loc[test_df["Date"] == first_day, "pred_{}".format(col)] = y_pred[:, i]
print(first_day, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 if val:
print(evaluate(test_df[test_df["Date"] == first_day]))
y_prevs = [None]*NUM_SHIFT
for i in range(1, NUM_SHIFT):
y_prevs[i] = temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]].values
for d in range(1, num_days):
date = first_day + timedelta(days=d)
 print(date, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 temp_df = test_df.loc[test_df["Date"] == date].copy()
temp_df[prev_targets] = y_pred
for i in range(2, NUM_SHIFT+1):
temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]] = y_prevs[i-1]
y_pred, y_prevs = predict_one(temp_df, models), [None, y_pred] + y_prevs[1:-1]
for i, col in enumerate(TARGETS):
test_df.loc[test_df["Date"] == date, "pred_{}".format(col)] = y_pred[:, i]
if val:
print(evaluate(test_df[test_df["Date"] == date]))
return test_df
test_df = predict(test_df, TEST_FIRST, TEST_DAYS, models, val=True)
 print(evaluate(test_df))
for col in TARGETS:
test_df[col] = np.expm1(test_df[col])
 test_df["pred_{}".format(col)] = np.expm1(test_df["pred_{}".format(col)])
 models = train_models(df, save=True)
 sub_df_public = sub_df[sub_df["Date"] <= df["Date"].max() ].copy()
sub_df_private = sub_df[sub_df["Date"] > df["Date"].max() ].copy()
pred_cols = ["pred_{}".format(col)for col in TARGETS]
sub_df_public = sub_df_public.merge(test_df[["Date"] + loc_group + TARGETS], how="left", on=["Date"] + loc_group)
 SUB_FIRST = sub_df_private["Date"].min()
SUB_DAYS =(sub_df_private["Date"].max() - sub_df_private["Date"].min() ).days + 1
sub_df_private = df.append(sub_df_private, sort=False)
 for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
sub_df_private["prev_{}_{}".format(col, s)] = sub_df_private.groupby(loc_group)[col].shift(s)
 sub_df_private = sub_df_private[sub_df_private["Date"] >= SUB_FIRST].copy()
sub_df_private = predict(sub_df_private, SUB_FIRST, SUB_DAYS, models)
 for col in TARGETS:
sub_df_private[col] = np.expm1(sub_df_private["pred_{}".format(col)])
 sub_df = sub_df_public.append(sub_df_private, sort=False)
 sub_df["ForecastId"] = sub_df["ForecastId"].astype(np.int16)
 return sub_df[["ForecastId"] + TARGETS]
def get_nn_sub2() :
df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 sub_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 coo_df = pd.read_csv(".. /input/covid19week1/train.csv" ).rename(columns={"Country/Region": "Country_Region"})
 coo_df = coo_df.groupby("Country_Region")[["Lat", "Long"]].mean().reset_index()
coo_df = coo_df[coo_df["Country_Region"].notnull() ]
loc_group = ["Province_State", "Country_Region"]
def preprocess(df):
df["Date"] = df["Date"].astype("datetime64[ms]")
 df["days"] =(df["Date"] - pd.to_datetime("2020-01-01")).dt.days
df["weekend"] = df["Date"].dt.dayofweek//5
df = df.merge(coo_df, how="left", on="Country_Region")
 df["Lat"] =(df["Lat"] // 30 ).astype(np.float32 ).fillna(0)
 df["Long"] =(df["Long"] // 60 ).astype(np.float32 ).fillna(0)
 for col in loc_group:
df[col].fillna("none", inplace=True)
 return df
df = preprocess(df)
 sub_df = preprocess(sub_df)
 print(df.shape)
 TARGETS = ["ConfirmedCases", "Fatalities"]
for col in TARGETS:
df[col] = np.log1p(df[col])
 NUM_SHIFT = 5
features = ["Lat", "Long"]
for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
df["prev_{}_{}".format(col, s)] = df.groupby(loc_group)[col].shift(s)
 features.append("prev_{}_{}".format(col, s))
df = df[df["Date"] >= df["Date"].min() + timedelta(days=NUM_SHIFT)].copy()
TEST_FIRST = sub_df["Date"].min()
TEST_DAYS =(df["Date"].max() - TEST_FIRST ).days + 1
dev_df, test_df = df[df["Date"] < TEST_FIRST].copy() , df[df["Date"] >= TEST_FIRST].copy()
def nn_block(input_layer, size, dropout_rate, activation):
out_layer = KL.Dense(size, activation=None )(input_layer)
 out_layer = KL.Activation(activation )(out_layer)
 out_layer = KL.Dropout(dropout_rate )(out_layer)
 return out_layer
def get_model1() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swishE )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model2() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation='linear' )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model3() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model4() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, mish)
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, mish)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model5() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model6() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, phrishII)
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, phrishII)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model7() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
model1 = get_model1()
model2 = get_model2()
model3 = get_model3()
model4 = get_model4()
model5 = get_model5()
model6 = get_model6()
model7 = get_model7()
inp = KL.Input(shape=(len(features),))
y1 = model1(inp)
 y2 = model2(inp)
 y3 = model3(inp)
 y4 = model4(inp)
 y5 = model5(inp)
 y6 = model6(inp)
 y7 = model7(inp)
 outputs = tf.keras.layers.average([y1, y2, y3, y4, y5, y6, y7])
 ensemble_model = tf.keras.Model(inputs=inp, outputs=outputs)
 ensemble_model.summary()
plot_model(ensemble_model, to_file='model.png',show_shapes=True, show_layer_names=True, expand_nested=True)
 def get_input(df):
return [df[features]]
NUM_MODELS = 10
def train_models(df, save=False):
models = []
for i in range(NUM_MODELS):
model = ensemble_model
model.compile(loss="mean_squared_error", optimizer=tf.keras.optimizers.RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08, decay=0.0))
if save:
model.load_weights(".. /input/nn10final/model_2_{}.h5".format(i))
else:
model.load_weights(".. /input/nn10final/model_2_Dev_{}.h5".format(i))
models.append(model)
 return models
models = train_models(dev_df)
 prev_targets = ['prev_ConfirmedCases_1', 'prev_Fatalities_1']
def predict_one(df, models):
pred = np.zeros(( df.shape[0], 2))
for model in models:
pred += model.predict(get_input(df)) /len(models)
 pred = np.maximum(pred, df[prev_targets].values)
 pred[:, 0] = np.log1p(np.expm1(pred[:, 0])+ 0.1)
 pred[:, 1] = np.log1p(np.expm1(pred[:, 1])+ 0.01)
 return np.clip(pred, None, 15)
 print([mean_squared_error(dev_df[TARGETS[i]], predict_one(dev_df, models)[:, i])for i in range(len(TARGETS)) ])
 def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
def evaluate(df):
error = 0
for col in TARGETS:
error += rmse(df[col].values, df["pred_{}".format(col)].values)
 return np.round(error/len(TARGETS), 5)
 def predict(test_df, first_day, num_days, models, val=False):
temp_df = test_df.loc[test_df["Date"] == first_day].copy()
y_pred = predict_one(temp_df, models)
 for i, col in enumerate(TARGETS):
test_df["pred_{}".format(col)] = 0
test_df.loc[test_df["Date"] == first_day, "pred_{}".format(col)] = y_pred[:, i]
print(first_day, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 if val:
print(evaluate(test_df[test_df["Date"] == first_day]))
y_prevs = [None]*NUM_SHIFT
for i in range(1, NUM_SHIFT):
y_prevs[i] = temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]].values
for d in range(1, num_days):
date = first_day + timedelta(days=d)
 print(date, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 temp_df = test_df.loc[test_df["Date"] == date].copy()
temp_df[prev_targets] = y_pred
for i in range(2, NUM_SHIFT+1):
temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]] = y_prevs[i-1]
y_pred, y_prevs = predict_one(temp_df, models), [None, y_pred] + y_prevs[1:-1]
for i, col in enumerate(TARGETS):
test_df.loc[test_df["Date"] == date, "pred_{}".format(col)] = y_pred[:, i]
if val:
print(evaluate(test_df[test_df["Date"] == date]))
return test_df
test_df = predict(test_df, TEST_FIRST, TEST_DAYS, models, val=True)
 print(evaluate(test_df))
for col in TARGETS:
test_df[col] = np.expm1(test_df[col])
 test_df["pred_{}".format(col)] = np.expm1(test_df["pred_{}".format(col)])
 models = train_models(df, save=True)
 sub_df_public = sub_df[sub_df["Date"] <= df["Date"].max() ].copy()
sub_df_private = sub_df[sub_df["Date"] > df["Date"].max() ].copy()
pred_cols = ["pred_{}".format(col)for col in TARGETS]
sub_df_public = sub_df_public.merge(test_df[["Date"] + loc_group + TARGETS], how="left", on=["Date"] + loc_group)
 SUB_FIRST = sub_df_private["Date"].min()
SUB_DAYS =(sub_df_private["Date"].max() - sub_df_private["Date"].min() ).days + 1
sub_df_private = df.append(sub_df_private, sort=False)
 for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
sub_df_private["prev_{}_{}".format(col, s)] = sub_df_private.groupby(loc_group)[col].shift(s)
 sub_df_private = sub_df_private[sub_df_private["Date"] >= SUB_FIRST].copy()
sub_df_private = predict(sub_df_private, SUB_FIRST, SUB_DAYS, models)
 for col in TARGETS:
sub_df_private[col] = np.expm1(sub_df_private["pred_{}".format(col)])
 sub_df = sub_df_public.append(sub_df_private, sort=False)
 sub_df["ForecastId"] = sub_df["ForecastId"].astype(np.int16)
 return sub_df[["ForecastId"] + TARGETS]
sub2 = get_nn_sub()
sub4 = get_nn_sub2()
sub3 = get_ridgeCV_sub()
sub3['ForecastId'] = sub3['ForecastId'].astype('int')
 sub2.sort_values("ForecastId", inplace=True)
 sub4.sort_values("ForecastId", inplace=True)
 sub3.sort_values("ForecastId", inplace=True)
 TARGETS = ["ConfirmedCases", "Fatalities"]
[np.sqrt(mean_squared_error(np.log1p(sub3[t].values), np.log1p(sub2[t].values)))for t in TARGETS]
[np.sqrt(mean_squared_error(np.log1p(sub3[t].values), np.log1p(sub4[t].values)))for t in TARGETS]
sub_df = sub3.copy()
sub_df_final = sub3.copy()
for t in TARGETS:
sub_df[t] = np.expm1(np.log1p(sub2[t].values)*0.3 + np.log1p(sub3[t].values)*0.4 + np.log1p(sub4[t].values)*0.3)
 sub_df.to_csv("submission1.csv", index=False )
print('Making number of channels by IP, Day, HOUR...')
 group_ip_dhc = train[['ip', 'day', 'hour', 'channel']].groupby(by=['ip', 'day', 'hour'])
 group_ip_dhc = group_ip_dhc[['channel']].count().reset_index().rename(index=str, columns={'channel': 'qty'})
 train = train.merge(group_ip_dhc, on=['ip', 'day', 'hour'], how='left')
 del group_ip_dhc
gc.collect
train.head()
print('End of Making channel count by IDH: {time_taken}'.format(time_taken=datetime.now() - start_time))<merge>
gc.collect()
print('Making number of channels by IP, APP...')
 start_time = datetime.now()
group_ip_ac = train[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])
 group_ip_ac = group_ip_ac[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})
 train = train.merge(group_ip_ac, on=['ip', 'app'], how='left')
 del group_ip_ac
gc.collect()
train.head()
print('End of Making channel count by IA: {time_taken}'.format(time_taken=datetime.now() - start_time))<merge>
def get_nn_sub() :
df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 sub_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 coo_df = pd.read_csv(".. /input/covid19week1/train.csv" ).rename(columns={"Country/Region": "Country_Region"})
 coo_df = coo_df.groupby("Country_Region")[["Lat", "Long"]].mean().reset_index()
coo_df = coo_df[coo_df["Country_Region"].notnull() ]
loc_group = ["Province_State", "Country_Region"]
def preprocess(df):
df["Date"] = df["Date"].astype("datetime64[ms]")
 df["days"] =(df["Date"] - pd.to_datetime("2020-01-01")).dt.days
df["weekend"] = df["Date"].dt.dayofweek//5
df = df.merge(coo_df, how="left", on="Country_Region")
 df["Lat"] =(df["Lat"] // 30 ).astype(np.float32 ).fillna(0)
 df["Long"] =(df["Long"] // 60 ).astype(np.float32 ).fillna(0)
 for col in loc_group:
df[col].fillna("none", inplace=True)
 return df
df = preprocess(df)
 sub_df = preprocess(sub_df)
 print(df.shape)
 TARGETS = ["ConfirmedCases", "Fatalities"]
for col in TARGETS:
df[col] = np.log1p(df[col])
 NUM_SHIFT = 5
features = ["Lat", "Long"]
for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
df["prev_{}_{}".format(col, s)] = df.groupby(loc_group)[col].shift(s)
 features.append("prev_{}_{}".format(col, s))
df = df[df["Date"] >= df["Date"].min() + timedelta(days=NUM_SHIFT)].copy()
TEST_FIRST = sub_df["Date"].min()
TEST_DAYS =(df["Date"].max() - TEST_FIRST ).days + 1
dev_df, test_df = df[df["Date"] < TEST_FIRST].copy() , df[df["Date"] >= TEST_FIRST].copy()
def nn_block(input_layer, size, dropout_rate, activation):
out_layer = KL.Dense(size, activation=None )(input_layer)
 out_layer = KL.Activation(activation )(out_layer)
 out_layer = KL.Dropout(dropout_rate )(out_layer)
 return out_layer
def get_model1() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swishE )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model2() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation='linear' )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model3() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model4() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, mish)
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, mish)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model5() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model6() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, phrishII)
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, phrishII)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model7() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
model1 = get_model1()
model2 = get_model2()
model3 = get_model3()
model4 = get_model4()
model5 = get_model5()
model6 = get_model6()
model7 = get_model7()
inp = KL.Input(shape=(len(features),))
y1 = model1(inp)
 y2 = model2(inp)
 y3 = model3(inp)
 y4 = model4(inp)
 y5 = model5(inp)
 y6 = model6(inp)
 y7 = model7(inp)
 outputs = tf.keras.layers.average([y1, y2, y3, y4, y5, y6, y7])
 ensemble_model = tf.keras.Model(inputs=inp, outputs=outputs)
 ensemble_model.summary()
plot_model(ensemble_model, to_file='model.png',show_shapes=True, show_layer_names=True, expand_nested=True)
 def get_input(df):
return [df[features]]
NUM_MODELS = 40
def train_models(df, save=False):
models = []
for i in range(NUM_MODELS):
model = ensemble_model
model.compile(loss="mean_squared_error", optimizer= tfa.optimizers.Lookahead(tfa.optimizers.AdamW(lr=0.005,weight_decay=6.924e-5), sync_period=15, slow_step_size=0.5))
if save:
model.load_weights(".. /input/n40final/model_1_{}.h5".format(i))
else:
model.load_weights(".. /input/n40final/model_1_dev_{}.h5".format(i))
models.append(model)
 return models
models = train_models(dev_df)
 prev_targets = ['prev_ConfirmedCases_1', 'prev_Fatalities_1']
def predict_one(df, models):
pred = np.zeros(( df.shape[0], 2))
for model in models:
pred += model.predict(get_input(df)) /len(models)
 pred = np.maximum(pred, df[prev_targets].values)
 pred[:, 0] = np.log1p(np.expm1(pred[:, 0])+ 0.1)
 pred[:, 1] = np.log1p(np.expm1(pred[:, 1])+ 0.01)
 return np.clip(pred, None, 15)
 print([mean_squared_error(dev_df[TARGETS[i]], predict_one(dev_df, models)[:, i])for i in range(len(TARGETS)) ])
 def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
def evaluate(df):
error = 0
for col in TARGETS:
error += rmse(df[col].values, df["pred_{}".format(col)].values)
 return np.round(error/len(TARGETS), 5)
 def predict(test_df, first_day, num_days, models, val=False):
temp_df = test_df.loc[test_df["Date"] == first_day].copy()
y_pred = predict_one(temp_df, models)
 for i, col in enumerate(TARGETS):
test_df["pred_{}".format(col)] = 0
test_df.loc[test_df["Date"] == first_day, "pred_{}".format(col)] = y_pred[:, i]
print(first_day, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 if val:
print(evaluate(test_df[test_df["Date"] == first_day]))
y_prevs = [None]*NUM_SHIFT
for i in range(1, NUM_SHIFT):
y_prevs[i] = temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]].values
for d in range(1, num_days):
date = first_day + timedelta(days=d)
 print(date, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 temp_df = test_df.loc[test_df["Date"] == date].copy()
temp_df[prev_targets] = y_pred
for i in range(2, NUM_SHIFT+1):
temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]] = y_prevs[i-1]
y_pred, y_prevs = predict_one(temp_df, models), [None, y_pred] + y_prevs[1:-1]
for i, col in enumerate(TARGETS):
test_df.loc[test_df["Date"] == date, "pred_{}".format(col)] = y_pred[:, i]
if val:
print(evaluate(test_df[test_df["Date"] == date]))
return test_df
test_df = predict(test_df, TEST_FIRST, TEST_DAYS, models, val=True)
 print(evaluate(test_df))
for col in TARGETS:
test_df[col] = np.expm1(test_df[col])
 test_df["pred_{}".format(col)] = np.expm1(test_df["pred_{}".format(col)])
 models = train_models(df, save=True)
 sub_df_public = sub_df[sub_df["Date"] <= df["Date"].max() ].copy()
sub_df_private = sub_df[sub_df["Date"] > df["Date"].max() ].copy()
pred_cols = ["pred_{}".format(col)for col in TARGETS]
sub_df_public = sub_df_public.merge(test_df[["Date"] + loc_group + TARGETS], how="left", on=["Date"] + loc_group)
 SUB_FIRST = sub_df_private["Date"].min()
SUB_DAYS =(sub_df_private["Date"].max() - sub_df_private["Date"].min() ).days + 1
sub_df_private = df.append(sub_df_private, sort=False)
 for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
sub_df_private["prev_{}_{}".format(col, s)] = sub_df_private.groupby(loc_group)[col].shift(s)
 sub_df_private = sub_df_private[sub_df_private["Date"] >= SUB_FIRST].copy()
sub_df_private = predict(sub_df_private, SUB_FIRST, SUB_DAYS, models)
 for col in TARGETS:
sub_df_private[col] = np.expm1(sub_df_private["pred_{}".format(col)])
 sub_df = sub_df_public.append(sub_df_private, sort=False)
 sub_df["ForecastId"] = sub_df["ForecastId"].astype(np.int16)
 return sub_df[["ForecastId"] + TARGETS]
def get_nn_sub2() :
df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 sub_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 coo_df = pd.read_csv(".. /input/covid19week1/train.csv" ).rename(columns={"Country/Region": "Country_Region"})
 coo_df = coo_df.groupby("Country_Region")[["Lat", "Long"]].mean().reset_index()
coo_df = coo_df[coo_df["Country_Region"].notnull() ]
loc_group = ["Province_State", "Country_Region"]
def preprocess(df):
df["Date"] = df["Date"].astype("datetime64[ms]")
 df["days"] =(df["Date"] - pd.to_datetime("2020-01-01")).dt.days
df["weekend"] = df["Date"].dt.dayofweek//5
df = df.merge(coo_df, how="left", on="Country_Region")
 df["Lat"] =(df["Lat"] // 30 ).astype(np.float32 ).fillna(0)
 df["Long"] =(df["Long"] // 60 ).astype(np.float32 ).fillna(0)
 for col in loc_group:
df[col].fillna("none", inplace=True)
 return df
df = preprocess(df)
 sub_df = preprocess(sub_df)
 print(df.shape)
 TARGETS = ["ConfirmedCases", "Fatalities"]
for col in TARGETS:
df[col] = np.log1p(df[col])
 NUM_SHIFT = 5
features = ["Lat", "Long"]
for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
df["prev_{}_{}".format(col, s)] = df.groupby(loc_group)[col].shift(s)
 features.append("prev_{}_{}".format(col, s))
df = df[df["Date"] >= df["Date"].min() + timedelta(days=NUM_SHIFT)].copy()
TEST_FIRST = sub_df["Date"].min()
TEST_DAYS =(df["Date"].max() - TEST_FIRST ).days + 1
dev_df, test_df = df[df["Date"] < TEST_FIRST].copy() , df[df["Date"] >= TEST_FIRST].copy()
def nn_block(input_layer, size, dropout_rate, activation):
out_layer = KL.Dense(size, activation=None )(input_layer)
 out_layer = KL.Activation(activation )(out_layer)
 out_layer = KL.Dropout(dropout_rate )(out_layer)
 return out_layer
def get_model1() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swishE )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model2() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation='linear' )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model3() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model4() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, mish)
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, mish)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model5() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model6() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, phrishII)
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, phrishII)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_model7() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, gelu_new)
 gate_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = nn_block(hidden_layer, 32, 0.0, gelu_new)
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation=swish )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
model1 = get_model1()
model2 = get_model2()
model3 = get_model3()
model4 = get_model4()
model5 = get_model5()
model6 = get_model6()
model7 = get_model7()
inp = KL.Input(shape=(len(features),))
y1 = model1(inp)
 y2 = model2(inp)
 y3 = model3(inp)
 y4 = model4(inp)
 y5 = model5(inp)
 y6 = model6(inp)
 y7 = model7(inp)
 outputs = tf.keras.layers.average([y1, y2, y3, y4, y5, y6, y7])
 ensemble_model = tf.keras.Model(inputs=inp, outputs=outputs)
 ensemble_model.summary()
plot_model(ensemble_model, to_file='model.png',show_shapes=True, show_layer_names=True, expand_nested=True)
 def get_input(df):
return [df[features]]
NUM_MODELS = 40
def train_models(df, save=False):
models = []
for i in range(NUM_MODELS):
model = ensemble_model
model.compile(loss="mean_squared_error", optimizer=tf.keras.optimizers.RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08, decay=0.0))
if save:
model.load_weights(".. /input/n40final/model_2_{}.h5".format(i))
else:
model.load_weights(".. /input/n40final/model_2_Dev_{}.h5".format(i))
models.append(model)
 return models
models = train_models(dev_df)
 prev_targets = ['prev_ConfirmedCases_1', 'prev_Fatalities_1']
def predict_one(df, models):
pred = np.zeros(( df.shape[0], 2))
for model in models:
pred += model.predict(get_input(df)) /len(models)
 pred = np.maximum(pred, df[prev_targets].values)
 pred[:, 0] = np.log1p(np.expm1(pred[:, 0])+ 0.1)
 pred[:, 1] = np.log1p(np.expm1(pred[:, 1])+ 0.01)
 return np.clip(pred, None, 15)
 print([mean_squared_error(dev_df[TARGETS[i]], predict_one(dev_df, models)[:, i])for i in range(len(TARGETS)) ])
 def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
def evaluate(df):
error = 0
for col in TARGETS:
error += rmse(df[col].values, df["pred_{}".format(col)].values)
 return np.round(error/len(TARGETS), 5)
 def predict(test_df, first_day, num_days, models, val=False):
temp_df = test_df.loc[test_df["Date"] == first_day].copy()
y_pred = predict_one(temp_df, models)
 for i, col in enumerate(TARGETS):
test_df["pred_{}".format(col)] = 0
test_df.loc[test_df["Date"] == first_day, "pred_{}".format(col)] = y_pred[:, i]
print(first_day, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 if val:
print(evaluate(test_df[test_df["Date"] == first_day]))
y_prevs = [None]*NUM_SHIFT
for i in range(1, NUM_SHIFT):
y_prevs[i] = temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]].values
for d in range(1, num_days):
date = first_day + timedelta(days=d)
 print(date, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 temp_df = test_df.loc[test_df["Date"] == date].copy()
temp_df[prev_targets] = y_pred
for i in range(2, NUM_SHIFT+1):
temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]] = y_prevs[i-1]
y_pred, y_prevs = predict_one(temp_df, models), [None, y_pred] + y_prevs[1:-1]
for i, col in enumerate(TARGETS):
test_df.loc[test_df["Date"] == date, "pred_{}".format(col)] = y_pred[:, i]
if val:
print(evaluate(test_df[test_df["Date"] == date]))
return test_df
test_df = predict(test_df, TEST_FIRST, TEST_DAYS, models, val=True)
 print(evaluate(test_df))
for col in TARGETS:
test_df[col] = np.expm1(test_df[col])
 test_df["pred_{}".format(col)] = np.expm1(test_df["pred_{}".format(col)])
 models = train_models(df, save=True)
 sub_df_public = sub_df[sub_df["Date"] <= df["Date"].max() ].copy()
sub_df_private = sub_df[sub_df["Date"] > df["Date"].max() ].copy()
pred_cols = ["pred_{}".format(col)for col in TARGETS]
sub_df_public = sub_df_public.merge(test_df[["Date"] + loc_group + TARGETS], how="left", on=["Date"] + loc_group)
 SUB_FIRST = sub_df_private["Date"].min()
SUB_DAYS =(sub_df_private["Date"].max() - sub_df_private["Date"].min() ).days + 1
sub_df_private = df.append(sub_df_private, sort=False)
 for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
sub_df_private["prev_{}_{}".format(col, s)] = sub_df_private.groupby(loc_group)[col].shift(s)
 sub_df_private = sub_df_private[sub_df_private["Date"] >= SUB_FIRST].copy()
sub_df_private = predict(sub_df_private, SUB_FIRST, SUB_DAYS, models)
 for col in TARGETS:
sub_df_private[col] = np.expm1(sub_df_private["pred_{}".format(col)])
 sub_df = sub_df_public.append(sub_df_private, sort=False)
 sub_df["ForecastId"] = sub_df["ForecastId"].astype(np.int16)
 return sub_df[["ForecastId"] + TARGETS]
sub2 = get_nn_sub()
sub4 = get_nn_sub2()
sub2.sort_values("ForecastId", inplace=True)
 sub4.sort_values("ForecastId", inplace=True)
 TARGETS = ["ConfirmedCases", "Fatalities"]
[np.sqrt(mean_squared_error(np.log1p(sub3[t].values), np.log1p(sub2[t].values)))for t in TARGETS]
[np.sqrt(mean_squared_error(np.log1p(sub3[t].values), np.log1p(sub4[t].values)))for t in TARGETS]
sub_df40 = sub3.copy()
for t in TARGETS:
sub_df40[t] = np.expm1(np.log1p(sub2[t].values)*0.3 + np.log1p(sub3[t].values)*0.4 + np.log1p(sub4[t].values)*0.3)
 sub_df40.to_csv("submission50.csv", index=False )
print('Making number of channels by IP, APP, OS...')
 start_time = datetime.now()
group_ip_aoc = train[['ip', 'app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])
 group_ip_aoc = group_ip_aoc[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})
 train = train.merge(group_ip_aoc, on=['ip', 'app', 'os'], how='left')
 del group_ip_aoc
gc.collect()
print('End of Making channel count by IAO: {time_taken}'.format(time_taken=datetime.now() - start_time))<import_modules>
gc.collect()
from sklearn.preprocessing import LabelEncoder<categorify>
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
def get_cpmp_sub(save_oof=False, save_public_test=False):
train = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test
day_min = train['day'].min()
train['day'] -= day_min
test['day'] -= day_min
min_test_val_day = test.day.min()
max_test_val_day = train.day.max()
max_test_day = test.day.max()
num_days = max_test_day + 1
min_test_val_day, max_test_val_day, num_days
train['ForecastId'] = -1
test['Id'] = -1
test['ConfirmedCases'] = 0
test['Fatalities'] = 0
debug = False
data = pd.concat([train,
test[test.day > max_test_val_day][train.columns]
] ).reset_index(drop=True)
 if debug:
data = data[data['geo'] >= 'France_'].reset_index(drop=True)
 gc.collect()
dates = data[data['geo'] == 'France_'].Date.values
if 0:
gr = data.groupby('geo')
 data['ConfirmedCases'] = gr.ConfirmedCases.transform('cummax')
 data['Fatalities'] = gr.Fatalities.transform('cummax')
 geo_data = data.pivot(index='geo', columns='day', values='ForecastId')
 num_geo = geo_data.shape[0]
geo_data
geo_id = {}
for i,g in enumerate(geo_data.index):
geo_id[g] = i
ConfirmedCases = data.pivot(index='geo', columns='day', values='ConfirmedCases')
 Fatalities = data.pivot(index='geo', columns='day', values='Fatalities')
 if debug:
cases = ConfirmedCases.values
deaths = Fatalities.values
else:
cases = np.log1p(ConfirmedCases.values)
 deaths = np.log1p(Fatalities.values)
 def get_dataset(start_pred, num_train, lag_period):
days = np.arange(start_pred - num_train + 1, start_pred + 1)
 lag_cases = np.vstack([cases[:, d - lag_period : d] for d in days])
 lag_deaths = np.vstack([deaths[:, d - lag_period : d] for d in days])
 target_cases = np.vstack([cases[:, d : d + 1] for d in days])
 target_deaths = np.vstack([deaths[:, d : d + 1] for d in days])
 geo_ids = np.vstack([geo_ids_base for d in days])
 country_ids = np.vstack([country_ids_base for d in days])
 return lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days
def update_valid_dataset(data, pred_death, pred_case):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
day = days[-1] + 1
new_lag_cases = np.hstack([lag_cases[:, 1:], pred_case])
 new_lag_deaths = np.hstack([lag_deaths[:, 1:], pred_death])
 new_target_cases = cases[:, day:day+1]
new_target_deaths = deaths[:, day:day+1]
new_geo_ids = geo_ids
new_country_ids = country_ids
new_days = 1 + days
return new_lag_cases, new_lag_deaths, new_target_cases, new_target_deaths, new_geo_ids, new_country_ids, new_days
def fit_eval(lr_death, lr_case, data, start_lag_death, end_lag_death, num_lag_case, fit, score):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], country_ids])
 X_death = np.hstack([lag_deaths[:, -num_lag_case:], country_ids])
 X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], lag_deaths[:, -num_lag_case:], country_ids])
 y_death = target_deaths
y_death_prev = lag_deaths[:, -1:]
if fit:
if 0:
keep =(y_death > 0 ).ravel()
X_death = X_death[keep]
y_death = y_death[keep]
y_death_prev = y_death_prev[keep]
lr_death.fit(X_death, y_death)
 pickle.dump(lr_death, open("lr_death.pickle.dat", "wb"))
y_pred_death = lr_death.predict(X_death)
 y_pred_death = np.maximum(y_pred_death, y_death_prev)
 X_case = np.hstack([lag_cases[:, -num_lag_case:], geo_ids])
 X_case = lag_cases[:, -num_lag_case:]
y_case = target_cases
y_case_prev = lag_cases[:, -1:]
if fit:
lr_case.fit(X_case, y_case)
 pickle.dump(lr_case, open("lr_case.pickle.dat", "wb"))
y_pred_case = lr_case.predict(X_case)
 y_pred_case = np.maximum(y_pred_case, y_case_prev)
 if score:
death_score = val_score(y_death, y_pred_death)
 case_score = val_score(y_case, y_pred_case)
 else:
death_score = 0
case_score = 0
return death_score, case_score, y_pred_death, y_pred_case
def train_model(train, valid, start_lag_death, end_lag_death, num_lag_case, num_val, score=True):
alpha = 2
lr_death = Ridge(alpha=alpha, fit_intercept=False)
 lr_case = Ridge(alpha=alpha, fit_intercept=True)
(train_death_score, train_case_score, train_pred_death, train_pred_case, 
)= fit_eval(lr_death, lr_case, train, start_lag_death, end_lag_death, num_lag_case, fit=True, score=score)
 death_scores = []
case_scores = []
death_pred = []
case_pred = []
for i in range(num_val): 
(valid_death_score, valid_case_score, valid_pred_death, valid_pred_case, 
)= fit_eval(lr_death, lr_case, valid, start_lag_death, end_lag_death, num_lag_case, fit=False, score=score)
 death_scores.append(valid_death_score)
 case_scores.append(valid_case_score)
 death_pred.append(valid_pred_death)
 case_pred.append(valid_pred_case)
 if 0:
print('val death: %0.3f' % valid_death_score,
'val case: %0.3f' % valid_case_score,
'val : %0.3f' % np.mean([valid_death_score, valid_case_score]),
flush=True)
 valid = update_valid_dataset(valid, valid_pred_death, valid_pred_case)
 if score:
death_scores = np.sqrt(np.mean([s**2 for s in death_scores]))
case_scores = np.sqrt(np.mean([s**2 for s in case_scores]))
if 0:
print('train death: %0.3f' % train_death_score,
'train case: %0.3f' % train_case_score,
'val death: %0.3f' % death_scores,
'val case: %0.3f' % case_scores,
'val : %0.3f' %(( death_scores + case_scores)/ 2),
flush=True)
 else:
print('%0.4f' % case_scores,
', %0.4f' % death_scores,
'= %0.4f' %(( death_scores + case_scores)/ 2),
flush=True)
 death_pred = np.hstack(death_pred)
 case_pred = np.hstack(case_pred)
 return death_scores, case_scores, death_pred, case_pred
countries = [g.split('_')[0] for g in geo_data.index]
countries = pd.factorize(countries)[0]
country_ids_base = countries.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 country_ids_base = 0.2 * ohe.fit_transform(country_ids_base)
 country_ids_base.shape
geo_ids_base = np.arange(num_geo ).reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 geo_ids_base = 0.1 * ohe.fit_transform(geo_ids_base)
 geo_ids_base.shape
def val_score(true, pred):
pred = np.log1p(np.round(np.expm1(pred)- 0.2))
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def val_score(true, pred):
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
start_lag_death, end_lag_death = 14, 6,
num_train = 6
num_lag_case = 14
lag_period = max(start_lag_death, num_lag_case)
 def get_oof(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[start_val], start_val, num_val)
 train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = train[['Date', 'Id', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[sub.day >= start_val]
sub = sub[['Id', 'ConfirmedCases', 'Fatalities']].copy()
return sub
if save_oof:
for start_val_delta, date in zip(range(3, -8, -3),
['2020-03-22', '2020-03-19', '2020-03-16', '2020-03-13']):
print(date, end=' ')
 oof = get_oof(start_val_delta)
 oof.to_csv('.. /submissions/cpmp-%s.csv' % date, index=None)
 def get_sub(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = 14
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub.fillna(0)
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
return sub
return sub
known_test = train[['geo', 'day', 'ConfirmedCases', 'Fatalities']
].merge(test[['geo', 'day', 'ForecastId']], how='left', on=['geo', 'day'])
 known_test = known_test[['ForecastId', 'ConfirmedCases', 'Fatalities']][known_test.ForecastId.notnull() ].copy()
known_test
unknow_test = test[test.day > max_test_val_day]
unknow_test
def get_final_sub() :
start_val = max_test_val_day + 1
last_train = start_val - 1
num_val = max_test_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = num_val + 3
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
(_, _, val_death_preds, val_case_preds 
)= train_model(train_data, valid_data, start_lag_death, end_lag_death, num_lag_case, num_val, score=False)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
print(unknow_test.shape, pred_deaths.shape, pred_cases.shape)
 sub = unknow_test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
sub = pd.concat([known_test, sub])
 return sub
if save_public_test:
sub = get_sub()
else:
sub = get_final_sub()
return sub
print('Label encoding and fit transformation...')
 start_time = datetime.now()
train[['app', 'device', 'os', 'channel', 'hour', 'day', 'wday']].apply(LabelEncoder().fit_transform)
 print('End of Making channel count by IAO: {time_taken}'.format(time_taken=datetime.now() - start_time)) 
<prepare_x_and_y>
gc.collect()
print('Splitting train and test data...')
 start_time = datetime.now()
test = train[train_size:]
print(len(test))
train = train[:train_size]
y_train = train['is_attributed'].values
print(len(train))
train.drop(['click_id', 'click_time', 'ip', 'is_attributed'], 1, inplace=True)
 print('End of Splitting Train and Test Dataset: {time_taken}'.format(time_taken=datetime.now() - start_time))<import_modules>
def get_nn_sub3() :
df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 sub_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 coo_df = pd.read_csv(".. /input/covid19week1/train.csv" ).rename(columns={"Country/Region": "Country_Region"})
 coo_df = coo_df.groupby("Country_Region")[["Lat", "Long"]].mean().reset_index()
coo_df = coo_df[coo_df["Country_Region"].notnull() ]
loc_group = ["Province_State", "Country_Region"]
def preprocess(df):
df["Date"] = df["Date"].astype("datetime64[ms]")
 df["days"] =(df["Date"] - pd.to_datetime("2020-01-01")).dt.days
df["weekend"] = df["Date"].dt.dayofweek//5
df = df.merge(coo_df, how="left", on="Country_Region")
 df["Lat"] =(df["Lat"] // 30 ).astype(np.float32 ).fillna(0)
 df["Long"] =(df["Long"] // 60 ).astype(np.float32 ).fillna(0)
 for col in loc_group:
df[col].fillna("none", inplace=True)
 return df
df = preprocess(df)
 sub_df = preprocess(sub_df)
 print(df.shape)
 TARGETS = ["ConfirmedCases", "Fatalities"]
for col in TARGETS:
df[col] = np.log1p(df[col])
 NUM_SHIFT = 5
features = ["Lat", "Long"]
for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
df["prev_{}_{}".format(col, s)] = df.groupby(loc_group)[col].shift(s)
 features.append("prev_{}_{}".format(col, s))
df = df[df["Date"] >= df["Date"].min() + timedelta(days=NUM_SHIFT)].copy()
TEST_FIRST = sub_df["Date"].min()
TEST_DAYS =(df["Date"].max() - TEST_FIRST ).days + 1
dev_df, test_df = df[df["Date"] < TEST_FIRST].copy() , df[df["Date"] >= TEST_FIRST].copy()
def nn_block(input_layer, size, dropout_rate, activation):
out_layer = KL.Dense(size, activation=None )(input_layer)
 out_layer = KL.Activation(activation )(out_layer)
 out_layer = KL.Dropout(dropout_rate )(out_layer)
 return out_layer
def get_model() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 208, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 104, 0.0, "hard_sigmoid")
 hidden_layer = nn_block(hidden_layer, 104, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
get_model().summary()
def get_input(df):
return [df[features]]
NUM_MODELS = 100
def train_models(df, save=False):
models = []
for i in range(NUM_MODELS):
model = get_model()
model.compile(loss="mean_squared_error", optimizer=Nadam(lr=1e-4))
if save:
model.load_weights(".. /input/nn100final/model_2_{}.h5".format(i))
else:
model.load_weights(".. /input/nn100final/model_2_Dev_{}.h5".format(i))
models.append(model)
 return models
models = train_models(dev_df)
 prev_targets = ['prev_ConfirmedCases_1', 'prev_Fatalities_1']
def predict_one(df, models):
pred = np.zeros(( df.shape[0], 2))
for model in models:
pred += model.predict(get_input(df)) /len(models)
 pred = np.maximum(pred, df[prev_targets].values)
 pred[:, 0] = np.log1p(np.expm1(pred[:, 0])+ 0.1)
 pred[:, 1] = np.log1p(np.expm1(pred[:, 1])+ 0.01)
 return np.clip(pred, None, 15)
 print([mean_squared_error(dev_df[TARGETS[i]], predict_one(dev_df, models)[:, i])for i in range(len(TARGETS)) ])
 def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
def evaluate(df):
error = 0
for col in TARGETS:
error += rmse(df[col].values, df["pred_{}".format(col)].values)
 return np.round(error/len(TARGETS), 5)
 def predict(test_df, first_day, num_days, models, val=False):
temp_df = test_df.loc[test_df["Date"] == first_day].copy()
y_pred = predict_one(temp_df, models)
 for i, col in enumerate(TARGETS):
test_df["pred_{}".format(col)] = 0
test_df.loc[test_df["Date"] == first_day, "pred_{}".format(col)] = y_pred[:, i]
print(first_day, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 if val:
print(evaluate(test_df[test_df["Date"] == first_day]))
y_prevs = [None]*NUM_SHIFT
for i in range(1, NUM_SHIFT):
y_prevs[i] = temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]].values
for d in range(1, num_days):
date = first_day + timedelta(days=d)
 print(date, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 temp_df = test_df.loc[test_df["Date"] == date].copy()
temp_df[prev_targets] = y_pred
for i in range(2, NUM_SHIFT+1):
temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]] = y_prevs[i-1]
y_pred, y_prevs = predict_one(temp_df, models), [None, y_pred] + y_prevs[1:-1]
for i, col in enumerate(TARGETS):
test_df.loc[test_df["Date"] == date, "pred_{}".format(col)] = y_pred[:, i]
if val:
print(evaluate(test_df[test_df["Date"] == date]))
return test_df
test_df = predict(test_df, TEST_FIRST, TEST_DAYS, models, val=True)
 print(evaluate(test_df))
for col in TARGETS:
test_df[col] = np.expm1(test_df[col])
 test_df["pred_{}".format(col)] = np.expm1(test_df["pred_{}".format(col)])
 models = train_models(df, save=True)
 sub_df_public = sub_df[sub_df["Date"] <= df["Date"].max() ].copy()
sub_df_private = sub_df[sub_df["Date"] > df["Date"].max() ].copy()
pred_cols = ["pred_{}".format(col)for col in TARGETS]
sub_df_public = sub_df_public.merge(test_df[["Date"] + loc_group + TARGETS], how="left", on=["Date"] + loc_group)
 SUB_FIRST = sub_df_private["Date"].min()
SUB_DAYS =(sub_df_private["Date"].max() - sub_df_private["Date"].min() ).days + 1
sub_df_private = df.append(sub_df_private, sort=False)
 for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
sub_df_private["prev_{}_{}".format(col, s)] = sub_df_private.groupby(loc_group)[col].shift(s)
 sub_df_private = sub_df_private[sub_df_private["Date"] >= SUB_FIRST].copy()
sub_df_private = predict(sub_df_private, SUB_FIRST, SUB_DAYS, models)
 for col in TARGETS:
sub_df_private[col] = np.expm1(sub_df_private["pred_{}".format(col)])
 sub_df = sub_df_public.append(sub_df_private, sort=False)
 sub_df["ForecastId"] = sub_df["ForecastId"].astype(np.int16)
 return sub_df[["ForecastId"] + TARGETS]
sub20 = get_nn_sub3()
sub10 = get_cpmp_sub()
sub10['ForecastId'] = sub10['ForecastId'].astype('int')
 sub10.sort_values("ForecastId", inplace=True)
 sub20.sort_values("ForecastId", inplace=True)
 TARGETS = ["ConfirmedCases", "Fatalities"]
[np.sqrt(mean_squared_error(np.log1p(sub10[t].values), np.log1p(sub20[t].values)))for t in TARGETS]
sub_df20 = sub10.copy()
for t in TARGETS:
sub_df20[t] = np.expm1(np.log1p(sub10[t].values)*0.5 + np.log1p(sub20[t].values)*0.5 )
from keras.layers import Input, Embedding, Dense, Flatten, Dropout, concatenate
from keras.layers import BatchNormalization, SpatialDropout1D, GRU
from keras.callbacks import Callback
from keras.models import Model
from keras.optimizers import Adam<define_variables>
gc.collect()
max_app = np.max([train['app'].max() , test['app'].max() ])+ 1
max_channel = np.max([train['channel'].max() , test['channel'].max() ])+ 1
max_device = np.max([train['device'].max() , test['device'].max() ])+ 1
max_os = np.max([train['os'].max() , test['os'].max() ])+ 1
max_hour = np.max([train['hour'].max() , test['hour'].max() ])+ 1
max_day = np.max([train['day'].max() , test['day'].max() ])+ 1
max_wday = np.max([train['wday'].max() , test['wday'].max() ])+ 1
max_qty = np.max([train['qty'].max() , test['qty'].max() ])+ 1
max_c1 = np.max([train['ip_app_count'].max() , test['ip_app_count'].max() ])+ 1
max_c2 = np.max([train['ip_app_os_count'].max() , test['ip_app_os_count'].max() ])+ 1 
<prepare_x_and_y>
train = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 train['Date'] = pd.to_datetime(train['Date'])
 def dealing_with_null_values(dataset):
dataset = dataset
for i in dataset.columns:
replace = []
data = dataset[i].isnull()
count = 0
for j,k in zip(data,dataset[i]):
if(j==True):
count = count+1
replace.append('No Information Available')
 else:
replace.append(k)
 print("Num of null values(",i,"):",count)
 dataset[i] = replace
return dataset
train = dealing_with_null_values(train)
 def fillState(state, country):
if state == 'No Information Available': return country
return state
train['Province_State'] = train.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 train.loc[:, 'Date'] = train.Date.dt.strftime("%m%d")
 train["Date"] = train["Date"].astype(int)
 le = preprocessing.LabelEncoder()
train.Country_Region = le.fit_transform(train.Country_Region)
 train.Province_State = le.fit_transform(train.Province_State)
 data = pd.DataFrame()
data['Province_State']=train['Province_State']
data['Country_Region']=train['Country_Region']
data['Date']=train['Date']
xmodel1 = XGBRegressor(n_estimators=1000)
 xmodel1.fit(data,train['ConfirmedCases'])
 xmodel2 = XGBRegressor(n_estimators=1000)
 xmodel2.fit(data,train['Fatalities'])
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-4/test.csv')
 test['Date'] = pd.to_datetime(test['Date'], infer_datetime_format=True)
 test = dealing_with_null_values(test)
 test['Province_State'] = test.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 test.loc[:, 'Date'] = test.Date.dt.strftime("%m%d")
 test["Date"] = test["Date"].astype(int)
 le = preprocessing.LabelEncoder()
test.Country_Region = le.fit_transform(test.Country_Region)
 test.Province_State = le.fit_transform(test.Province_State)
 test_data = pd.DataFrame()
test_data['Province_State'] = test['Province_State']
test_data['Country_Region'] = test['Country_Region']
test_data['Date'] = test['Date']
filterwarnings('ignore')
 output = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for i in test['Country_Region'].unique() :
s = test[test['Country_Region'] == i].Province_State.unique()
for j in s:
output_train = data[(data['Country_Region'] == i)&(data['Province_State'] == j)]
output_test = test_data[(test_data['Country_Region']==i)&(test_data['Province_State'] == j)]
output_train_with_labels = train[(train['Country_Region'] == i)&(train['Province_State'] == j)]
index_test = test[(test['Country_Region']==i)&(test['Province_State'] == j)]
index_test = index_test['ForecastId']
xmodel1 = XGBRegressor(n_estimators=2000)
 xmodel1.fit(output_train,output_train_with_labels['ConfirmedCases'])
 xmodel2 = XGBRegressor(n_estimators=2000)
 xmodel2.fit(output_train,output_train_with_labels['Fatalities'])
 xmodel11 = XGBRegressor(n_estimators=1500)
 xmodel11.fit(output_train,output_train_with_labels['ConfirmedCases'])
 xmodel22 = XGBRegressor(n_estimators=1500)
 xmodel22.fit(output_train,output_train_with_labels['Fatalities'])
 xmodel111 = XGBRegressor(n_estimators=2500)
 xmodel111.fit(output_train,output_train_with_labels['ConfirmedCases'])
 xmodel222 = XGBRegressor(n_estimators=2500)
 xmodel222.fit(output_train,output_train_with_labels['Fatalities'])
 xmodel1111 = XGBRegressor(n_estimators = 2000, num_parallel_tree=2)
 xmodel1111.fit(output_train,output_train_with_labels['ConfirmedCases'])
 xmodel2222 = XGBRegressor(n_estimators = 2000, num_parallel_tree=2)
 xmodel2222.fit(output_train,output_train_with_labels['Fatalities'])
 y1_xpred_output =.25*xmodel1.predict(output_test)+.25*xmodel11.predict(output_test)+.25*xmodel111.predict(output_test)+.25*xmodel1111.predict(output_test)
 y2_xpred_output =.25*xmodel2.predict(output_test)+.25*xmodel22.predict(output_test)+.25*xmodel222.predict(output_test)+.25*xmodel2222.predict(output_test)
 for_output = pd.DataFrame()
for_output['ForecastId'] = index_test
for_output['ConfirmedCases'] = y1_xpred_output
for_output['Fatalities']=y2_xpred_output
output = pd.concat([output, for_output], axis=0)
 output['ForecastId']= output['ForecastId'].astype('int' )
print('Preparing dataset for training...')
 start_time = datetime.now()
def get_keras_data(dataset):
X = {
'app': np.array(dataset.app),
'channel': np.array(dataset.channel),
'device': np.array(dataset.device),
'os': np.array(dataset.os),
'hour': np.array(dataset.hour),
'day': np.array(dataset.day),
'wday': np.array(dataset.wday),
'qty': np.array(dataset.qty),
'c1': np.array(dataset.ip_app_count),
'c2': np.array(dataset.ip_app_os_count)
 }
return X
train = get_keras_data(train)
 in_app = Input(shape=[1], name='app')
 emb_app = Embedding(max_app, EMBEDDING_N )(in_app)
 in_channel = Input(shape=[1], name='channel')
 emb_channel = Embedding(max_channel, EMBEDDING_N )(in_channel)
 in_device = Input(shape=[1], name='device')
 emb_device = Embedding(max_device, EMBEDDING_N )(in_device)
 in_os = Input(shape=[1], name='os')
 emb_os = Embedding(max_os, EMBEDDING_N )(in_os)
 in_hour = Input(shape=[1], name='hour')
 emb_hour = Embedding(max_hour, EMBEDDING_N )(in_hour)
 in_day = Input(shape=[1], name='day')
 emb_day = Embedding(max_day, EMBEDDING_N )(in_day)
 in_wday = Input(shape=[1], name='wday')
 emb_wday = Embedding(max_wday, EMBEDDING_N )(in_wday)
 in_qty = Input(shape=[1], name='qty')
 emb_qty = Embedding(max_qty, EMBEDDING_N )(in_qty)
 in_c1 = Input(shape=[1], name='c1')
 emb_c1 = Embedding(max_c1, EMBEDDING_N )(in_c1)
 in_c2 = Input(shape=[1], name='c2')
 emb_c2 = Embedding(max_c2, EMBEDDING_N )(in_c2)
 print('End of Dataset Preparation: {time_taken}'.format(time_taken=datetime.now() - start_time))<categorify>
gc.collect()
<choose_model_class><EOS> 
buscc = output["ConfirmedCases"]
busf = output["Fatalities"]
sdfcc = sub_df["ConfirmedCases"]
sdff = sub_df["Fatalities"]
sdfcc2 = sub_df20["ConfirmedCases"]
sdff2 = sub_df20["Fatalities"]
sdfcc3 = sub_df40["ConfirmedCases"]
sdff3 = sub_df40["Fatalities"]
sdfcc4 = df_sub1["ConfirmedCases"]
sdff4 = df_sub1["Fatalities"]
sdfcc5 = full_pred["ConfirmedCases"]
sdff5 = full_pred["Fatalities"]
sdfcc6 = df_sub2["ConfirmedCases"]
sdff6 = df_sub2["Fatalities"]
sub_df_final["ConfirmedCases"] = 0.15 * buscc.values + 0.20 * sdfcc.values + 0.075 * sdfcc2.values + 0.075 * sdfcc3.values + 0.1 * sdfcc4.values + 0.3 * sdfcc5.values + 0.1 * sdfcc6.values
sub_df_final["Fatalities"] = 0.15 * busf.values + 0.20 * sdff.values + 0.075 * sdff2.values + 0.075 * sdff3.values + 0.1 * sdff4.values + 0.3 * sdff5.values + 0.1 * sdff6.values
sub_df_final.to_csv('submission.csv',index=False)