Kaggle competition: COVID19 Global Forecasting (Week 3)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-week-3<load_from_csv>
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
 sns.set_palette(sns.color_palette('tab20', 20))
tourney_result = pd.read_csv('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyDetailedResults.csv')
 tourney_seed = pd.read_csv('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')
 season_result = pd.read_csv('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonDetailedResults.csv')
 test_df = pd.read_csv('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv' )<drop_column>
COMP = '.. /input/covid19-global-forecasting-week-3'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '')
 train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')
 test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in tqdm(df.groupby('Location')) :
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1
season_win_result = season_result[['Season', 'WTeamID', 'WScore', 'WFGM', 'WFGA', 'WFGM3', 'WFGA3', 'WFTM', 'WFTA', 'WOR', 'WDR',
'WAst', 'WTO', 'WStl', 'WBlk', 'WPF']]
season_lose_result = season_result[['Season', 'LTeamID', 'LScore', 'LFGM', 'LFGA', 'LFGM3', 'LFGA3', 'LFTM', 'LFTA', 'LOR', 'LDR',
'LAst', 'LTO', 'LStl', 'LBlk', 'LPF']]
season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score', 'WFGM':'FGM', 'WFGA':'FGA', 'WFGM3':'FGM3', 'WFGA3':'FGA3',
'WFTM':'FTM', 'WFTA':'FTA', 'WOR':'OR', 'WDR':'DR', 'WAst':'Ast', 'WTO':'TO', 'WStl':'Stl',
'WBlk':'Blk', 'WPF':'PF'}, inplace=True)
 season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score', 'LFGM':'FGM', 'LFGA':'FGA', 'LFGM3':'FGM3', 'LFGA3':'FGA3',
'LFTM':'FTM', 'LFTA':'FTA', 'LOR':'OR', 'LDR':'DR', 'LAst':'Ast', 'LTO':'TO', 'LStl':'Stl',
'LBlk':'Blk', 'LPF':'PF'}, inplace=True)
 season_result = pd.concat(( season_win_result, season_lose_result)).reset_index(drop=True )<merge>
train = train.sort_values(by='Date')
 countries_latest_state = train[train['Date'] == TRAIN_END].groupby(['Country_Region'] ).sum() [['ConfirmedCases',
'Fatalities']].reset_index()
countries_latest_state['Log10Confirmed'] = np.log10(countries_latest_state.ConfirmedCases + 1)
 countries_latest_state['Log10Fatalities'] = np.log10(countries_latest_state.Fatalities + 1)
 countries_latest_state = countries_latest_state.sort_values(by='Fatalities', ascending=False)
 countries_latest_state['DeathConfirmedRatio'] =(countries_latest_state.Fatalities + 1)/(countries_latest_state.ConfirmedCases + 1)
 countries_latest_state['DeathConfirmedRatio'] = countries_latest_state['DeathConfirmedRatio'].clip(0, 0.1)
 countries_latest_state.shape
countries_latest_state.head()
tourney_result['Score_difference'] = tourney_result['WScore'] - tourney_result['LScore']
tourney_result = tourney_result[['Season', 'WTeamID', 'LTeamID', 'Score_difference']]
tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result['WSeed'] = tourney_result['WSeed'].apply(lambda x: int(x[1:3]))
tourney_result['LSeed'] = tourney_result['LSeed'].apply(lambda x: int(x[1:3]))
print(tourney_result.info(null_counts=True))<merge>
latest_loc = train[train['Date'] == TRAIN_END][['Location', 'ConfirmedCases', 'Fatalities']]
max_loc = train.groupby(['Location'])[['ConfirmedCases', 'Fatalities']].max().reset_index()
check = pd.merge(latest_loc, max_loc, on='Location')
 np.mean(check.ConfirmedCases_x == check.ConfirmedCases_y)
 np.mean(check.Fatalities_x == check.Fatalities_y)
 check[check.Fatalities_x != check.Fatalities_y]
check[check.ConfirmedCases_x != check.ConfirmedCases_y]
test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))
test_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))
test_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))
test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1 )<merge>
train_clean['Location'].nunique()
for col in season_result.columns[2:]:
season_result_map_mean = season_result.groupby(['Season', 'TeamID'])[col].mean().reset_index()
tourney_result = pd.merge(tourney_result, season_result_map_mean, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={f'{col}':f'W{col}MeanT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, season_result_map_mean, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={f'{col}':f'L{col}MeanT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_result_map_mean, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={f'{col}':f'W{col}MeanT'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_result_map_mean, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={f'{col}':f'L{col}MeanT'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1 )<drop_column>
country_progress = train_clean.groupby(['Date', 'DateTime', 'Country_Region'] ).sum() [[
'ConfirmedCases', 'Fatalities', 'ConfirmedDelta', 'FatalitiesDelta']].reset_index()
tourney_win_result = tourney_result.drop(['WTeamID', 'LTeamID'], axis=1)
 for col in tourney_win_result.columns[2:]:
if col[0] == 'W':
tourney_win_result.rename(columns={f'{col}':f'{col[1:]+"1"}'}, inplace=True)
 elif col[0] == 'L':
tourney_win_result.rename(columns={f'{col}':f'{col[1:]+"2"}'}, inplace=True)
 tourney_lose_result = tourney_win_result.copy()
for col in tourney_lose_result.columns:
if col[-1] == '1':
col2 = col[:-1] + '2'
tourney_lose_result[col] = tourney_win_result[col2]
tourney_lose_result[col2] = tourney_win_result[col]
tourney_lose_result.columns<feature_engineering>
countries_0301 = country_progress[country_progress.Date == '2020-03-01'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_0331 = country_progress[country_progress.Date == '2020-03-31'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_in_march = pd.merge(countries_0301, countries_0331, on='Country_Region', suffixes=['_0301', '_0331'])
 countries_in_march['IncreaseInMarch'] = countries_in_march.ConfirmedCases_0331 /(countries_in_march.ConfirmedCases_0301 + 1)
 countries_in_march = countries_in_march[countries_in_march.ConfirmedCases_0331 > 200].sort_values(
 by='IncreaseInMarch', ascending=False)
 countries_in_march.tail(15 )
tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']
tourney_win_result['ScoreMeanT_diff'] = tourney_win_result['ScoreMeanT1'] - tourney_win_result['ScoreMeanT2']
tourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']
tourney_lose_result['ScoreMeanT_diff'] = tourney_lose_result['ScoreMeanT1'] - tourney_lose_result['ScoreMeanT2']
tourney_lose_result['Score_difference'] = -tourney_lose_result['Score_difference']
tourney_win_result['result'] = 1
tourney_lose_result['result'] = 0
tourney_result = pd.concat(( tourney_win_result, tourney_lose_result)).reset_index(drop=True )<feature_engineering>
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_confirmed_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_confirmed_deltas = latest.merge(daily_confirmed_deltas, on='Geo
daily_confirmed_deltas.shape
daily_confirmed_deltas.head()
for col in test_df.columns[2:]:
if col[0] == 'W':
test_df.rename(columns={f'{col}':f'{col[1:]+"1"}'}, inplace=True)
 elif col[0] == 'L':
test_df.rename(columns={f'{col}':f'{col[1:]+"2"}'}, inplace=True)
 test_df['Seed1'] = test_df['Seed1'].apply(lambda x: int(x[1:3]))
test_df['Seed2'] = test_df['Seed2'].apply(lambda x: int(x[1:3]))
test_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']
test_df['ScoreMeanT_diff'] = test_df['ScoreMeanT1'] - test_df['ScoreMeanT2']
test_df = test_df.drop(['ID', 'Pred', 'Season'], axis=1 )<init_hyperparams>
deltas = train_clean[np.logical_and(
 train_clean.LogConfirmed > 2,
~train_clean.Location.str.startswith('China')
)].dropna().sort_values(by='LogConfirmedDelta', ascending=False)
 deltas['start'] = deltas['LogConfirmed'].round(0)
 confirmed_deltas = pd.concat([
deltas.groupby('start')[['LogConfirmedDelta']].mean() ,
deltas.groupby('start')[['LogConfirmedDelta']].std() ,
deltas.groupby('start')[['LogConfirmedDelta']].count()
], axis=1)
 deltas.mean()
confirmed_deltas.columns = ['avg', 'std', 'cnt']
confirmed_deltas
confirmed_deltas.to_csv('confirmed_deltas.csv' )
features = [x for x in tourney_result.columns if x not in ['result', 'Score_difference', 'Season']]
params = {'num_leaves': 400,
'min_child_weight': 0.034,
'feature_fraction': 0.379,
'bagging_fraction': 0.418,
'min_data_in_leaf': 106,
'max_depth': -1,
'learning_rate': 0.0068,
"boosting_type": "gbdt",
"bagging_seed": 11,
'reg_alpha': 0.3899,
'reg_lambda': 0.648,
'random_state': 47,
}
step_size = 20
steps = 250
boosting_rounds = [step_size*(x+1)for x in range(steps)]
def run_boost_round_test(boosting_rounds, step_size):
training_scores, oof_scores, holdback_scores = [], [], []
model = NCAA_model(params, tourney_result, test_df, use_holdback=[2019], regression=False, verbose=False)
 print(f'Training for {step_size*steps} rounds.')
 for rounds in range(step_size,boosting_rounds+1,step_size):
print(f'{"*"*50}')
 print(f'Rounds: {rounds}')
 if model.use_holdback:
tr_score, oof_score, hb_score = model.train(features, n_splits=10, n_boost_round=step_size, early_stopping_rounds=None)
 else:
tr_score, oof_score = model.train(features, n_splits=10, n_boost_round=step_size, early_stopping_rounds=None)
 clips, clip_s = model.fit_clipper(verbose=True)
 spline, spline_s = model.fit_spline_model(verbose=True)
 training_scores.append([tr_score, model.postprocess_preds(clips, use_data = 'train'),
model.postprocess_preds(spline, use_data = 'train', method='spline')])
 oof_scores.append([oof_score, model.postprocess_preds(clips, use_data = 'oof'),
model.postprocess_preds(spline, use_data = 'oof', method='spline')])
 holdback_scores.append([hb_score, model.postprocess_preds(clips, use_data = 'hb'),
model.postprocess_preds(spline, use_data = 'hb', method='spline')])
 return training_scores, oof_scores, holdback_scores, model, clips, spline
training_scores, oof_scores, holdback_scores, model, clips, spline = run_boost_round_test(boosting_rounds[-1], step_size )<save_to_csv>
DECAY = 0.93
DECAY ** 7, DECAY ** 14, DECAY ** 21, DECAY ** 28
confirmed_deltas = train.groupby(['Location', 'Country_Region'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
confirmed_deltas['DELTA'] = GLOBAL_DELTA
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Korea South', 'DELTA'] = 0.011
confirmed_deltas.loc[confirmed_deltas.Country_Region=='US', 'DELTA'] = 0.15
confirmed_deltas.loc[confirmed_deltas.Country_Region=='China', 'DELTA'] = 0.00
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Japan', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Singapore', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Norway', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iceland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Austria', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Italy', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Spain', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Portugal', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Israel', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iran', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Germany', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Russia', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Brazil', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Turkey', 'DELTA'] = 0.16
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Philippines', 'DELTA'] = 0.16
confirmed_deltas.loc[confirmed_deltas.Location=='France-', 'DELTA'] = 0.1
confirmed_deltas.loc[confirmed_deltas.Location=='United Kingdom-', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
confirmed_deltas.loc[confirmed_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Location=='San Marino-', 'DELTA'] = 0.03
confirmed_deltas.shape, confirmed_deltas.DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].shape, confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA]
confirmed_deltas.describe()
y_preds = model.postprocess_preds(spline, method='spline')
 submission_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')
 submission_df['Pred'] = y_preds
submission_df.to_csv('submission.csv', index=False)
 submission_df.describe()<set_options>
confirmed_prediciton = pd.melt(daily_log_confirmed[:25], id_vars='Location')
 confirmed_prediciton['ConfirmedCases'] = to_exp(confirmed_prediciton['value'] )
pd.set_option('max_columns', None)
 plt.style.use('fivethirtyeight')
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 print("Libraries imported!" )<train_model>
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_death_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_death_deltas = latest.merge(daily_death_deltas, on='Geo
daily_death_deltas.shape
daily_death_deltas.head()
class BaseModel(object):

 def __init__(self, train_df, test_df, target, features, categoricals=[],
n_splits=3, cv_method="KFold", group=None, task="regression",
parameter_tuning=False, scaler=None, verbose=True):
self.train_df = train_df
self.test_df = test_df
self.target = target
self.features = features
self.n_splits = n_splits
self.categoricals = categoricals
self.cv_method = cv_method
self.group = group
self.task = task
self.parameter_tuning = parameter_tuning
self.scaler = scaler
self.cv = self.get_cv()
self.verbose = verbose
self.params = self.get_params()
self.y_pred, self.score, self.model, self.oof, self.y_val, self.fi_df = self.fit()
def train_model(self, train_set, val_set):
raise NotImplementedError
def get_params(self):
raise NotImplementedError
def convert_dataset(self, x_train, y_train, x_val, y_val):
raise NotImplementedError
def convert_x(self, x):
return x
def calc_metric(self, y_true, y_pred):
if self.task == "classification":
return log_loss(y_true, y_pred)
 elif self.task == "regression":
return np.sqrt(mean_squared_error(y_true, y_pred))
def get_cv(self):
if self.cv_method == "KFold":
cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df)
 elif self.cv_method == "StratifiedKFold":
cv = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target])
 elif self.cv_method == "TimeSeriesSplit":
cv = TimeSeriesSplit(max_train_size=None, n_splits=self.n_splits)
 return cv.split(self.train_df)
 elif self.cv_method == "GroupKFold":
cv = GroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target], self.group)
 elif self.cv_method == "StratifiedGroupKFold":
cv = StratifiedGroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target], self.group)
 def fit(self):
oof_pred = np.zeros(( self.train_df.shape[0],))
y_vals = np.zeros(( self.train_df.shape[0],))
y_pred = np.zeros(( self.test_df.shape[0],))
if self.group is not None:
if self.group in self.features:
self.features.remove(self.group)
 if self.group in self.categoricals:
self.categoricals.remove(self.group)
 fi = np.zeros(( self.n_splits, len(self.features)))
 if self.scaler is not None:
numerical_features = [f for f in self.features if f not in self.categoricals]
self.train_df[numerical_features] = self.train_df[numerical_features].fillna(self.train_df[numerical_features].median())
 self.test_df[numerical_features] = self.test_df[numerical_features].fillna(self.test_df[numerical_features].median())
 self.train_df[self.categoricals] = self.train_df[self.categoricals].fillna(self.train_df[self.categoricals].mode().iloc[0])
 self.test_df[self.categoricals] = self.test_df[self.categoricals].fillna(self.test_df[self.categoricals].mode().iloc[0])
 if self.scaler == "MinMax":
scaler = MinMaxScaler()
elif self.scaler == "Standard":
scaler = StandardScaler()
df = pd.concat([self.train_df[numerical_features], self.test_df[numerical_features]], ignore_index=True)
 scaler.fit(df[numerical_features])
 x_test = self.test_df.copy()
x_test[numerical_features] = scaler.transform(x_test[numerical_features])
 x_test = [np.absolute(x_test[i])for i in self.categoricals] + [x_test[numerical_features]]
else:
x_test = self.test_df[self.features]
for fold,(train_idx, val_idx)in enumerate(self.cv):
x_train, x_val = self.train_df.loc[train_idx, self.features], self.train_df.loc[val_idx, self.features]
y_train, y_val = self.train_df.loc[train_idx, self.target], self.train_df.loc[val_idx, self.target]
if self.scaler is not None:
x_train[numerical_features] = scaler.transform(x_train[numerical_features])
 x_val[numerical_features] = scaler.transform(x_val[numerical_features])
 x_train = [np.absolute(x_train[i])for i in self.categoricals] + [x_train[numerical_features]]
x_val = [np.absolute(x_val[i])for i in self.categoricals] + [x_val[numerical_features]]
train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)
 model, importance = self.train_model(train_set, val_set)
 fi[fold, :] = importance
conv_x_val = self.convert_x(x_val)
 y_vals[val_idx] = y_val
oof_pred[val_idx] = model.predict(conv_x_val ).reshape(oof_pred[val_idx].shape)
 x_test = self.convert_x(x_test)
 y_pred += model.predict(x_test ).reshape(y_pred.shape)/ self.n_splits
print('Partial score of fold {} is: {}'.format(fold, self.calc_metric(y_val, oof_pred[val_idx])))
 fi_df = pd.DataFrame()
for n in np.arange(self.n_splits):
tmp = pd.DataFrame()
tmp["features"] = self.features
tmp["importance"] = fi[n, :]
tmp["fold"] = n
fi_df = pd.concat([fi_df, tmp], ignore_index=True)
 gfi = fi_df[["features", "importance"]].groupby(["features"] ).mean().reset_index()
fi_df = fi_df.merge(gfi, on="features", how="left", suffixes=('', '_mean'))
loss_score = self.calc_metric(self.train_df[self.target], oof_pred)
 if self.verbose:
print('Our oof loss score is: ', loss_score)
 return y_pred, loss_score, model, oof_pred, y_vals, fi_df
def plot_feature_importance(self, rank_range=[1, 50]):
fig, ax = plt.subplots(1, 1, figsize=(10, 20))
sorted_df = self.fi_df.sort_values(by = "importance_mean", ascending=False ).reset_index().iloc[self.n_splits *(rank_range[0]-1): self.n_splits * rank_range[1]]
sns.barplot(data=sorted_df, x ="importance", y ="features", orient='h')
 ax.set_xlabel("feature importance")
 ax.spines['top'].set_visible(False)
 ax.spines['right'].set_visible(False)
 return sorted_df<train_model>
death_deltas = train.groupby(['Location', 'Country_Region'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
death_deltas['DELTA'] = GLOBAL_DELTA
death_deltas.loc[death_deltas.Country_Region=='China', 'DELTA'] = 0.000
death_deltas.loc[death_deltas.Country_Region=='Korea South', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Japan', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Singapore', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='US', 'DELTA'] = 0.17
death_deltas.loc[death_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Norway', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Iceland', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Country_Region=='Austria', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Italy', 'DELTA'] = 0.07
death_deltas.loc[death_deltas.Country_Region=='Spain', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Portugal', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Israel', 'DELTA'] = 0.16
death_deltas.loc[death_deltas.Country_Region=='Iran', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='Germany', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.10
death_deltas.loc[death_deltas.Country_Region=='Russia', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Brazil', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Turkey', 'DELTA'] = 0.22
death_deltas.loc[death_deltas.Country_Region=='Philippines', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Location=='France-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='United Kingdom-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
death_deltas.loc[death_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Location=='San Marino-', 'DELTA'] = 0.05
death_deltas.shape
death_deltas.DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].shape
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA]
death_deltas.describe()
class LgbModel(BaseModel):

 def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
model = lgb.train(self.params, train_set, num_boost_round = 5000, valid_sets=[train_set, val_set], verbose_eval=verbosity)
 fi = model.feature_importance(importance_type="gain")
 return model, fi
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)
 val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)
 return train_set, val_set
def get_params(self):
params = {
'num_leaves': 127,
'min_data_in_leaf': 50,
'max_depth': -1,
'learning_rate': 0.005,
"boosting_type": "gbdt",
"bagging_seed": 11,
"verbosity": -1,
'random_state': 42,
}
if self.task == "regression":
params["objective"] = "regression"
params["metric"] = "rmse"
elif self.task == "classification":
params["objective"] = "binary"
params["metric"] = "binary_logloss"
if self.parameter_tuning == True:
def objective(trial):
train_x, test_x, train_y, test_y = train_test_split(self.train_df[self.features],
self.train_df[self.target],
test_size=0.3, random_state=42)
 dtrain = lgb.Dataset(train_x, train_y, categorical_feature=self.categoricals)
 dtest = lgb.Dataset(test_x, test_y, categorical_feature=self.categoricals)
 hyperparams = {'num_leaves': trial.suggest_int('num_leaves', 24, 1024),
'boosting_type': 'gbdt',
'objective': params["objective"],
'metric': params["metric"],
'max_depth': trial.suggest_int('max_depth', 4, 16),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
'early_stopping_rounds': 100
}
model = lgb.train(hyperparams, dtrain, valid_sets=dtest, verbose_eval=500)
 pred = model.predict(test_x)
 if self.task == "classification":
return log_loss(test_y, pred)
 elif self.task == "regression":
return np.sqrt(mean_squared_error(test_y, pred))
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials: {}'.format(len(study.trials)))
 print('Best trial:')
 trial = study.best_trial
print(' Value: {}'.format(trial.value))
print(' Params: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))
params = trial.params
params["learning_rate"] = 0.001
plot_optimization_history(study)
 return params<train_model>
confirmed_prediciton = pd.melt(daily_log_deaths[:25], id_vars='Location')
 confirmed_prediciton['Fatalities'] = to_exp(confirmed_prediciton['value'])
 confirmed_prediciton.shape
class CatbModel(BaseModel):

 def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
if self.task == "regression":
model = CatBoostRegressor(**self.params)
 elif self.task == "classification":
model = CatBoostClassifier(**self.params)
 model.fit(train_set['X'], train_set['y'], eval_set=(val_set['X'], val_set['y']),
verbose=verbosity, cat_features=self.categoricals)
 return model, model.get_feature_importance()
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = {'X': x_train, 'y': y_train}
val_set = {'X': x_val, 'y': y_val}
return train_set, val_set
def get_params(self):
params = { 'task_type': "CPU",
'learning_rate': 0.01,
'iterations': 1000,
'random_seed': 42,
'use_best_model': True
}
if self.task == "regression":
params["loss_function"] = "RMSE"
elif self.task == "classification":
params["loss_function"] = "Logloss"
return params<normalization>
confirmed = []
fatalities = []
for id, d, loc in tqdm(test[['ForecastId', 'Date', 'Location']].values):
c = to_exp(daily_log_confirmed.loc[daily_log_confirmed.Location == loc, d].values[0])
 f = to_exp(daily_log_deaths.loc[daily_log_deaths.Location == loc, d].values[0])
 confirmed.append(c)
 fatalities.append(f )
class Mish(Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 def build(self, input_shape):
super(Mish, self ).build(input_shape)
 def call(self, x):
return x * K.tanh(K.softplus(x))
def compute_output_shape(self, input_shape):
return input_shape
class LayerNormalization(keras.layers.Layer):
def __init__(self,
center=True,
scale=True,
epsilon=None,
gamma_initializer='ones',
beta_initializer='zeros',
gamma_regularizer=None,
beta_regularizer=None,
gamma_constraint=None,
beta_constraint=None,
**kwargs):

 super(LayerNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.center = center
self.scale = scale
if epsilon is None:
epsilon = K.epsilon() * K.epsilon()
self.epsilon = epsilon
self.gamma_initializer = keras.initializers.get(gamma_initializer)
 self.beta_initializer = keras.initializers.get(beta_initializer)
 self.gamma_regularizer = keras.regularizers.get(gamma_regularizer)
 self.beta_regularizer = keras.regularizers.get(beta_regularizer)
 self.gamma_constraint = keras.constraints.get(gamma_constraint)
 self.beta_constraint = keras.constraints.get(beta_constraint)
 self.gamma, self.beta = None, None
def get_config(self):
config = {
'center': self.center,
'scale': self.scale,
'epsilon': self.epsilon,
'gamma_initializer': keras.initializers.serialize(self.gamma_initializer),
'beta_initializer': keras.initializers.serialize(self.beta_initializer),
'gamma_regularizer': keras.regularizers.serialize(self.gamma_regularizer),
'beta_regularizer': keras.regularizers.serialize(self.beta_regularizer),
'gamma_constraint': keras.constraints.serialize(self.gamma_constraint),
'beta_constraint': keras.constraints.serialize(self.beta_constraint),
}
base_config = super(LayerNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def compute_mask(self, inputs, input_mask=None):
return input_mask
def build(self, input_shape):
shape = input_shape[-1:]
if self.scale:
self.gamma = self.add_weight(
 shape=shape,
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint,
name='gamma', 
)
 if self.center:
self.beta = self.add_weight(
 shape=shape,
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint,
name='beta', 
)
 super(LayerNormalization, self ).build(input_shape)
 def call(self, inputs, training=None):
mean = K.mean(inputs, axis=-1, keepdims=True)
 variance = K.mean(K.square(inputs - mean), axis=-1, keepdims=True)
 std = K.sqrt(variance + self.epsilon)
 outputs =(inputs - mean)/ std
if self.scale:
outputs *= self.gamma
if self.center:
outputs += self.beta
return outputs<train_model>
my_submission = test.copy()
my_submission['ConfirmedCases'] = confirmed
my_submission['Fatalities'] = fatalities
my_submission.shape
my_submission.head()
class NeuralNetworkModel(BaseModel):

 def train_model(self, train_set, val_set):
inputs = []
embeddings = []
embedding_out_dim = self.params['embedding_out_dim']
n_neuron = self.params['hidden_units']
for i in self.categoricals:
input_ = Input(shape=(1,))
embedding = Embedding(int(np.absolute(self.train_df[i] ).max() + 1), embedding_out_dim, input_length=1 )(input_)
 embedding = Reshape(target_shape=(embedding_out_dim,))(embedding)
 inputs.append(input_)
 embeddings.append(embedding)
 input_numeric = Input(shape=(len(self.features)- len(self.categoricals),))
embedding_numeric = Dense(n_neuron )(input_numeric)
 embedding_numeric = Mish()(embedding_numeric)
 inputs.append(input_numeric)
 embeddings.append(embedding_numeric)
 x = Concatenate()(embeddings)
 for i in np.arange(self.params['hidden_layers'] - 1):
x = Dense(n_neuron //(2 *(i+1)) )(x)
 x = Mish()(x)
 x = Dropout(self.params['hidden_dropout'] )(x)
 x = LayerNormalization()(x)
 if self.task == "regression":
out = Dense(1, activation="linear", name = "out" )(x)
 loss = "mse"
elif self.task == "classification":
out = Dense(1, activation='sigmoid', name = 'out' )(x)
 loss = "binary_crossentropy"
model = Model(inputs=inputs, outputs=out)
 model.compile(loss=loss, optimizer=Adam(lr=1e-04, beta_1=0.9, beta_2=0.999, decay=1e-04))
er = EarlyStopping(patience=10, min_delta=1e-4, restore_best_weights=True, monitor='val_loss')
 ReduceLR = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=7, verbose=1, epsilon=1e-4, mode='min')
 model.fit(train_set['X'], train_set['y'], callbacks=[er, ReduceLR],
epochs=self.params['epochs'], batch_size=self.params['batch_size'],
validation_data=[val_set['X'], val_set['y']])
 fi = np.zeros(len(self.features))
return model, fi
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = {'X': x_train, 'y': y_train}
val_set = {'X': x_val, 'y': y_val}
return train_set, val_set
def get_params(self):

 params = {
'input_dropout': 0.0,
'hidden_layers': 2,
'hidden_units': 64,
'embedding_out_dim': 4,
'hidden_activation': 'relu',
'hidden_dropout': 0.02,
'batch_norm': 'before_act',
'optimizer': {'type': 'adam', 'lr': 0.001},
'batch_size': 256,
'epochs': 80
}
return params<load_from_csv>
my_submission.Location.nunique()
data_dict = {}
for i in glob.glob('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/*'):
name = i.split('/')[-1].split('.')[0]
if name != 'MTeamSpellings':
data_dict[name] = pd.read_csv(i)
 else:
data_dict[name] = pd.read_csv(i, encoding='cp1252' )<feature_engineering>
my_submission[['ConfirmedCases','Fatalities']]= my_submission[['ConfirmedCases','Fatalities']].round(1 )
<load_from_csv><EOS> 
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 print(DECAY)
 my_submission.head()
my_submission.tail()
my_submission.shape