Kaggle competition: COVID19 Global Forecasting (Week 3)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-week-3<prepare_output>
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
tourney_lose_result = tourney_win_result.copy()
tourney_lose_result['Seed1'] = tourney_win_result['Seed2']
tourney_lose_result['Seed2'] = tourney_win_result['Seed1']
tourney_lose_result['ScoreT1'] = tourney_win_result['ScoreT2']
tourney_lose_result['ScoreT2'] = tourney_win_result['ScoreT1']
tourney_lose_result<feature_engineering>
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
sns.set_palette(sns.color_palette('tab20', 20))
tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']
tourney_win_result['ScoreT_diff'] = tourney_win_result['ScoreT1'] - tourney_win_result['ScoreT2']
tourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']
tourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']<feature_engineering>
COMP = '.. /input/covid19-global-forecasting-week-3'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '')
 train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')
 test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 country_codes = pd.read_csv('.. /input/covid19-metadata/country_codes.csv', keep_default_na=False)
 train = train.merge(country_codes, on='Country_Region', how='left')
 test = test.merge(country_codes, on='Country_Region', how='left')
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in tqdm(df.groupby('Location')) :
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1
test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))
test_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))
test_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))
test_df<merge>
train[train.geo_region.isna() ].Country_Region.unique()
train = train.fillna('
test = test.fillna('
train[train.duplicated(['Date', 'Location'])]
train.count()
test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df<feature_engineering>
train = train.sort_values(by='Date')
 countries_latest_state = train[train['Date'] == TRAIN_END].groupby([
'Country_Region', 'continent', 'geo_region', 'country_iso_code_3'] ).sum() [[
'ConfirmedCases', 'Fatalities']].reset_index()
countries_latest_state['Log10Confirmed'] = np.log10(countries_latest_state.ConfirmedCases + 1)
 countries_latest_state['Log10Fatalities'] = np.log10(countries_latest_state.Fatalities + 1)
 countries_latest_state = countries_latest_state.sort_values(by='Fatalities', ascending=False)
 countries_latest_state.to_csv('countries_latest_state.csv', index=False)
 countries_latest_state.shape
countries_latest_state.head()
test_df['Seed1'] = test_df['Seed1'].map(lambda x: get_seed(x))
test_df['Seed2'] = test_df['Seed2'].map(lambda x: get_seed(x))
test_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']
test_df['ScoreT_diff'] = test_df['ScoreT1'] - test_df['ScoreT2']
test_df = test_df.drop(['ID', 'Pred', 'Season', 'WTeamID', 'LTeamID'], axis=1)
 test_df<concatenate>
latest_loc = train[train['Date'] == TRAIN_END][['Location', 'ConfirmedCases', 'Fatalities']]
max_loc = train.groupby(['Location'])[['ConfirmedCases', 'Fatalities']].max().reset_index()
check = pd.merge(latest_loc, max_loc, on='Location')
 np.mean(check.ConfirmedCases_x == check.ConfirmedCases_y)
 np.mean(check.Fatalities_x == check.Fatalities_y)
 check[check.Fatalities_x != check.Fatalities_y]
check[check.ConfirmedCases_x != check.ConfirmedCases_y]
tourney_win_result['result'] = 1
tourney_lose_result['result'] = 0
tourney_result = pd.concat(( tourney_win_result, tourney_lose_result)).reset_index(drop=True)
 tourney_result<prepare_x_and_y>
regional_progress = train_clean.groupby(['DateTime', 'continent'] ).sum() [['ConfirmedCases', 'Fatalities']].reset_index()
regional_progress['Log10Confirmed'] = np.log10(regional_progress.ConfirmedCases + 1)
 regional_progress['Log10Fatalities'] = np.log10(regional_progress.Fatalities + 1)
 regional_progress = regional_progress[regional_progress.continent != '
X_train = tourney_result.drop('result', axis=1)
 y_train = tourney_result.result<import_modules>
countries_0301 = country_progress[country_progress.Date == '2020-03-01'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_0331 = country_progress[country_progress.Date == '2020-03-31'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_in_march = pd.merge(countries_0301, countries_0331, on='Country_Region', suffixes=['_0301', '_0331'])
 countries_in_march['IncreaseInMarch'] = countries_in_march.ConfirmedCases_0331 /(countries_in_march.ConfirmedCases_0301 + 1)
 countries_in_march = countries_in_march[countries_in_march.ConfirmedCases_0331 > 200].sort_values(
 by='IncreaseInMarch', ascending=False)
 countries_in_march.tail(15 )
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
from sklearn.utils import shuffle
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
import lightgbm as lgb
import xgboost as xgb
from xgboost import XGBClassifier
import gc<init_hyperparams>
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_confirmed_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_confirmed_deltas = latest.merge(daily_confirmed_deltas, on='Geo
daily_confirmed_deltas.shape
daily_confirmed_deltas.head()
daily_confirmed_deltas.to_csv('daily_confirmed_deltas.csv', index=False )
params_lgb = {'num_leaves': 400,
'min_child_weight': 0.034,
'feature_fraction': 0.379,
'bagging_fraction': 0.418,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': 50,
'learning_rate': 0.0068,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'logloss',
"verbosity": -1,
'reg_alpha': 0.3899,
'reg_lambda': 0.648,
'random_state': 47,
}
params_xgb = {'colsample_bytree': 0.8,
'learning_rate': 0.0004,
'max_depth': 31,
'subsample': 1,
'objective':'binary:logistic',
'eval_metric':'logloss',
'min_child_weight':3,
'gamma':0.25,
'n_estimators':5000
}<split>
deltas = train_clean[np.logical_and(
 train_clean.LogConfirmed > 2,
~train_clean.Location.str.startswith('China')
)].dropna().sort_values(by='LogConfirmedDelta', ascending=False)
 deltas['start'] = deltas['LogConfirmed'].round(0)
 confirmed_deltas = pd.concat([
deltas.groupby('start')[['LogConfirmedDelta']].mean() ,
deltas.groupby('start')[['LogConfirmedDelta']].std() ,
deltas.groupby('start')[['LogConfirmedDelta']].count()
], axis=1)
 deltas.mean()
confirmed_deltas.columns = ['avg', 'std', 'cnt']
confirmed_deltas
confirmed_deltas.to_csv('confirmed_deltas.csv' )
NFOLDS = 200
folds = KFold(n_splits=NFOLDS)
 columns = X_train.columns
splits = folds.split(X_train, y_train )<define_variables>
DECAY = 0.95
DECAY ** 7, DECAY ** 14, DECAY ** 21, DECAY ** 28
confirmed_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.07
confirmed_deltas['DELTA'] = GLOBAL_DELTA
confirmed_deltas.loc[confirmed_deltas.continent=='Africa', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.continent=='Oceania', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.continent=='
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Korea South', 'DELTA'] = 0.01
confirmed_deltas.loc[confirmed_deltas.Country_Region=='US', 'DELTA'] = 0.09
confirmed_deltas.loc[confirmed_deltas.Country_Region=='China', 'DELTA'] = 0.008
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Japan', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Singapore', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.02
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.03
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Czechia', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Serbia', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Romania', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Hungary', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Poland', 'DELTA'] = 0.09
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Norway', 'DELTA'] = 0.03
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iceland', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Luxembourg', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Austria', 'DELTA'] = 0.03
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Italy', 'DELTA'] = 0.03
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Spain', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Portugal', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Israel', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iran', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Germany', 'DELTA'] = 0.045
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Thailand', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Russia', 'DELTA'] = 0.15
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Belarus', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='South Africa', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Egypt', 'DELTA'] = 0.1
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Moldova', 'DELTA'] = 0.11
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Brazil', 'DELTA'] = 0.11
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Mexico', 'DELTA'] = 0.11
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Peru', 'DELTA'] = 0.11
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Turkey', 'DELTA'] = 0.13
confirmed_deltas.loc[confirmed_deltas.Country_Region=='India', 'DELTA'] = 0.15
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Pakistan', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Philippines', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='United Arab Emirates', 'DELTA'] = 0.13
confirmed_deltas.loc[confirmed_deltas.Location=='France-', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Location=='United Kingdom-', 'DELTA'] = 0.09
confirmed_deltas.loc[confirmed_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
confirmed_deltas.loc[confirmed_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.03
confirmed_deltas.loc[confirmed_deltas.Location=='San Marino-', 'DELTA'] = 0.03
confirmed_deltas.loc[confirmed_deltas.Location=='US-New York', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Location=='US-Washington', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Location=='Netherlands-', 'DELTA'] = 0.06
confirmed_deltas.shape, confirmed_deltas.DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].shape, confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA]
confirmed_deltas.describe()
y_preds_lgb = np.zeros(test_df.shape[0])
 y_oof_lgb = np.zeros(X_train.shape[0] )<train_model>
daily_log_confirmed = train_clean.pivot('Location', 'Date', 'LogConfirmed' ).reset_index()
daily_log_confirmed = daily_log_confirmed.sort_values(TRAIN_END, ascending=False)
 daily_log_confirmed.to_csv('daily_log_confirmed.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in confirmed_deltas.Location.values:
confirmed_delta = confirmed_deltas.loc[confirmed_deltas.Location == loc, 'DELTA'].values[0]
daily_log_confirmed.loc[daily_log_confirmed.Location == loc, new_day] = daily_log_confirmed.loc[daily_log_confirmed.Location == loc, last_day] + \
confirmed_delta * DECAY ** i
for fold_n,(train_index, valid_index)in enumerate(splits):
print('Fold:',fold_n+1)
 X_train1, X_valid1 = X_train[columns].iloc[train_index], X_train[columns].iloc[valid_index]
y_train1, y_valid1 = y_train.iloc[train_index], y_train.iloc[valid_index]
dtrain = lgb.Dataset(X_train1, label=y_train1)
 dvalid = lgb.Dataset(X_valid1, label=y_valid1)
 clf = lgb.train(params_lgb, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200)
 y_pred_valid = clf.predict(X_valid1)
 y_oof_lgb[valid_index] = y_pred_valid
y_preds_lgb += clf.predict(test_df)/ NFOLDS
del X_train1, X_valid1, y_train1, y_valid1
gc.collect()<prepare_x_and_y>
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_death_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_death_deltas = latest.merge(daily_death_deltas, on='Geo
daily_death_deltas.shape
daily_death_deltas.head()
daily_death_deltas.to_csv('daily_death_deltas.csv', index=False )
del X_train2, X_valid2, y_train2, y_valid2
gc.collect()<load_from_csv>
death_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.07
death_deltas['DELTA'] = GLOBAL_DELTA
death_deltas.loc[death_deltas.continent=='Oceania', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.continent=='Africa', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='China', 'DELTA'] = 0.005
death_deltas.loc[death_deltas.Country_Region=='Korea South', 'DELTA'] = 0.025
death_deltas.loc[death_deltas.Country_Region=='Japan', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.03
death_deltas.loc[death_deltas.Country_Region=='US', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.07
death_deltas.loc[death_deltas.Country_Region=='Norway', 'DELTA'] = 0.10
death_deltas.loc[death_deltas.Country_Region=='Sweden', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Belgium', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Country_Region=='Netherlands', 'DELTA'] = 0.10
death_deltas.loc[death_deltas.Country_Region=='Poland', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Hungary', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Romania', 'DELTA'] = 0.11
death_deltas.loc[death_deltas.Country_Region=='Iceland', 'DELTA'] = 0.02
death_deltas.loc[death_deltas.Country_Region=='Austria', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Country_Region=='Italy', 'DELTA'] = 0.045
death_deltas.loc[death_deltas.Country_Region=='Spain', 'DELTA'] = 0.045
death_deltas.loc[death_deltas.Country_Region=='Portugal', 'DELTA'] = 0.09
death_deltas.loc[death_deltas.Country_Region=='Ireland', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Israel', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Iran', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Germany', 'DELTA'] = 0.10
death_deltas.loc[death_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Russia', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Brazil', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Turkey', 'DELTA'] = 0.16
death_deltas.loc[death_deltas.Country_Region=='Philippines', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Country_Region=='Saudi Arabia', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Location=='France-', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Location=='United Kingdom-', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
death_deltas.loc[death_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Location=='San Marino-', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Location=='Netherlands-', 'DELTA'] = 0.09
death_deltas.loc[death_deltas.Country_Region=='Mexico', 'DELTA'] = 0.18
death_deltas.loc[death_deltas.Country_Region=='Peru', 'DELTA'] = 0.18
death_deltas.loc[death_deltas.Country_Region=='India', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Pakistan', 'DELTA'] = 0.16
death_deltas.shape
death_deltas.DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].shape
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA]
death_deltas.describe()
submission_df = pd.read_csv(path + 'WSampleSubmissionStage1_2020.csv')
 submission_df['Pred'] = y_preds_lgb
submission_df<save_to_csv>
daily_log_deaths = train_clean.pivot('Location', 'Date', 'LogFatalities' ).reset_index()
daily_log_deaths = daily_log_deaths.sort_values(TRAIN_END, ascending=False)
 daily_log_deaths.to_csv('daily_log_deaths.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in death_deltas.Location:
death_delta = death_deltas.loc[death_deltas.Location == loc, 'DELTA'].values[0]
daily_log_deaths.loc[daily_log_deaths.Location == loc, new_day] = daily_log_deaths.loc[daily_log_deaths.Location == loc, last_day] + \
death_delta * DECAY ** i
submission_df.to_csv('submission.csv', index=False )<set_options>
confirmed = []
fatalities = []
for id, d, loc in tqdm(test[['ForecastId', 'Date', 'Location']].values):
c = to_exp(daily_log_confirmed.loc[daily_log_confirmed.Location == loc, d].values[0])
 f = to_exp(daily_log_deaths.loc[daily_log_deaths.Location == loc, d].values[0])
 confirmed.append(c)
 fatalities.append(f )
warnings.filterwarnings('ignore' )<load_from_csv>
my_submission = test.copy()
my_submission['ConfirmedCases'] = confirmed
my_submission['Fatalities'] = fatalities
my_submission.shape
my_submission.head()
df_features = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/features.csv.zip', sep=',')
 df_stores = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/stores.csv', sep=',')
 df_features_stores = df_features.merge(df_stores, how='inner', on='Store')
 df_features_stores.head()<load_from_csv>
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 print(DECAY)
 my_submission.head()
my_submission.tail()
my_submission.shape
df_train = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/train.csv.zip', sep=',')
 train = df_train.merge(df_features_stores, how='inner', on=['Store','Date','IsHoliday'])
 train.head()<load_from_csv>
end = dt.datetime.now()
print('Finished', end,(end - start ).seconds, 's' )