Kaggle competition: Corporación Favorita Grocery Sales Forecasting
<SOS> metric: NWRMSLE Kaggle data source: favorita-grocery-sales-forecasting<set_options>
MAX_PRED = 1000
MAX_ROUNDS = 2500
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', 500)
 pd.set_option('display.expand_frame_repr', False)
 display(HTML("<style>div.output_scroll { height: 35em; }</style>"))
%matplotlib inline
%config InlineBackend.figure_format ='retina'
warnings.filterwarnings('ignore' )<define_variables>
indir = '.. /input/preparing-data-ii/'
indir2 = '.. /input/favorita-grocery-sales-forecasting/'
days_forecast = 40<load_from_csv>
X_test = pd.read_csv(indir + 'X_test.csv')
 X_val = pd.read_csv(indir + 'X_val.csv')
 X_train = pd.read_csv(indir + 'X_train.csv')
 y_train = np.array(pd.read_csv(indir + 'y_train.csv'))
y_val = np.array(pd.read_csv(indir + 'y_val.csv'))
stores_items = pd.read_csv(indir + 'stores_items.csv', index_col=['store_nbr','item_nbr'])
 test_ids = pd.read_csv(indir + 'test_ids.csv', parse_dates=['date'] ).set_index(
 ['store_nbr', 'item_nbr', 'date'] )
conf_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Confirmed.csv')
 deaths_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Deaths.csv')
 recv_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Recovered.csv' )<filter>
items = pd.read_csv(indir2 + 'items.csv' ).set_index("item_nbr")
 items = items.reindex(stores_items.index.get_level_values(1))
full_table = full_table[full_table['Province/State'].str.contains(',')!=True]<feature_engineering>
params = {
'num_leaves': 31,
'objective': 'regression',
'min_data_in_leaf': 200,
'learning_rate': 0.027,
'feature_fraction': 0.8,
'bagging_fraction': 0.8,
'bagging_freq': 2,
'metric': 'l2',
'num_threads': 4
}
cases = ['Confirmed', 'Deaths', 'Recovered', 'Active']
full_table['Active'] = full_table['Confirmed'] - full_table['Deaths'] - full_table['Recovered']
full_table['Country/Region'] = full_table['Country/Region'].replace('Mainland China', 'China')
 full_table[['Province/State']] = full_table[['Province/State']].fillna('' )<groupby>
val_pred = []
test_pred = []
cate_vars = []
for i in range(16):
print("=" * 50)
 print("Step %d" %(i+1))
print("=" * 50)
 dtrain = lgb.Dataset(
 X_train, label=y_train[:, i],
categorical_feature=cate_vars,
weight=pd.concat([items["perishable"]] * 6)* 0.25 + 1 
)
 dval = lgb.Dataset(
 X_val, label=y_val[:, i], reference=dtrain,
weight=items["perishable"] * 0.25 + 1,
categorical_feature=cate_vars)
 bst = lgb.train(
 params, dtrain, num_boost_round=MAX_ROUNDS,
valid_sets=[dtrain, dval], early_stopping_rounds=150, verbose_eval=200 
)
 print("
".join(( "%s: %.2f" % x)for x in sorted(
 zip(X_train.columns, bst.feature_importance("gain")) ,
key=lambda x: x[1], reverse=True 
)))
 val_pred.append(bst.predict(
 X_val, num_iteration=bst.best_iteration or MAX_ROUNDS))
test_pred.append(bst.predict(
 X_test, num_iteration=bst.best_iteration or MAX_ROUNDS))
df = full_table.groupby(['Country/Region', 'Province/State'])['Confirmed', 'Deaths', 'Recovered', 'Active'].max()
df = full_table.groupby('Date')['Confirmed', 'Deaths', 'Recovered', 'Active'].sum().reset_index()
df = df[df['Date']==max(df['Date'])].reset_index(drop=True)
 df<create_dataframe>
n_public = 5
weights=pd.concat([items["perishable"]])* 0.25 + 1
print("Unweighted validation mse: ", mean_squared_error(
 y_val, np.array(val_pred ).transpose()))
print("Full validation mse: ", mean_squared_error(
 y_val, np.array(val_pred ).transpose() , sample_weight=weights))
print("'Public' validation mse: ", mean_squared_error(
 y_val[:,:n_public], np.array(val_pred ).transpose() [:,:n_public], sample_weight=weights))
print("'Private' validation mse: ", mean_squared_error(
 y_val[:,n_public:], np.array(val_pred ).transpose() [:,n_public:], sample_weight=weights))
df_model_select = full_table.groupby(['Country/Region','Province/State'] ).last().reset_index()
df_model_select = df_model_select[['Country/Region','Province/State','Confirmed']]
df_model_select['Offset error'] = 100
df_model_select['No Offset error'] = 100<data_type_conversions>
y_test = np.array(test_pred ).transpose()
df_preds = pd.DataFrame(
 y_test, index=stores_items.index,
columns=pd.date_range("2017-08-16", periods=16)
 ).stack().to_frame("unit_sales")
 df_preds.index.set_names(["store_nbr", "item_nbr", "date"], inplace=True )
<create_dataframe><EOS> 
submission = test_ids.join(df_preds, how="left" ).fillna(0)
 submission["unit_sales"] = np.clip(np.expm1(submission["unit_sales"]), 0, MAX_PRED)
 submission.to_csv('lgb_including_unsold.csv', float_format='%.4f', index=None )