Kaggle competition: Corporación Favorita Grocery Sales Forecasting
<SOS> metric: NWRMSLE Kaggle data source: favorita-grocery-sales-forecasting<data_type_conversions>
from datetime import date, timedelta
import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error
import lightgbm as lgb
countries = list(set(full_table['Country/Region']))
countries.sort()
for country in countries:
clusters = list(set(full_table['Province/State'][(full_table['Country/Region'] == country)]))
clusters.sort()
for cluster in clusters:
print(' ')
 print('-----------------')
 print(country + ' - ' + cluster)
 df = full_table[(full_table['Country/Region'] == country)&(full_table['Province/State'] == cluster)]
df = df.groupby(['Date','Country/Region'] ).sum().reset_index()
df['Date'] = pd.to_datetime(df['Date'])
 df = df.sort_values(by=['Date'])
 df = df.set_index('Date')[['Deaths']]
df.drop(df.tail(3 ).index,inplace=True)
 df_result = df.copy()
def model(N, a, alpha, t0, t):
return N *(1 - math.e **(-a *(t-t0)))** alpha
def model_loss(params):
N, a, alpha, t0 = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t0, t)- df.iloc[t, 0])** 2
return r
try:
N = df['Deaths'][-1]
T = -df['Deaths'][0]
except:
N = 10000
T = 0
opt = minimize(model_loss, x0=np.array([N, 1.5, 5, T]), method='Nelder-Mead', tol=1e-7 ).x
print('Offset' + str(opt))
x_actual = pd.to_datetime(df.reset_index().iloc[:,0])
 x_actual =list(x_actual)
 y_actual = list(df.reset_index().iloc[:,1])
 start_date = pd.to_datetime(df.index[0])
 x_model = []
y_model = []
for t in range(len(df)+ days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['Offset'])
 df2.index.name = 'Date'
df_result = pd.merge(df_result,
df2,
how='outer',
left_on=['Date'],
right_on=['Date'])
 def model(N, a, alpha, t):
return N *(1 - math.e **(-a *(t)))** alpha
def model_loss(params):
N, a, alpha = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t)- df.iloc[t, 0])** 2
return r
try:
N = df['Deaths'][-1]
except:
N = 10000
opt = minimize(model_loss, x0=np.array([N, 1.5, 5]), method='Nelder-Mead', tol=1e-7 ).x
print('No offset' + str(opt))
try:
start_date = pd.to_datetime(df.index[0])
 x_model = []
y_model = []
for t in range(len(df)+ days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['No Offset'])
 df2.index.name = 'Date'
df_result = pd.merge(df_result,
df2,
how='outer',
left_on=['Date'],
right_on=['Date'])
 df_result = df_result[df_result['Deaths'].notnull() ]
err_offset = 0
err_no_offset = 0
for t in range(len(df_result)) :
err_offset +=(math.log(df_result['Offset'].iloc[t]+1)-math.log(df_result['Deaths'].iloc[t]+1)) **2
err_no_offset +=(math.log(df_result['No Offset'].iloc[t]+1)-math.log(df_result['Deaths'].iloc[t]+1)) **2
err_offset = math.sqrt(err_offset/len(df_result))
err_no_offset = math.sqrt(err_no_offset/len(df_result))
df_model_select['Offset error'][(df_model_select['Country/Region']==country)&(df_model_select['Province/State']==cluster)] = err_offset
df_model_select['No Offset error'][(df_model_select['Country/Region']==country)&(df_model_select['Province/State']==cluster)] = err_no_offset
except:
pass<load_from_csv>
df_train = pd.read_csv(
 '.. /input/train.csv', usecols=[1, 2, 3, 4, 5],
dtype={'onpromotion': bool},
converters={'unit_sales': lambda u: np.log1p(
 float(u)) if float(u)> 0 else 0},
parse_dates=["date"],
skiprows=range(1, 66458909)
)
 df_test = pd.read_csv(
 ".. /input/test.csv", usecols=[0, 1, 2, 3, 4],
dtype={'onpromotion': bool},
parse_dates=["date"]
).set_index(
 ['store_nbr', 'item_nbr', 'date'] 
)
 items = pd.read_csv(
 ".. /input/items.csv",
).set_index("item_nbr" )
df_ca_train = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
 df_ca_test = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv')
 df_ca_submission = pd.read_csv('.. /input/covid19-global-forecasting-week-1/submission.csv' )<data_type_conversions>
df_2017 = df_train[df_train.date.isin(
 pd.date_range("2017-05-31", periods=7 * 11)) ].copy()
del df_train
full_table = df_ca_train
full_table[['Province/State']] = full_table[['Province/State']].fillna('')
 df_comp = df_ca_test
df_comp[['Province/State']] = df_comp[['Province/State']].fillna('')
 df_comp['Date'] = pd.to_datetime(df_comp['Date'])
 df_comp['ConfirmedCases']=0
df_comp['Fatalities']=0<data_type_conversions>
promo_2017_train = df_2017.set_index(
 ["store_nbr", "item_nbr", "date"])[["onpromotion"]].unstack(
 level=-1 ).fillna(False)
 promo_2017_train.columns = promo_2017_train.columns.get_level_values(1)
 promo_2017_test = df_test[["onpromotion"]].unstack(level=-1 ).fillna(False)
 promo_2017_test.columns = promo_2017_test.columns.get_level_values(1)
 promo_2017_test = promo_2017_test.reindex(promo_2017_train.index ).fillna(False)
 promo_2017 = pd.concat([promo_2017_train, promo_2017_test], axis=1)
 del promo_2017_test, promo_2017_train
countries = list(set(full_table['Country/Region']))
countries.sort()
for country in countries:
clusters = list(set(full_table['Province/State'][(full_table['Country/Region'] == country)]))
clusters.sort()
for cluster in clusters:
print(' ')
 print('-----------------')
 print(str(country)+ ' - ' + str(cluster))
df = full_table[(full_table['Country/Region'] == country)&(full_table['Province/State'] == cluster)]
df = df.groupby(['Date','Country/Region'] ).sum().reset_index()
df['Date'] = pd.to_datetime(df['Date'])
 df = df.sort_values(by=['Date'])
 df = df.set_index('Date')[['ConfirmedCases']]
df_result = df.copy()
def model(N, a, alpha, t):
return N *(1 - math.e **(-a *(t)))** alpha
def model_loss(params):
N, a, alpha = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t)- df.iloc[t, 0])** 2
return r
try:
N = df['ConfirmedCases'][-1]
except:
N = 10000
opt = minimize(model_loss, x0=np.array([N, 1.5, 5]), method='Nelder-Mead', tol=1e-7 ).x
print(opt)
 x_actual = pd.to_datetime(df.reset_index().iloc[:,0])
 x_actual = list(x_actual)
 y_actual = list(df.reset_index().iloc[:,1])
 start_date = pd.to_datetime(df.index[0])
 days_forecast = len(df)+len(df_ca_test)-7
x_model = []
y_model = []
for t in range(days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['ConfirmedCases'])
 df2.index.name = 'Date'
df2['Country/Region']=country
df2['Province/State']=cluster
df_comp = pd.merge(df_comp,
df2,
how='left',
on=['Date','Country/Region','Province/State'] 
)
 df_comp = df_comp.rename(columns={'ConfirmedCases_y': 'ConfirmedCases'})
 df_comp['ConfirmedCases'] = df_comp['ConfirmedCases'].fillna(df_comp['ConfirmedCases_x'])
 df_comp = df_comp[['ForecastId','Province/State','Country/Region','Date','ConfirmedCases','Fatalities']] 
<data_type_conversions>
items = items.reindex(df_2017.index.get_level_values(1))
items.head()
countries = list(set(full_table['Country/Region']))
countries.sort()
for country in countries:
clusters = list(set(full_table['Province/State'][(full_table['Country/Region'] == country)]))
clusters.sort()
for cluster in clusters:
print(' ')
 print('-----------------')
 print(str(country)+ ' - ' + str(cluster))
df = full_table[(full_table['Country/Region'] == country)&(full_table['Province/State'] == cluster)]
df = df.groupby(['Date','Country/Region'] ).sum().reset_index()
df['Date'] = pd.to_datetime(df['Date'])
 df = df.sort_values(by=['Date'])
 df = df.set_index('Date')[['Fatalities']]
df_result = df.copy()
def model(N, a, alpha, t):
return N *(1 - math.e **(-a *(t)))** alpha
def model_loss(params):
N, a, alpha = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t)- df.iloc[t, 0])** 2
return r
try:
N = df['Fatalities'][-1]
except:
N = 10000
opt = minimize(model_loss, x0=np.array([N, 1.5, 5]), method='Nelder-Mead', tol=1e-7 ).x
print(opt)
 x_actual = pd.to_datetime(df.reset_index().iloc[:,0])
 x_actual = list(x_actual)
 y_actual = list(df.reset_index().iloc[:,1])
 start_date = pd.to_datetime(df.index[0])
 days_forecast = len(df)+len(df_ca_test)-7
x_model = []
y_model = []
for t in range(days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['Fatalities'])
 df2.index.name = 'Date'
df2['Country/Region']=country
df2['Province/State']=cluster
df_comp = pd.merge(df_comp,
df2,
how='left',
on=['Date','Country/Region','Province/State'] 
)
 df_comp = df_comp.rename(columns={'Fatalities_y': 'Fatalities'})
 df_comp['Fatalities'] = df_comp['Fatalities'].fillna(df_comp['Fatalities_x'])
 df_comp = df_comp[['ForecastId','Province/State','Country/Region','Date','ConfirmedCases','Fatalities']] 
<filter>
def get_timespan(df, dt, minus, periods):
return df[
pd.date_range(dt - timedelta(days=minus), periods=periods)
 ]
df_comp[df_comp['Fatalities']>0]<data_type_conversions>
def prepare_dataset(t2017, is_train=True):
X = pd.DataFrame({
"mean_3_2017": get_timespan(df_2017, t2017, 3, 3 ).mean(axis=1 ).values,
"mean_7_2017": get_timespan(df_2017, t2017, 7, 7 ).mean(axis=1 ).values,
"mean_14_2017": get_timespan(df_2017, t2017, 14, 14 ).mean(axis=1 ).values,
"promo_14_2017": get_timespan(promo_2017, t2017, 14, 14 ).sum(axis=1 ).values
})
 for i in range(16):
X["promo_{}".format(i)] = promo_2017[
t2017 + timedelta(days=i)].values.astype(np.uint8)
 if is_train:
y = df_2017[
pd.date_range(t2017, periods=16)
 ].values
return X, y
return X
df_ca_test['Date'] = pd.to_datetime(df_ca_test['Date'] )<prepare_output>
print("Preparing dataset...")
 t2017 = date(2017, 6, 21)
 X_l, y_l = [], []
for i in range(4):
delta = timedelta(days=7 * i)
 X_tmp, y_tmp = prepare_dataset(
 t2017 + delta 
)
 X_l.append(X_tmp)
 y_l.append(y_tmp)
 X_train = pd.concat(X_l, axis=0)
 y_train = np.concatenate(y_l, axis=0)
 del X_l, y_l
X_val, y_val = prepare_dataset(date(2017, 7, 26))
X_test = prepare_dataset(date(2017, 8, 16), is_train=False )
df_sub = df_comp[['ForecastId','ConfirmedCases','Fatalities']]<save_to_csv>
print("Training and predicting models...")
 params = {
'num_leaves': 2**5 - 1,
'objective': 'regression_l2',
'max_depth': 8,
'min_data_in_leaf': 50,
'learning_rate': 0.05,
'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'bagging_freq': 1,
'metric': 'l2',
'num_threads': 4
}
df_sub.to_csv('submission.csv',index=False )<import_modules>
MAX_ROUNDS = 1000
val_pred = []
test_pred = []
cate_vars = []
for i in range(16):
print("=" * 50)
 print("Step %d" %(i+1))
print("=" * 50)
 dtrain = lgb.Dataset(
 X_train, label=y_train[:, i],
categorical_feature=cate_vars,
weight=pd.concat([items["perishable"]] * 4)* 0.25 + 1 
)
 dval = lgb.Dataset(
 X_val, label=y_val[:, i], reference=dtrain,
weight=items["perishable"] * 0.25 + 1,
categorical_feature=cate_vars)
 bst = lgb.train(
 params, dtrain, num_boost_round=MAX_ROUNDS,
valid_sets=[dtrain, dval], early_stopping_rounds=50, verbose_eval=50 
)
 print("
".join(( "%s: %.2f" % x)for x in sorted(
 zip(X_train.columns, bst.feature_importance("gain")) ,
key=lambda x: x[1], reverse=True 
)))
 val_pred.append(bst.predict(
 X_val, num_iteration=bst.best_iteration or MAX_ROUNDS))
test_pred.append(bst.predict(
 X_test, num_iteration=bst.best_iteration or MAX_ROUNDS))
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.arima_model import ARIMA
from random import random<load_from_csv>
print("Validation mse:", mean_squared_error(
 y_val, np.array(val_pred ).transpose()))
df=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv",usecols=['Lat','Province/State','Country/Region','Date','ConfirmedCases','Fatalities'])
 df.fillna(' ',inplace=True)
 df['Lat']=df['Province/State']+df['Country/Region']
df.drop('Province/State',axis=1,inplace=True)
 df.drop('Country/Region',axis=1,inplace=True)
 countries_list=df.Lat.unique()
df1=[]
for i in countries_list:
df1.append(df[df['Lat']==i])
 print("we have "+ str(len(df1)) +" regions in our dataset")
 test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/test.csv" )<choose_model_class>
print("Making submission...")
 y_test = np.array(test_pred ).transpose()
df_preds = pd.DataFrame(
 y_test, index=df_2017.index,
columns=pd.date_range("2017-08-16", periods=16)
 ).stack().to_frame("unit_sales")
 df_preds.index.set_names(["store_nbr", "item_nbr", "date"], inplace=True)
 submission = df_test[["id"]].join(df_preds, how="left" ).fillna(0)
 submission["unit_sales"] = np.clip(np.expm1(submission["unit_sales"]), 0, 1000 )
submit_confirmed=[]
submit_fatal=[]
for i in df1:
data = i.ConfirmedCases.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_confirmed.extend(list(new[-43:]))
except:
submit_confirmed.extend(list(data[-10:-1]))
for j in range(34):
submit_confirmed.append(data[-1]*2)
 data = i.Fatalities.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_fatal.extend(list(new[-43:]))
except:
submit_fatal.extend(list(data[-10:-1]))
for j in range(34):
submit_fatal.append(data[-1]*2)
<data_type_conversions>
submission.to_csv('lgb.csv', float_format='%.4f', index=None )