Kaggle competition: Rossmann Store Sales
train_target = train_data[['SK_ID_CURR', 'TARGET']]<define_variables>
def info(function):
def wrapper(data,*args,**kargs):
tic = datetime.datetime.now()
result = function(data,*args,**kargs)
 toc = datetime.datetime.now()
print(function.__name__,' took ', toc-tic)
 print('Shape: ',data.shape,' ----> ', result.shape)
 print('NaN value: ', result.isna().sum() [result.isna().sum() != 0])
 print('
')
 return result
return wrapper
submit = test_data[['SK_ID_CURR']]<drop_column>
train = pd.read_csv('/kaggle/input/rossmann-store-sales/train.csv')
 test = pd.read_csv('/kaggle/input/rossmann-store-sales/test.csv')
 stores = pd.read_csv('/kaggle/input/rossmann-store-sales/store.csv')
 sample = pd.read_csv('/kaggle/input/rossmann-store-sales/sample_submission.csv' )
train_data.drop(columns=['TARGET'], inplace=True )<feature_engineering>
def ToWeight(y):
w = np.zeros(y.shape, dtype=float)
 ind = y != 0
w[ind] = 1./(y[ind]**2)
 return w
def rmspe(yhat, y):
w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y - yhat)**2))
return rmspe
test_data['IS_TRAIN'] = 0
train_data['IS_TRAIN'] = 1<concatenate>
@info
def log_transf(df):
df.Sales = np.log1p(df.Sales)
 df.Customers = np.log1p(df.Customers)
 return df
@info
def remove_outliers(df,column='Sales'):
q1 = df[column].quantile(0.2)
 q3 = df[column].quantile(0.8)
 iqr = q3-q1
iqr_lower = q1 - 1.5*iqr
iqr_upper = q3 + 1.5*iqr
df = df.loc[(df[column] > iqr_lower)&(df[column]< iqr_upper),:]
return df
application_data = train_data.append(test_data )<drop_column>
@info
def timeseries_features(df):
df.Date = pd.to_datetime(df.Date)
 df = df.sort_values('Date' ).reset_index(drop = True)
 df['month'] = df.Date.dt.month
df['dayofmonth'] = df.Date.dt.day
df['dayofyear'] = df.Date.dt.dayofyear
df['year'] = df.Date.dt.year
df['is_weekday'] = df.DayOfWeek.apply(lambda x: 0 if x in(6,7)else 1)
 df['is_month_start'] = df.Date.dt.is_month_start.astype(int)
 df['is_month_end'] = df.Date.dt.is_month_end.astype(int)
 holidays = calendar().holidays(start = df.Date.min() , end = df.Date.max())
 df['is_holiday'] = df.Date.isin(holidays ).astype(int)
 return df
@info
def clean_main(df):
df = df.loc[df.Sales != 0,:].reset_index(drop = True)
 df = df.drop(['Open'],axis = 1)
 df.StateHoliday = df.StateHoliday.astype(str)
 return df
@info
def clean_store(df):
df.CompetitionDistance.fillna(df.CompetitionDistance.mean() ,inplace = True)
 df.drop(['CompetitionOpenSinceMonth','CompetitionOpenSinceYear','Promo2SinceWeek','Promo2SinceYear'],axis = 1,inplace = True)
 month_dict = {v: k for k,v in enumerate(calendar.month_abbr)}
del month_dict['']
del month_dict['Sep']
month_dict['NaN'] = 0
month_dict['Sept'] = 9
df.PromoInterval = df.PromoInterval.fillna('NaN')
 df.PromoInterval = df.PromoInterval.str.split(',')
 df.PromoInterval = df.PromoInterval.apply(lambda x: [month_dict[value] for value in x if month_dict.get(value)])
 df['promo_len'] = df.PromoInterval.apply(lambda x: len(x))
return df
train_prep =(train 
.copy() 
.pipe(log_transf)
.pipe(remove_outliers)
.pipe(timeseries_features)
.pipe(clean_main)
)
 store_prep =(stores 
.copy() 
.pipe(clean_store)
)
 data_prep = pd.merge(train_prep,store_prep,how='left',on='Store')
 data_prep['is_promo'] = data_prep.apply(lambda x: 1 if x['month'] in x['PromoInterval'] else 0,axis = 1)
 data_prep = data_prep.drop('PromoInterval',axis=1 ).reset_index(drop=True )
del(train_data)
 del(test_data )<count_values>
test_bs = data_prep[data_prep.year == 2015]
train_bs = data_prep[data_prep.year < 2015]
predict_bs =(train_bs 
.groupby(['Store','month','dayofmonth'] ).Sales.mean().reset_index().rename({'Sales':'predictions'},axis = 1)
.merge(test_bs,how='right',on = ['Store','month','dayofmonth'])
.fillna(train_bs.Sales.mean())
.sort_values('Date')
)
 print('Baseline to overcome = {:.2f}'.format(rmspe(np.expm1(predict_bs.Sales),np.expm1(predict_bs.predictions))))
fig,ax = plt.subplots(1,3,figsize =(30,10))
rnd_store = np.random.randint(min(predict_bs.Store),max(predict_bs.Store),3)
 for idx,store in enumerate(rnd_store):
ax[idx].plot(predict_bs[predict_bs.Store == store].Date,np.expm1(predict_bs[predict_bs.Store == store].Sales), color = 'blue' ,label = 'Observed')
 ax[idx].plot(predict_bs[predict_bs.Store == store].Date,np.expm1(predict_bs[predict_bs.Store == store].predictions),color = 'red',label = 'Predicted')
 ax[idx].legend()
ax[idx].set_title('Store '+str(store))
appl_counts = application_data.count().sort_values() /len(application_data )<filter>
def mean_encoding(df,column,target,func = np.mean):
enc_col_name = target+'_enc_'+func.__name__
df_temp =(df 
.groupby(column)[target] 
.apply(func)
.reset_index() 
.rename({target:enc_col_name},axis=1)
)
 df = df.merge(df_temp,how='left',on = column)
 return df,df_temp
data_prep,dict_for_test = mean_encoding(data_prep,'Store','Customers',func = np.mean )
appl_counts[(appl_counts < 0.6)]<count_values>
counter = 0
for store in data_prep.Store.unique() :
df_store = data_prep.copy().loc[data_prep.Store == store,['Date','Sales']].set_index('Date')
 df_store = df_store.resample('D' ).fillna('bfill')
 adf = adfuller(df_store,regression='c', autolag='AIC')
 if adf[1] > 0.05:
print('Adfuller for store {} : p-value = {:.5f} > 5% -----> NON STATIONARY'.format(store,adf[1]*100))
counter+=1
print('
{:.2f} % of stores are non stationary '.format(counter/len(data_prep.Store.unique())*100))
appl_counts = application_data.count().sort_values() /len(application_data )<define_variables>
print('NaN summary
',data_prep.isna().sum() /len(data_prep)*100,'
')
 print('Number of absoulute dublicates:',data_prep.duplicated().sum())
 print('Number of Store - Date dublicates:',data_prep.duplicated(subset = ['Date','Store'] ).sum() )
cols = list(set(appl_counts[(appl_counts < 0.6)].index)- set(['EXT_SOURCE_1', 'OWN_CAR_AGE']))<categorify>
X = data_prep.drop(['Date','Sales','Customers','Store'],axis = 1)
 y = data_prep.Sales
X_train,X_val = X.loc[X.year < 2015,:],X.loc[X.year == 2015,:]
y_train,y_val = y[:X_train.index[-1]+1], y[X_train.index[-1]+1:]
le = LabelEncoder()
for col in application_data.select_dtypes('object'):
if len(application_data[col].unique())<= 2:
le.fit(application_data[col])
 application_data[col] = le.transform(application_data[col] )<categorify>
transformer = make_column_transformer(
(StandardScaler() ,['CompetitionDistance', 'Customers_enc_mean']), 
(OneHotEncoder() ,['StateHoliday', 'StoreType', 'Assortment', 'DayOfWeek', 'month']),
remainder = 'passthrough' 
)
 regressor = xgb.XGBRegressor(n_estimators = 200,
max_depth = 10 
)
 pipeline = make_pipeline(transformer,
regressor)
 pipeline.fit(X_train,y_train)
 print('TRAIN RMSPE = ',rmspe(np.expm1(pipeline.predict(X_train)) ,np.expm1(y_train)))
 print('VAL RMSPE = ',rmspe(np.expm1(pipeline.predict(X_val)) ,np.expm1(y_val)) )
application_data = pd.get_dummies(application_data, dummy_na=True )<count_values>
test_prep =(test 
.copy() 
.pipe(timeseries_features)
.drop(['Open','Date'],axis=1)
)
 test_prep = pd.merge(test_prep,store_prep,how='left',on='Store')
 test_prep['is_promo'] = test_prep.apply(lambda x: 1 if x['month'] in x['PromoInterval'] else 0,axis = 1)
 test_prep = pd.merge(test_prep,dict_for_test,how='left',on='Store')
 test_prep = test_prep.drop(['PromoInterval','Store'],axis=1 ).reset_index(drop=True)
appl_counts = application_data.count().sort_values() /len(application_data )<filter>
test_id = test_prep.Id
test_prep.drop('Id',axis=1,inplace = True)
 predict = np.expm1(pipeline.predict(test_prep))
appl_counts[(appl_counts < 1)]<merge>
sub = pd.DataFrame({'Id':test_id,'Sales':predict} ).sort_values('Id' ).reset_index(drop=True)
 sub.to_csv('submission.csv',index=False )