Kaggle competition: Walmart Recruiting - Store Sales Forecasting
<SOS> metric: WMAE Kaggle data source: walmart-recruiting-store-sales-forecasting<define_variables>
warnings.filterwarnings('ignore' )
features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
target = train_df['target']<split>
df_features = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/features.csv.zip', sep=',')
 df_stores = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/stores.csv', sep=',')
 df_features_stores = df_features.merge(df_stores, how='inner', on='Store')
 df_features_stores.head()
%%time
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2019)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
start = time.time()
for fold_,(trn_idx, val_idx)in enumerate(skf.split(train_df.values, target.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 10000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 100)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ 5
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
df_train = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/train.csv.zip', sep=',')
 train = df_train.merge(df_features_stores, how='inner', on=['Store','Date','IsHoliday'])
 train.head()
sub_df = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("lgb_submission.csv", index=False )<import_modules>
df_test = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/test.csv.zip', sep=',')
 test = df_test.merge(df_features_stores, how='inner', on=['Store','Date','IsHoliday'])
 test.head()
import pandas as pd
from pathlib import Path
import numpy as np<define_variables>
train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train['Week'] = train['Date'].dt.isocalendar().week
test['Week'] = test['Date'].dt.isocalendar().week
train['Year'] = train['Date'].dt.isocalendar().year
test['Year'] = test['Date'].dt.isocalendar().year
root = Path(".. /input" )<load_from_csv>
train['Date'] = pd.to_datetime(train['Date'])
 train['Type'] = train['Type'].apply(lambda x: 3 if x == 'A' else(2 if x == 'B' else 1))
train['IsHoliday'] = train['IsHoliday'].apply(lambda x: 1 if x == True else 0)
 cols = train.columns.drop(['Date'])
 train[cols] = train[cols].apply(pd.to_numeric, errors='coerce' )
train = pd.read_csv(root.joinpath("train.csv"))
test = pd.read_csv(root.joinpath("test.csv"))<drop_column>
test['Date'] = pd.to_datetime(test['Date'])
 test['Type'] = test['Type'].apply(lambda x: 3 if x == 'A' else(2 if x == 'B' else 1))
test['IsHoliday'] = test['IsHoliday'].apply(lambda x: 1 if x == True else 0)
 cols = test.columns.drop(['Date'])
 test[cols] = test[cols].apply(pd.to_numeric, errors='coerce' )
train_id = train.ID_code
test_id = test.ID_code
target = train.target
train.drop(columns=["ID_code", "target"], inplace=True)
 test.drop(columns=["ID_code"], inplace=True )<import_modules>
holiday_train = train[['Date','Week','Year','IsHoliday']]
holiday_train = holiday_train.loc[holiday_train['IsHoliday']==True].drop_duplicates()
holiday_test = test[['Date','Week','Year','IsHoliday']]
holiday_test = holiday_test.loc[holiday_test['IsHoliday']==True].drop_duplicates()
holidays = pd.concat([holiday_train, holiday_test])
 holidays
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
from catboost import Pool, CatBoostClassifier<choose_model_class>
def holiday_type(x):
if(x['IsHoliday']== 1)&(x['Week']==6):
return 1
elif(x['IsHoliday']== 1)&(x['Week']==36):
return 2
elif(x['IsHoliday']== 1)&(x['Week']==47):
return 3
elif(x['IsHoliday']== 1)&(x['Week']==52):
return 4
else:
return 0
model = CatBoostClassifier(loss_function="Logloss",
eval_metric="AUC",
task_type="GPU",
learning_rate=0.01,
iterations=70000,
l2_leaf_reg=50,
random_seed=432013,
od_type="Iter",
depth=5,
early_stopping_rounds=15000,
border_count=64
)<choose_model_class>
train['IsHoliday'] = train.apply(holiday_type, axis=1)
 train['IsHoliday'].unique()
n_split = 15
kf = KFold(n_splits=n_split, random_state=432013, shuffle=True )<prepare_x_and_y>
test['IsHoliday'] = test.apply(holiday_type, axis=1)
 test['IsHoliday'].unique()
y_valid_pred = 0 * target
y_test_pred = 0<train_model>
train = train.replace('None', np.nan)
 train = train.replace('NaN', np.nan)
 train = train.replace('NaT', np.nan)
 train = train.replace('', np.nan)
 train_nulls =(train.isnull().sum(axis = 0)/len(train)) *100
train_nulls
for idx,(train_index, valid_index)in enumerate(kf.split(train)) :
y_train, y_valid = target.iloc[train_index], target.iloc[valid_index]
X_train, X_valid = train.iloc[train_index,:], train.iloc[valid_index,:]
_train = Pool(X_train, label=y_train)
 _valid = Pool(X_valid, label=y_valid)
 print("
Fold ", idx)
 fit_model = model.fit(_train,
eval_set=_valid,
use_best_model=True,
verbose=5000,
plot=True 
)
 pred = fit_model.predict_proba(X_valid)[:,1]
print(" auc = ", roc_auc_score(y_valid, pred))
y_valid_pred.iloc[valid_index] = pred
y_test_pred += fit_model.predict_proba(test)[:,1]
y_test_pred /= n_split<save_to_csv>
test = test.replace('None', np.nan)
 test = test.replace('NaN', np.nan)
 test = test.replace('NaT', np.nan)
 test = test.replace('', np.nan)
 test_nulls =(test.isnull().sum(axis = 0)/len(test)) *100
test_nulls
submission = pd.read_csv(root.joinpath("sample_submission.csv"))
submission['target'] = y_test_pred
submission.to_csv('submission_cb_light_0.8999.csv', index=False )<set_options>
train = train.fillna(0)
 test = test.fillna(0)
 train.isnull().sum()
warnings.filterwarnings('ignore' )<define_variables>
weekly_sales = train.groupby(['Year','Week'] ).agg({'Weekly_Sales': ['mean', 'median']})
 weekly_sales2010 = train.loc[train['Year']==2010].groupby(['Week'] ).agg({'Weekly_Sales': ['mean', 'median']})
 weekly_sales2011 = train.loc[train['Year']==2011].groupby(['Week'] ).agg({'Weekly_Sales': ['mean', 'median']})
 weekly_sales2012 = train.loc[train['Year']==2012].groupby(['Week'] ).agg({'Weekly_Sales': ['mean', 'median']} )
num_submission = 3
random_state = 3213<load_from_csv>
sample_weight = train['IsHoliday'].apply(lambda x: 1 if x==0 else 5)
 sample_weight_frame = pd.DataFrame(sample_weight, index=train.index )
train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv' )<define_variables>
def WMAE(y_test, y_pred):
y_pred_df = pd.DataFrame(y_pred,index=y_test.index)
 weights_5 = sample_weight_frame.loc[(y_test.index)].loc[sample_weight_frame.IsHoliday==5].index
weights_1 = sample_weight_frame.loc[(y_test.index)].loc[sample_weight_frame.IsHoliday==1].index
sum_5 = np.sum(5*(abs(y_test.loc[weights_5].values-y_pred_df.loc[weights_5].values)))
 sum_1 = np.sum(abs(y_test.loc[weights_1].values-y_pred_df.loc[weights_1].values))
return np.round(( sum_5+sum_1)/(5*len(weights_5)+len(weights_1)) ,2)
 my_score = make_scorer(WMAE,greater_is_better=False )
target = train_df['target']<load_from_csv>
train_all = train.drop(['Date'],axis=1)
 train_all
train_stats = pd.read_csv('.. /input/stats-df/stats_train.csv')
 test_stats = pd.read_csv('.. /input/stats-df/stats_test.csv' )<concatenate>
y_train_all = train_all.loc[:, ['Weekly_Sales']]
x_train_all = train_all.drop(['Weekly_Sales'], axis=1 )
extra_train = pd.concat([train_df.transpose() , train_stats.transpose() ] ).transpose()<drop_column>
x_train, x_test, y_train, y_test = train_test_split(x_train_all, y_train_all, test_size=0.2, random_state=0)
 print(x_train.shape)
 print(x_test.shape )
del extra_train["Unnamed: 0"]<concatenate>
clf = RandomForestRegressor(random_state=0)
 pca = PCA()
pipe = Pipeline(steps=[('clf', clf)])
 param_grid = [ {
'clf':[RandomForestRegressor() ],
'clf__n_estimators': [50,100,150],
'clf__max_depth': [10,20,30]
},
{
'clf': [ExtraTreesRegressor() ],
'clf__n_estimators': [50,100,150],
'clf__max_depth': [10,20,30]
},
{
'clf': [XGBRegressor() ],
'clf__learning_rate':[0.1,0.05],
'clf__min_samples_split':[5,7,9],
'clf__max_depth':[10,20,30]
}
]
rscv_all_tree = RandomizedSearchCV(pipe, param_grid, cv = 3, scoring = my_score, n_jobs=-1)
 model_all_tree = rscv_all_tree.fit(x_train, y_train )
extra_test = pd.concat([test_df.transpose() , test_stats.transpose() ] ).transpose()<drop_column>
rscv_all_tree.best_estimator_
del extra_test["Unnamed: 0"]<drop_column>
y_pred = rscv_all_tree.best_estimator_.predict(x_test)
 print('WMAE:', WMAE(y_test, y_pred))
del train_df, test_df, train_stats, test_stats<define_variables>
train_relevant = train.drop(['Date','Temperature','Fuel_Price','MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5','CPI','Unemployment'],axis=1)
 train_relevant
features = [c for c in extra_train.columns if c not in ['ID_code', 'target']]<categorify>
y_relevant = train_relevant.loc[:, ['Weekly_Sales']]
x_relevant = train_relevant.drop(['Weekly_Sales'], axis=1 )
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<init_hyperparams>
x_train_relevant, x_test_relevant, y_train_relevant, y_test_relevant = train_test_split(x_relevant, y_relevant, test_size=0.2, random_state=0)
 print(x_train_relevant.shape)
 print(x_test_relevant.shape )
lgb_params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : 90,
"num_leaves" : 15,
"learning_rate" : 0.01000123,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 150,
"min_sum_heassian_in_leaf": 15,
"tree_learner": "voting",
"boost_from_average": "false",
"lambda_l1" : 10,
"lambda_l2" : 10,
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state
}<create_dataframe>
clf = RandomForestRegressor(random_state=0)
 pca = PCA()
pipe = Pipeline(steps=[('clf', clf)])
 param_grid = [ {
'clf':[RandomForestRegressor() ],
'clf__n_estimators': [50,100,150],
'clf__max_depth': [10,20,30]
},
{
'clf': [ExtraTreesRegressor() ],
'clf__n_estimators': [50,100,150],
'clf__max_depth': [10,20,30]
},
{
'clf': [XGBRegressor() ],
'clf__learning_rate':[0.1,0.05],
'clf__min_samples_split':[5,7,9],
'clf__max_depth':[10,20,30]
}
]
rscv_relevant_tree = RandomizedSearchCV(pipe, param_grid, cv = 3, scoring = my_score, n_jobs=-1)
 model_relevant_tree = rscv_relevant_tree.fit(x_train_relevant, y_train_relevant )
num_folds = 3
folds = StratifiedKFold(n_splits=num_folds, shuffle=False, random_state=random_state)
 oof = extra_train[['ID_code', 'target']]
oof['predict'] = 0
predictions = extra_test[['ID_code']]
val_aucs = []
feature_importance_df = pd.DataFrame()<prepare_x_and_y>
rscv_relevant_tree.best_estimator_
X_test = extra_test[features].values<split>
y_pred= rscv_relevant_tree.best_estimator_.predict(x_test_relevant)
 print('WMAE:', WMAE(y_test_relevant, y_pred))
for fold,(trn_idx, val_idx)in enumerate(folds.split(extra_train.values, target.values)) :
print("Fold :{}".format(fold + 1))
X_train, y_train = extra_train.iloc[trn_idx][features], target.iloc[trn_idx]
X_valid, y_valid = extra_train.iloc[val_idx][features], target.iloc[val_idx]
N = 4
p_valid,yp = 0,0
for i in range(N):
X_t, y_t = augment(X_train.values, y_train.values)
 X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 X_t.columns = [*X_t.columns[:200], *swap]
trn_data = lgb.Dataset(X_t.values, label=y_t)
 val_data = lgb.Dataset(X_valid.values, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(lgb_params,
trn_data,
110000,
valid_sets = [trn_data, val_data],
early_stopping_rounds=3500,
verbose_eval = 1000,
evals_result=evals_result 
)
 p_valid += lgb_clf.predict(X_valid.values)
 yp += lgb_clf.predict(X_test)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = lgb_clf.feature_importance()
fold_importance_df["fold"] = fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof['predict'][val_idx] = p_valid/N
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold+1)] = yp/N<compute_test_metric>
clf = RandomForestRegressor(random_state=0)
 pipe = Pipeline(steps=[('clf', clf)])
 param_grid_rf = [ {
'clf':[RandomForestRegressor() ],
'clf__n_estimators': [140,150,160],
'clf__max_depth': [25,30,35],
'clf__max_features': ['auto',5,6]
}
]
gscv_rf1 = GridSearchCV(pipe, param_grid_rf, cv = 3, scoring = my_score, n_jobs=-1)
 model_rf1 = gscv_rf1.fit(x_train_relevant, y_train_relevant )
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(oof['target'], oof['predict'])
 print("Mean auc: %.9f, std: %.9f.All auc: %.9f." %(mean_auc, std_auc, all_auc))<save_to_csv>
gscv_rf1.best_estimator_
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('lgb_all_predictions.csv', index=None)
 sub_df = pd.DataFrame({"ID_code":extra_test["ID_code"].values})
 sub_df["target"] = predictions['target']
sub_df.to_csv("lgb_stats_aug_submission{}.csv".format(num_submission), index=False)
 oof.to_csv('lgb_oof.csv', index=False )<import_modules>
y_pred_rf = gscv_rf1.best_estimator_.predict(x_test_relevant)
 print('WMAE:', WMAE(y_test_relevant, y_pred_rf))
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
import matplotlib.pyplot as plt
import lightgbm as lgb
import seaborn as sns
import warnings
import time<set_options>
date = test['Date']
test = test.drop(['Date'], axis=1 )
fold_n=5
folds = StratifiedKFold(n_splits=fold_n, shuffle=True, random_state=10)
 %matplotlib inline
%precision 4
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 np.set_printoptions(suppress=True)
 pd.set_option("display.precision", 15 )<load_from_csv>
test_relevant = test.drop(['Temperature','Fuel_Price','MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5','CPI', 'Unemployment'],axis=1)
 test_relevant = test_relevant.sort_values(['Store', 'Dept'], ascending=[True, True])
 y_pred_rf = gscv_rf1.best_estimator_.predict(test_relevant )
train= pd.read_csv(".. /input/train.csv")
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
test_relevant['Date'] = date
test_relevant = test_relevant.sort_values(['Store', 'Dept'], ascending=[True, True])
 test_relevant['Weekly_Sales'] = y_pred_rf
test_relevant
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 sample_submission.head()<count_missing_values>
sampleSubmission = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/sampleSubmission.csv.zip', sep=',' )
<count_missing_values><EOS> 
sampleSubmission['Weekly_Sales'] = y_pred_rf
sampleSubmission.to_csv('submission.csv',index=False)
 sampleSubmission