Kaggle competition: Walmart Recruiting - Store Sales Forecasting
test_df = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 subm_df = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv' )<import_modules>
%matplotlib inline
%precision 4
import time
import xgboost as xgb
from sklearn.cross_validation import train_test_split<define_variables>
features = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/features.csv.zip')
 train = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/train.csv.zip')
 stores = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/stores.csv')
 test = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/test.csv.zip')
 holidays = pd.read_csv('.. /input/holidays/holidays.csv' )
start_time=time.time()
columns=['ip','app','device','os','channel','click_time','is_attributed']
dtypes={
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
}<load_from_csv>
holidays["Date"]=pd.to_datetime(holidays["Date"])
 features["IsHoliday"] = np.where(features["IsHoliday"]==True,1,0)
 features["Date"] = pd.to_datetime(features["Date"])
 train["Date"] = pd.to_datetime(train["Date"])
 test["Date"] = pd.to_datetime(test["Date"] )
train=pd.read_csv('.. /input/train.csv',skiprows=range(1,149903891),nrows=35000000,usecols=columns,dtype=dtypes)
 test=pd.read_csv('.. /input/test.csv')
 print('[{}] Finished to load data'.format(time.time() - start_time))<create_dataframe>
train["category"] = "train"
test["category"] = "test"
data_join = pd.concat([train,test])
 data_join = pd.merge(data_join, features, on=["Store","Date", "IsHoliday"], how= "left")
 data_join = pd.merge(data_join, stores, on="Store", how= "inner")
 data_join = pd.merge(data_join, holidays, on="Date", how= "left")
 data_join["IsHoliday"] = data_join["IsHoliday"].astype(int)
 data_join.shape
sub = pd.DataFrame()
sub['click_id'] = test['click_id']<data_type_conversions>
data_join["YEAR"] = data_join["Date"].dt.year
data_join["MONTH"] = data_join["Date"].dt.month
data_join["WEEK"] = data_join["Date"].dt.week
data_join
def dataPreProcessTime(df):
df['date_click'] = pd.to_datetime(df['click_time'] ).dt.date
df['date_click'] = df['date_click'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 df['time_click'] = pd.to_datetime(df['click_time'] ).dt.time
df['time_click'] = df['time_click'].apply(lambda x: x.strftime('%H%M%S')).astype(int)
 df.drop('click_time', axis=1, inplace=True)
 return df 
<count_values>
data_sales = data_join[["Dept","Store","YEAR","WEEK","Weekly_Sales"]]
data_sales = data_sales.rename(columns={"Weekly_Sales":"Weekly_Sale_before"})
 data_sales["YEAR"] = data_sales["YEAR"] + 1
data_join2 = pd.merge(data_join, data_sales, on=["Dept","Store","YEAR","WEEK"], how="left")
 data_join2
print(train['is_attributed'].value_counts())
 print(train[train['is_attributed']==1]['is_attributed'].sum() /len(train))<categorify>
data_join2["FLAG_D72&W47"] = np.where(( data_join2["Dept"]==72)&(data_join2["WEEK"]==47),1,0)
 data_join2[data_join2["FLAG_D72&W47"] == 1]
train = dataPreProcessTime(train)
 test = dataPreProcessTime(test )<prepare_x_and_y>
data_join2.isnull().sum()
y=train['is_attributed']
train.drop(['is_attributed'],axis=1,inplace=True)
 test.drop(['click_id'],axis=1,inplace=True )<concatenate>
data_join2["Weekly_Sale_before"] = data_join2["Weekly_Sale_before"].fillna(0)
 data_join2["Weekly_Sale_before"] = np.where(data_join2["Weekly_Sale_before"]==0,data_join2["Weekly_Sales"],data_join2["Weekly_Sale_before"])
 data_join2
nrow_train = train.shape[0]
merge = pd.concat([train, test] )<merge>
data_join2.drop(columns=["Temperature","Fuel_Price","MarkDown1","MarkDown2","MarkDown3","MarkDown4","MarkDown5","CPI","Unemployment"],axis=1,inplace=True)
 data_join2
ip_count = merge.groupby('ip')['app'].count().reset_index()
ip_count.columns = ['ip', 'clicks_by_ip']
ip_count.tail()
merge = pd.merge(merge, ip_count, on='ip', how='left', sort=False)
 merge.drop('ip', axis=1, inplace=True )<init_hyperparams>
data_join2.loc[(data_join2.YEAR==2010)&(data_join2.WEEK==13), 'IsHoliday'] = True
data_join2.loc[(data_join2.YEAR==2011)&(data_join2.WEEK==16), 'IsHoliday'] = True
data_join2.loc[(data_join2.YEAR==2012)&(data_join2.WEEK==14), 'IsHoliday'] = True
data_join2.loc[(data_join2.YEAR==2013)&(data_join2.WEEK==13), 'IsHoliday'] = True
params = {'eta': 0.6,
'tree_method': "hist",
'grow_policy': "lossguide",
'max_leaves': 1400,
'max_depth': 0,
'subsample': 0.9,
'colsample_bytree': 0.7,
'colsample_bylevel':0.7,
'min_child_weight':0,
'alpha':4,
'objective': 'binary:logistic',
'scale_pos_weight':9,
'eval_metric': 'auc',
'nthread':8,
'random_state': 99,
'silent': True}<train_model>
watchlist = [(xgb.DMatrix(train, y), 'train')]
model = xgb.train(params, xgb.DMatrix(train, y), 15, watchlist, maximize=True, verbose_eval=1 )<train_model>
import lightgbm as lgb
print('[{}] Finish XGBoost Training'.format(time.time() - start_time))<save_to_csv>
data_train = data_join2[data_join2["category"]=="train"]
data_test = data_join2[data_join2["category"]=="test"]
train_x = data_train.drop(columns=['Date', 'Weekly_Sales', 'category'])
 train_y = data_train['Weekly_Sales']
test_x = data_test.drop(columns=['Date', 'Weekly_Sales', 'category'] )
sub['is_attributed'] = model.predict(xgb.DMatrix(test), ntree_limit=model.best_ntree_limit)
 sub.to_csv('xgb_sub.csv',index=False )<define_variables>
obj_col = [col for col in train_x.columns if train_x[col].dtype == 'O']
obj_col
VAL_FILE = '.. /input/training-and-validation-data-pickle/validation.pkl.gz'<define_variables>
for c in obj_col:
le = LabelEncoder()
le.fit(train_x[c].fillna('NA'))
train_x[c] = le.transform(train_x[c].fillna('NA'))
test_x[c] = le.transform(test_x[c].fillna('NA'))
almost_zero = 1e-10
almost_one = 1 - almost_zero<define_variables>
base_models = {
'lgb1 ': "assemblage of krishna's LGBM with time deltas",
'wbftl': "anttip's Wordbatch FM-FTRL",
'nngpu': "Downampled Neural Network run on GPU"
}<define_variables>
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score
cvfiles = {
'lgb1 ': '.. /input/validation-of-pas-un-m-lange/val_krishnas_r_lgb_bag3.csv',
'wbftl': '.. /input/validate-anttip-s-wordbatch-fm-ftrl-9752-version/wordbatch_fm_ftrl_val.csv',
'nngpu': '.. /input/gpu-nn-validation-more-features/gpu_val3.csv'
}<define_variables>
from sklearn.metrics import mean_absolute_error
subfiles = {
'lgb1 ': '.. /input/ceci-n-est-pas-un-m-lange/sub_krishnas_r_lgb_bag4.csv',
'wbftl': '.. /input/anttip-s-wordbatch-fm-ftrl-9752-version/wordbatch_fm_ftrl.csv',
'nngpu': '.. /input/new-talkingdata-gpu-example-with-multiple-runs/gpu_test3.csv'
}<define_variables>
lbscores = {
'lgb1 ':.9759,
'wbftl':.9752,
'nngpu':.9695
}<define_variables>
best_param = [16,8,2]
model_order = [m for m in base_models]<load_from_csv>
params = {"metric": "mae",
"max_depth":best_param[0],
"num_leaves":best_param[1],
"min_data_in_leaf":best_param[2]
}
train_scores = []
valid_scores = []
pred = np.zeros(test_x.shape[0])
 pred_valid = np.zeros(train_x.shape[0])
 kf = KFold(n_splits = 5, shuffle = True, random_state=0)
 for i,(tr_idx, va_idx)in enumerate(kf.split(train_x)) :
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
lgb_train = lgb.Dataset(tr_x, tr_y)
 lgb_eval = lgb.Dataset(va_x, va_y)
 gbm = lgb.train(params,lgb_train,valid_sets=lgb_eval,num_boost_round=10000,early_stopping_rounds=100,verbose_eval=500)
 tr_pred = gbm.predict(tr_x)
 va_pred = gbm.predict(va_x)
 pred += gbm.predict(test_x)/ 5
pred_valid[va_idx] = va_pred
train_MAE = mean_absolute_error(tr_y,tr_pred)
 valid_MAE = mean_absolute_error(va_y,va_pred)
 train_scores.append(train_MAE)
 valid_scores.append(valid_MAE)
 print(f'fold{i}: train_MAE={train_MAE} valid_MAE={valid_MAE}')
 mean_train_scores = np.mean(train_scores)
 mean_valid_scores = np.mean(valid_scores)
 print(f'mean train MAE={mean_train_scores} mean valid MAE={mean_valid_scores}' )
cvdata = pd.DataFrame({
m:pd.read_csv(cvfiles[m])['is_attributed'].rank()
for m in base_models
})
 X_train = np.array(cvdata[model_order])
 y_train = pd.read_pickle(VAL_FILE)['is_attributed']
n = len(y_train)
 X_train /= n<find_best_model_class>
True_Pred_map(tr_y,tr_pred )
X, X_val, y, y_val = train_test_split(X_train, y_train, test_size=.33, shuffle=False)
 for c in [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6]:
mod = LogisticRegression(C=c)
 mod.fit(X, y)
 val_pred = mod.predict_proba(X_val)[:,1]
print(c, roc_auc_score(y_val, mod.predict_proba(X_val)[:,1]))<train_model>
True_Pred_map(va_y,va_pred )
stack_model = LogisticRegression(C=1e6)
 stack_model.fit(X_train, y_train)
 stack_model.coef_<compute_train_metric>
weights = stack_model.coef_/stack_model.coef_.sum()
columns = cvdata[model_order].columns
scores = [ roc_auc_score(y_train, cvdata[c])for c in columns ]
names = [ base_models[c] for c in columns ]
lb = [ lbscores[c] for c in columns ]
pd.DataFrame(data={'LB score': lb, 'CV score':scores, 'weight':weights.reshape(-1)}, index=names )<compute_test_metric>
train_tmp = train_x.copy()
train_tmp["Weekly_Sales"] = train_y
train_tmp["pred"] = pred_valid
train_tmp["pred_diff"] = train_tmp['Weekly_Sales']-train_tmp["pred"]
train_tmp
print('Stacker score: ', roc_auc_score(y_train, stack_model.predict_proba(X_train)[:,1]))<load_from_csv>
train_tmp[(train_tmp["pred_diff"]<-200000)|(train_tmp["pred_diff"]>200000)]
final_sub = pd.DataFrame()
subs = {m:pd.read_csv(subfiles[m] ).rename({'is_attributed':m},axis=1)for m in base_models}
first_model = list(base_models.keys())[0]
final_sub['click_id'] = subs[first_model]['click_id']<merge>
data_join2[(data_join2["Store"] == 1)&(data_join2["Dept"] == 1)&(data_join2["Holiday_name"].notnull())]
df = subs[first_model]
for m in subs:
if m != first_model:
df = df.merge(subs[m], on='click_id')
 df.head()<predict_on_test>
test_pred = test.copy(deep=True)
 test_pred["Weekly_Sales"] = pred
test_pred[['Store', 'Dept', 'Date']] = test_pred[['Store', 'Dept', 'Date']].astype(str)
 test_pred["id"] = test_pred["Store"] + "_" + test_pred["Dept"] + "_" + test_pred["Date"]
X_test = np.array(df.drop(['click_id'],axis=1)[model_order].rank() /df.shape[0])
 final_sub['is_attributed'] = stack_model.predict_proba(X_test)[:,1]
final_sub.head(10 )<feature_engineering>
test_pred["Date"] = pd.to_datetime(test_pred["Date"])
 test_pred["YEAR"] = test_pred["Date"].dt.year
test_pred["MONTH"] = test_pred["Date"].dt.month
test_pred["WEEK"] = test_pred["Date"].dt.week
test_pred
final_sub['is_attributed'] = final_sub['is_attributed'].rank(method='dense')/ 1e8
pd.options.display.float_format =('{:,.8f}' ).format
final_sub.head(10 )<save_to_csv>
test_pred[['Store', 'Dept']] = test_pred[['Store', 'Dept']].astype(int)
 test_pred_bef = pd.merge(test_pred, data_sales, on=["Dept","Store","YEAR","WEEK"], how="left")
 test_pred_bef[(test_pred_bef["Dept"] == 72)&(test_pred_bef["WEEK"] == 47)]
final_sub.to_csv("sub_stacked.csv", index=False, float_format='%.8f' )<set_options>
test_pred_bef["Weekly_Sales"] = np.where(( test_pred_bef["Dept"] == 72)&(test_pred_bef["WEEK"] == 47), test_pred_bef["Weekly_Sale_before"], test_pred_bef["Weekly_Sales"])
 test_pred_bef[(test_pred_bef["Dept"] == 72)&(test_pred_bef["WEEK"] == 47)]
warnings.filterwarnings("ignore" )<load_from_csv>
test_pred2 = test_pred_bef.copy()
test_pred2["Weekly_Sales"] = np.where(( test_pred2["YEAR"]==2012)&(test_pred2["WEEK"]==52)&(test_pred2["Weekly_Sales"].shift(1)> 2*test_pred2["Weekly_Sales"])\
,test_pred2["Weekly_Sales"]+test_pred2["Weekly_Sales"].shift(1)*2.5/7, test_pred2["Weekly_Sales"] )
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 train_size = int(len(train)*0.8)
 train = train.sort_values(['YrSold', 'MoSold'], axis=0 ).reset_index(drop=True)
 train_X = train.drop(['SalePrice', 'Id'], axis=1 ).loc[:train_size-1, :]
valid_X = train.drop(['SalePrice', 'Id'], axis=1 ).loc[train_size:, :]
train_y = train['SalePrice'][:train_size]
valid_y = train['SalePrice'][train_size:]
test_X = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv', index_col='Id')
 test_X = test_X.sort_values(['YrSold', 'MoSold'], axis=0 )<categorify>
test_pred3 = test_pred2[["id","Weekly_Sales"]]
test_pred3.head()
class CustomImputer:
def __init__(self):
pass
def fit(self, data):
cat_cols = data.columns[data.dtypes == 'object']
self.impute_cols = cat_cols[data[cat_cols].isna().sum() > 0]
def transform(self, data):
for column in self.impute_cols:
if data[column].isin(['None', 'No', 'Othr'] ).sum() > 0:
replace_value = data.loc[data[column].isin(['None', 'No', 'Othr']), column].unique() [0]
data[column] = data[column].fillna(replace_value)
 else:
data[column] = data[column].fillna('None')
 return data
def fit_transform(self, data):
self.fit(data)
 return self.transform(data)
 def get_params(self):
pass<categorify>
submission = test_pred3[["id","Weekly_Sales"]]
submission.to_csv('submission15.csv', index=False )