Kaggle competition: Rossmann Store Sales
<SOS> metric: RootMeanSquarePercentageError Kaggle data source: rossmann-store-sales<save_to_csv>
%matplotlib inline
output_dir = OUTPUT_LOC
xform_oof_preds.to_csv(os.path.join(output_dir, OOF_OUTFILE_NAME))<define_variables>
train = pd.read_csv('.. /input/rossmann-store-sales/train.csv')
 test = pd.read_csv('.. /input/rossmann-store-sales/test.csv')
 store = pd.read_csv('.. /input/rossmann-store-sales/store.csv' )
input_dir = RESULTS_LOC
output_dir = OUTPUT_LOC
infile_stem = TEST_INFILE_STEM
outfile_stem = TEST_OUTFILE_STEM<load_from_csv>
train = train.loc[train.Open != 0]
train = train.loc[train.Sales > 0].reset_index(drop=True )
for ifold in range(N_FOLDS):
fp_in = os.path.join(input_dir, infile_stem + str(ifold)+ IN_SUFFIX)
 fp_out = os.path.join(output_dir, outfile_stem + str(ifold)+ OUT_SUFFIX)
 inpreds = pd.read_csv(fp_in ).set_index('patientId')
 pat_dict = {}
for pat,row in inpreds.iterrows() :
pred = row.PredictionString
boxes = []
confs = []
if isinstance(pred, str):
s = pred.split(' ')
 if s[-1]=='':
s.pop()
if s[0]=='':
s.pop(0)
 if(len(s)%5):
print('Bad prediction string.')
 while len(s):
conf = float(s.pop(0))
x = int(round(float(s.pop(0))))
y = int(round(float(s.pop(0))))
w = int(round(float(s.pop(0))))
h = int(round(float(s.pop(0))))
boxes.append([x,y,w,h])
 confs.append(expit(b0+b1*logit(conf)))
 predictionString = ''
if len(boxes):
for box, conf in zip(boxes, confs):
x, y, w, h = box
predictionString += '{:6.4f} '.format(conf)
 predictionString += str(x)+ ' ' + str(y)+ ' ' + str(w)+ ' ' + str(h)+ ' '
pat_dict[pat] = predictionString
outpreds = pd.DataFrame(pat_dict,
index=['PredictionString'] ).transpose()
outpreds.index.name = 'patientId'
outpreds.to_csv(fp_out)
 fp_out<save_to_csv>
train[train.isnull().values==True]
outprobs = prob_df.join(oof_preds[['fold']])
 b0_ = coefs.b0[outprobs.fold].values
b1_ = coefs.b1[outprobs.fold].values
outprobs['prob'] = expit(b0_ + logit(outprobs.prob)*b1_)
 outprobs.index.name = 'patientId'
output_dir = OUTPUT_LOC
outprobs.to_csv(os.path.join(output_dir, OOF_CLASS_PROBS_OUTFILE))
outprobs.head()<load_from_csv>
test[test.isnull().values==True]
output_dir = OUTPUT_LOC
box_outfile_stem = TEST_OUTFILE_STEM
probs_outfile_stem = TEST_CLASS_PROBS_OUTFILE_STEM
minprob_out = expit(b0 + b1*logit(MINPROB))
for ifold in range(N_FOLDS):
fp_in = os.path.join(output_dir, box_outfile_stem + str(ifold)+ OUT_SUFFIX)
 inpreds = pd.read_csv(fp_in ).set_index('patientId')
 fp_out = os.path.join(output_dir, probs_outfile_stem + str(ifold)+ OUT_SUFFIX)
 probs_dict = {}
for p, r in inpreds.iterrows() :
probs_dict[p] = minprob_out
inpreds_positive = inpreds[~inpreds.PredictionString.isnull() ]
for p, r in inpreds_positive.iterrows() :
s = r.PredictionString.split(' ')
 if s[-1]=='':
s.pop()
if s[0]=='':
s.pop(0)
 if(len(s)%5):
print('Bad prediction string.')
 while len(s):
conf = float(s.pop(0))
s.pop(0)
 s.pop(0)
 s.pop(0)
 s.pop(0)
 if conf>probs_dict[p]:
probs_dict[p] = conf
outprob_df = pd.DataFrame(probs_dict,index=['prob'] ).transpose()
outprob_df.index.name = 'patientId'
outprob_df.to_csv(fp_out)
 if not ifold:
allprobs_df = outprob_df.rename(columns={'prob':ifold})
 else:
allprobs_df = allprobs_df.join(outprob_df.rename(columns={'prob':ifold}))<load_from_csv>
train = pd.merge(train, store, on='Store')
 test = pd.merge(test, store, on='Store' )
fp = OOF_CLASS_PROBS_OUTFILE
oof_probs_mrcnn = pd.read_csv(fp ).set_index('patientId' ).rename(columns={'prob':'ph'} )<load_from_csv>
def build_features(features, data):
features.extend(['Store','CompetitionDistance','CompetitionOpenSinceMonth','StateHoliday','StoreType','Assortment',
'SchoolHoliday','CompetitionOpenSinceYear', 'Promo', 'Promo2', 'Promo2SinceWeek', 'Promo2SinceYear'])
 features.extend(['Year','Month','Day','DayOfWeek','WeekOfYear'])
 data['Year'] = data.Date.dt.year
data['Month'] = data.Date.dt.month
data['Day'] = data.Date.dt.day
data['DayOfWeek'] = data.Date.dt.dayofweek
data['WeekOfYear'] = data.Date.dt.weekofyear
features.extend(['CompetitionOpen','PromoOpen'])
 data['CompetitionOpen'] = 12*(data.Year-data.CompetitionOpenSinceYear)+(data.Month-data.CompetitionOpenSinceMonth)
 data['PromoOpen'] = 12*(data.Year-data.Promo2SinceYear)+(data.WeekOfYear-data.Promo2SinceWeek)/4.0
data['CompetitionOpen'] = data.CompetitionOpen.apply(lambda x: x if x > 0 else 0)
 data['PromoOpen'] = data.PromoOpen.apply(lambda x: x if x > 0 else 0)
 features.append('IsPromoMonth')
 month2str = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug', 9:'Sept', 10:'Oct', 11:'Nov', 12:'Dec'}
data['monthStr'] = data.Month.map(month2str)
 data.loc[data.PromoInterval==0, 'PromoInterval'] = ''
data['IsPromoMonth'] = 0
for interval in data.PromoInterval.unique() :
if interval != '':
for month in interval.split(','):
data.loc[(data.monthStr == month)&(data.PromoInterval == interval), 'IsPromoMonth'] = 1
mappings = {'0':0, 'a':1, 'b':2, 'c':3, 'd':4}
data.StoreType.replace(mappings, inplace=True)
 data.Assortment.replace(mappings, inplace=True)
 data.StateHoliday.replace(mappings, inplace=True)
 data['StoreType'] = data['StoreType'].astype(int)
 data['Assortment'] = data['Assortment'].astype(int)
 data['StateHoliday'] = data['StateHoliday'].astype(int )
for i in range(N_FOLDS):
fp = TEST_CLASS_PROBS_OUTFILE_STEM + str(i)+ '.csv'
indf = pd.read_csv(fp ).set_index('patientId' ).rename(columns={'prob':'ph'+str(i)})
 if i:
test_prob_df = test_prob_df.join(indf)
 else:
test_prob_df = indf<load_from_csv>
train.Date = pd.to_datetime(train.Date, errors='coerce')
 test.Date = pd.to_datetime(test.Date, errors='coerce')
 features = []
build_features(features, train)
 build_features([], test)
 print(features )
f0 = pd.read_csv(RESULTS_LOC + '/' + ADEMSE_OOF_STEM + '0.csv' ).set_index('patientId')
 f1 = pd.read_csv(RESULTS_LOC + '/' + ADEMSE_OOF_STEM + '1.csv' ).set_index('patientId')
 f2 = pd.read_csv(RESULTS_LOC + '/' + ADEMSE_OOF_STEM + '2.csv' ).set_index('patientId')
 f3 = pd.read_csv(RESULTS_LOC + '/' + ADEMSE_OOF_STEM + '3.csv' ).set_index('patientId')
 f4 = pd.read_csv(RESULTS_LOC + '/' + ADEMSE_OOF_STEM + '4.csv' ).set_index('patientId' )<concatenate>
def ToWeight(y):
w = np.zeros(y.shape, dtype=float)
 ind = y != 0
w[ind] = 1./(y[ind]**2)
 return w
def rmspe(yhat, y):
w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y-yhat)**2))
return rmspe
def rmspe_xg(yhat, y):
y = y.get_label()
y = np.expm1(y)
 yhat = np.expm1(yhat)
 w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y-yhat)**2))
return "rmspe", rmspe
def neg_rmspe(yhat, y):
y = np.expm1(y)
 yhat = np.expm1(yhat)
 w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y-yhat)**2))
return -rmspe
den = pd.concat([f0,f1,f2,f3,f4],axis=0)
 print(den.shape)
 den.head()<load_from_csv>
regressor = DecisionTreeRegressor(random_state=2)
 cv_sets = ShuffleSplit(n_splits=5, test_size=0.2)
 params = {'max_depth':range(10,40,2)}
scoring_fnc = make_scorer(neg_rmspe)
 grid = GridSearchCV(regressor,params,scoring_fnc,cv=cv_sets)
 grid = grid.fit(train[features], np.log1p(train.Sales))
DTR = grid.best_estimator_
tf0 = pd.read_csv(RESULTS_LOC + '/' + ADENSE_TEST_STEM + '0.csv' ).set_index('patientId')
 tf1 = pd.read_csv(RESULTS_LOC + '/' + ADENSE_TEST_STEM + '1.csv' ).set_index('patientId')
 tf2 = pd.read_csv(RESULTS_LOC + '/' + ADENSE_TEST_STEM + '2.csv' ).set_index('patientId')
 tf3 = pd.read_csv(RESULTS_LOC + '/' + ADENSE_TEST_STEM + '3.csv' ).set_index('patientId')
 tf4 = pd.read_csv(RESULTS_LOC + '/' + ADENSE_TEST_STEM + '4.csv' ).set_index('patientId' )<merge>
DTR.get_params()
test_den = tf0.rename(columns={'predicted':'pa0'})
 test_den = test_den.join(tf1.rename(columns={'predicted':'pa1'}))
test_den = test_den.join(tf2.rename(columns={'predicted':'pa2'}))
test_den = test_den.join(tf3.rename(columns={'predicted':'pa3'}))
test_den = test_den.join(tf4.rename(columns={'predicted':'pa4'}))
print(test_den.shape)
 test_den.head()<prepare_x_and_y>
submission = pd.DataFrame({"Id": test["Id"], "Sales": np.expm1(DTR.predict(test[features])) })
 submission.to_csv("benchmark.csv", index=False )
p0 = f0.index.values
p1 = f1.index.values
p2 = f2.index.values
p3 = f3.index.values
p4 = f4.index.values
pt = test_den.index.values<load_from_csv>
params = {'objective': 'reg:linear',
'eta': 0.01,
'max_depth': 11,
'subsample': 0.5,
'colsample_bytree': 0.5,
'silent': 1,
'seed': 1
}
num_trees = 10000
c0 = pd.read_csv(RESULTS_LOC + '/' + CHEXNET_OOF_STEM + '0.csv' ).set_index('patientId')
 c1 = pd.read_csv(RESULTS_LOC + '/' + CHEXNET_OOF_STEM + '1.csv' ).set_index('patientId')
 c2 = pd.read_csv(RESULTS_LOC + '/' + CHEXNET_OOF_STEM + '2.csv' ).set_index('patientId')
 c3 = pd.read_csv(RESULTS_LOC + '/' + CHEXNET_OOF_STEM + '3.csv' ).set_index('patientId')
 c4 = pd.read_csv(RESULTS_LOC + '/' + CHEXNET_OOF_STEM + '4.csv' ).set_index('patientId')
 chex = pd.concat([c0,c1,c2,c3,c4],axis=0)
 print(chex.shape)
 chex.head()<merge>
X_train, X_test = train_test_split(train, test_size=0.2, random_state=2)
 dtrain = xgb.DMatrix(X_train[features], np.log1p(X_train.Sales))
dvalid = xgb.DMatrix(X_test[features], np.log1p(X_test.Sales))
dtest = xgb.DMatrix(test[features])
 watchlist = [(dtrain, 'train'),(dvalid, 'eval')]
gbm = xgb.train(params, dtrain, num_trees, evals=watchlist, early_stopping_rounds=50, feval=rmspe_xg, verbose_eval=False )
<load_from_csv><EOS> 
test_probs = gbm.predict(xgb.DMatrix(test[features]), ntree_limit=gbm.best_ntree_limit)
 indices = test_probs < 0
test_probs[indices] = 0
submission = pd.DataFrame({"Id": test["Id"], "Sales": np.expm1(test_probs)})
 submission.to_csv("xgboost.csv", index=False )