Kaggle competition: Rossmann Store Sales
<SOS> metric: RootMeanSquarePercentageError Kaggle data source: rossmann-store-sales<categorify>
%matplotlib inline
df = phase0_output.join(probs.set_index('patientId'))
out = df.copy()
out.loc[df.prob<THRESH,'PredictionString'] = np.nan
out.loc[df.PredictionString=='','PredictionString'] = np.nan
phase1_output = out.drop(['prob'],axis=1 )<load_from_csv>
train = pd.read_csv('.. /input/rossmann-store-sales/train.csv')
 test = pd.read_csv('.. /input/rossmann-store-sales/test.csv')
 store = pd.read_csv('.. /input/rossmann-store-sales/store.csv' )
df1 = pd.read_csv(KERNEL_OUTPUT_PATH ).set_index('patientId')
 df2 = phase1_output
df1_cases = df1[~df1.PredictionString.isnull() ]
df2_cases = df2[~df2.PredictionString.isnull() ]
df1_pos_ids = df1_cases.index.values
df2_pos_ids = df2_cases.index.values<feature_engineering>
train = train.loc[train.Open != 0]
train = train.loc[train.Sales > 0].reset_index(drop=True )
df1_only_dict = {}
for p in df1_pos_ids:
if not p in df2_pos_ids:
df1_only_dict[p] = float(df1_cases.loc[p,'PredictionString'].split(' ')[0] )<feature_engineering>
train[train.isnull().values==True]
df2_only_dict = {}
for p in df2_pos_ids:
if not p in df1_pos_ids:
df2_only_dict[p] = float(df2_cases.loc[p,'PredictionString'].split(' ')[0] )<prepare_output>
test[test.isnull().values==True]
df_out = df1.copy()
for p,r in df1.iterrows() :
if p in df2_only_dict:
if df2_only_dict[p] > MIN_MAX_MRCNN_CONF:
df_out.loc[p,'PredictionString'] = df2.loc[p,'PredictionString']
phase2_output = df_out<load_from_csv>
train = pd.merge(train, store, on='Store')
 test = pd.merge(test, store, on='Store' )
input_dir = RESULTS_LOC
infile_stem = OOF_INFILE_STEM
oof_preds_input = []
for ifold in range(N_FOLDS):
fp_in = os.path.join(input_dir, infile_stem + str(ifold)+ IN_OOF_SUFFIX)
 oof_preds_input.append(pd.read_csv(fp_in))
oof_preds_input[-1]['fold'] = ifold
oof_preds = pd.concat(oof_preds_input ).set_index('patientId' )<prepare_x_and_y>
def build_features(features, data):
features.extend(['Store','CompetitionDistance','CompetitionOpenSinceMonth','StateHoliday','StoreType','Assortment',
'SchoolHoliday','CompetitionOpenSinceYear', 'Promo', 'Promo2', 'Promo2SinceWeek', 'Promo2SinceYear'])
 features.extend(['Year','Month','Day','DayOfWeek','WeekOfYear'])
 data['Year'] = data.Date.dt.year
data['Month'] = data.Date.dt.month
data['Day'] = data.Date.dt.day
data['DayOfWeek'] = data.Date.dt.dayofweek
data['WeekOfYear'] = data.Date.dt.weekofyear
features.extend(['CompetitionOpen','PromoOpen'])
 data['CompetitionOpen'] = 12*(data.Year-data.CompetitionOpenSinceYear)+(data.Month-data.CompetitionOpenSinceMonth)
 data['PromoOpen'] = 12*(data.Year-data.Promo2SinceYear)+(data.WeekOfYear-data.Promo2SinceWeek)/4.0
data['CompetitionOpen'] = data.CompetitionOpen.apply(lambda x: x if x > 0 else 0)
 data['PromoOpen'] = data.PromoOpen.apply(lambda x: x if x > 0 else 0)
 features.append('IsPromoMonth')
 month2str = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug', 9:'Sept', 10:'Oct', 11:'Nov', 12:'Dec'}
data['monthStr'] = data.Month.map(month2str)
 data.loc[data.PromoInterval==0, 'PromoInterval'] = ''
data['IsPromoMonth'] = 0
for interval in data.PromoInterval.unique() :
if interval != '':
for month in interval.split(','):
data.loc[(data.monthStr == month)&(data.PromoInterval == interval), 'IsPromoMonth'] = 1
mappings = {'0':0, 'a':1, 'b':2, 'c':3, 'd':4}
data.StoreType.replace(mappings, inplace=True)
 data.Assortment.replace(mappings, inplace=True)
 data.StateHoliday.replace(mappings, inplace=True)
 data['StoreType'] = data['StoreType'].astype(int)
 data['Assortment'] = data['Assortment'].astype(int)
 data['StateHoliday'] = data['StateHoliday'].astype(int )
probs_dict = {}
for p, r in oof_preds.iterrows() :
probs_dict[p] = MINPROB
oof_positive = oof_preds[~oof_preds.PredictionString.isnull() ]
for p, r in oof_positive.iterrows() :
s = r.PredictionString.split(' ')
 if s[-1]=='':
s.pop()
if s[0]=='':
s.pop(0)
 if(len(s)%5):
print('Bad prediction string.')
 while len(s):
conf = float(s.pop(0))
x = int(round(float(s.pop(0))))
y = int(round(float(s.pop(0))))
w = int(round(float(s.pop(0))))
h = int(round(float(s.pop(0))))
if conf>probs_dict[p]:
probs_dict[p] = conf
prob_df = pd.DataFrame(probs_dict,index=['prob'] ).transpose()<count_duplicates>
train.Date = pd.to_datetime(train.Date, errors='coerce')
 test.Date = pd.to_datetime(test.Date, errors='coerce')
 features = []
build_features(features, train)
 build_features([], test)
 print(features )
input_dir = RAW_DATA_LOC
fp = os.path.join(input_dir, 'stage_2_train_labels.csv')
 act = pd.read_csv(fp ).set_index('patientId' ).rename(columns={'Target':'actual'})[['actual']]
act = act[~act.index.duplicated() ]<merge>
def ToWeight(y):
w = np.zeros(y.shape, dtype=float)
 ind = y != 0
w[ind] = 1./(y[ind]**2)
 return w
def rmspe(yhat, y):
w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y-yhat)**2))
return rmspe
def rmspe_xg(yhat, y):
y = y.get_label()
y = np.expm1(y)
 yhat = np.expm1(yhat)
 w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y-yhat)**2))
return "rmspe", rmspe
def neg_rmspe(yhat, y):
y = np.expm1(y)
 yhat = np.expm1(yhat)
 w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y-yhat)**2))
return -rmspe
df = act.join(prob_df,how='right')
 df.head()<train_model>
regressor = DecisionTreeRegressor(random_state=2)
 cv_sets = ShuffleSplit(n_splits=5, test_size=0.2)
 params = {'max_depth':range(10,40,2)}
scoring_fnc = make_scorer(neg_rmspe)
 grid = GridSearchCV(regressor,params,scoring_fnc,cv=cv_sets)
 grid = grid.fit(train[features], np.log1p(train.Sales))
DTR = grid.best_estimator_
rawprobs = df.prob.values.reshape(-1,1)
 lr = LogisticRegression(C=C1)
 lr.fit(logit(rawprobs),df.actual)
 b1 = lr.coef_[0,0]
b0 = lr.intercept_[0]
b0, b1<define_variables>
DTR.get_params()
folds = []
for ifold in range(N_FOLDS):
folds.append(oof_preds[oof_preds.fold==ifold].index.values )<prepare_x_and_y>
submission = pd.DataFrame({"Id": test["Id"], "Sales": np.expm1(DTR.predict(test[features])) })
 submission.to_csv("benchmark.csv", index=False )
def run_logistic_by_fold() :
df = act.join(prob_df,how='right')
 logits = df.prob.apply(logit)
 df['oofprob'] = np.nan
b1 = []
b0 = []
for i in range(len(folds)) :
fs = folds.copy()
te = fs.pop(i)
 tr = np.concatenate(fs)
 Xtr = logits[tr].values.reshape(-1,1)
 Xte = logits[te].values.reshape(-1,1)
 ytr = df.actual[tr].copy()
yte = df.actual[te].copy()
lr.fit(Xtr, ytr)
 df.loc[te,'oofprob'] = lr.predict_proba(Xte)[:,1]
b1.append(lr.coef_[0,0])
 b0.append(lr.intercept_[0])
 coefs = pd.DataFrame({'b0':b0, 'b1':b1}, index=range(len(folds)))
 coefs.index.name = 'fold'
return(df.sort_values('oofprob'), coefs )<choose_model_class>
params = {'objective': 'reg:linear',
'eta': 0.01,
'max_depth': 11,
'subsample': 0.5,
'colsample_bytree': 0.5,
'silent': 1,
'seed': 1
}
num_trees = 10000
lr = LogisticRegression(C=C2 )<feature_engineering>
X_train, X_test = train_test_split(train, test_size=0.2, random_state=2)
 dtrain = xgb.DMatrix(X_train[features], np.log1p(X_train.Sales))
dvalid = xgb.DMatrix(X_test[features], np.log1p(X_test.Sales))
dtest = xgb.DMatrix(test[features])
 watchlist = [(dtrain, 'train'),(dvalid, 'eval')]
gbm = xgb.train(params, dtrain, num_trees, evals=watchlist, early_stopping_rounds=50, feval=rmspe_xg, verbose_eval=False )
<define_variables><EOS> 
test_probs = gbm.predict(xgb.DMatrix(test[features]), ntree_limit=gbm.best_ntree_limit)
 indices = test_probs < 0
test_probs[indices] = 0
submission = pd.DataFrame({"Id": test["Id"], "Sales": np.expm1(test_probs)})
 submission.to_csv("xgboost.csv", index=False )