Kaggle competition: Recruit Restaurant Visitor Forecasting
<SOS> metric: RMSLE Kaggle data source: recruit-restaurant-visitor-forecasting<load_pretrained>
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=512, shuffle=False )<load_pretrained>
data = {
'tra': pd.read_csv('.. /input/air_visit_data.csv'),
'as': pd.read_csv('.. /input/air_store_info.csv'),
'hs': pd.read_csv('.. /input/hpg_store_info.csv'),
'ar': pd.read_csv('.. /input/air_reserve.csv'),
'hr': pd.read_csv('.. /input/hpg_reserve.csv'),
'id': pd.read_csv('.. /input/store_id_relation.csv'),
'tes': pd.read_csv('.. /input/sample_submission.csv'),
'hol': pd.read_csv('.. /input/date_info.csv' ).rename(columns={'calendar_date':'visit_date'})
 }
data['hr'] = pd.merge(data['hr'], data['id'], how='inner', on=['hpg_store_id'])
 for df in ['ar','hr']:
data[df]['visit_datetime'] = pd.to_datetime(data[df]['visit_datetime'])
 data[df]['visit_dow'] = data[df]['visit_datetime'].dt.dayofweek
data[df]['visit_datetime'] = data[df]['visit_datetime'].dt.date
data[df]['reserve_datetime'] = pd.to_datetime(data[df]['reserve_datetime'])
 data[df]['reserve_datetime'] = data[df]['reserve_datetime'].dt.date
data[df]['reserve_datetime_diff'] = data[df].apply(lambda r:(r['visit_datetime'] - r['reserve_datetime'] ).days, axis=1)
 data[df] = data[df][data[df]['reserve_datetime_diff'] > data[df]['visit_dow']]
tmp1 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].sum().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs1', 'reserve_visitors':'rv1'})
 tmp2 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].mean().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs2', 'reserve_visitors':'rv2'})
 data[df] = pd.merge(tmp1, tmp2, how='inner', on=['air_store_id','visit_date'])
 data['tra']['visit_date'] = pd.to_datetime(data['tra']['visit_date'])
 data['tra']['dow'] = data['tra']['visit_date'].dt.dayofweek
data['tra']['doy'] = data['tra']['visit_date'].dt.dayofyear
data['tra']['year'] = data['tra']['visit_date'].dt.year
data['tra']['month'] = data['tra']['visit_date'].dt.month
data['tra']['week'] = data['tra']['visit_date'].dt.week
data['tra']['visit_date'] = data['tra']['visit_date'].dt.date
data['tes']['visit_date'] = data['tes']['id'].map(lambda x: str(x ).split('_')[2])
 data['tes']['air_store_id'] = data['tes']['id'].map(lambda x: '_'.join(x.split('_')[:2]))
data['tes']['visit_date'] = pd.to_datetime(data['tes']['visit_date'])
 data['tes']['dow'] = data['tes']['visit_date'].dt.dayofweek
data['tes']['doy'] = data['tes']['visit_date'].dt.dayofyear
data['tes']['year'] = data['tes']['visit_date'].dt.year
data['tes']['month'] = data['tes']['visit_date'].dt.month
data['tes']['week'] = data['tes']['visit_date'].dt.week
data['tes']['visit_date'] = data['tes']['visit_date'].dt.date
unique_stores = data['tes']['air_store_id'].unique()
stores = pd.concat([pd.DataFrame({'air_store_id': unique_stores, 'dow': [i]*len(unique_stores)})for i in range(7)], axis=0, ignore_index=True ).reset_index(drop=True)
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].min().rename(columns={'visitors':'min_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].mean().rename(columns={'visitors':'mean_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].median().rename(columns={'visitors':'median_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].count().rename(columns={'visitors':'count_observations'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 stores = pd.merge(stores, data['as'], how='left', on=['air_store_id'])
 stores['air_genre_name'] = stores['air_genre_name'].map(lambda x: str(str(x ).replace('/',' ')))
 stores['air_area_name'] = stores['air_area_name'].map(lambda x: str(str(x ).replace('-',' ')))
 lbl = preprocessing.LabelEncoder()
for i in range(10):
stores['air_genre_name'+str(i)] = lbl.fit_transform(stores['air_genre_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_area_name'+str(i)] = lbl.fit_transform(stores['air_area_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_genre_name'] = lbl.fit_transform(stores['air_genre_name'])
 stores['air_area_name'] = lbl.fit_transform(stores['air_area_name'])
 data['hol']['visit_date'] = pd.to_datetime(data['hol']['visit_date'])
 data['hol']['day_of_week'] = lbl.fit_transform(data['hol']['day_of_week'])
 data['hol']['visit_date'] = data['hol']['visit_date'].dt.date
train = pd.merge(data['tra'], data['hol'], how='left', on=['visit_date'])
 test = pd.merge(data['tes'], data['hol'], how='left', on=['visit_date'])
 train = pd.merge(train, stores, how='inner', on=['air_store_id','dow'])
 test = pd.merge(test, stores, how='left', on=['air_store_id','dow'])
 for df in ['ar','hr']:
train = pd.merge(train, data[df], how='left', on=['air_store_id','visit_date'])
 test = pd.merge(test, data[df], how='left', on=['air_store_id','visit_date'])
 train['id'] = train.apply(lambda r: '_'.join([str(r['air_store_id']), str(r['visit_date'])]), axis=1)
 train['total_reserv_sum'] = train['rv1_x'] + train['rv1_y']
train['total_reserv_mean'] =(train['rv2_x'] + train['rv2_y'])/ 2
train['total_reserv_dt_diff_mean'] =(train['rs2_x'] + train['rs2_y'])/ 2
test['total_reserv_sum'] = test['rv1_x'] + test['rv1_y']
test['total_reserv_mean'] =(test['rv2_x'] + test['rv2_y'])/ 2
test['total_reserv_dt_diff_mean'] =(test['rs2_x'] + test['rs2_y'])/ 2
train['date_int'] = train['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 test['date_int'] = test['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 train['var_max_lat'] = train['latitude'].max() - train['latitude']
train['var_max_long'] = train['longitude'].max() - train['longitude']
test['var_max_lat'] = test['latitude'].max() - test['latitude']
test['var_max_long'] = test['longitude'].max() - test['longitude']
train['lon_plus_lat'] = train['longitude'] + train['latitude']
test['lon_plus_lat'] = test['longitude'] + test['latitude']
lbl = preprocessing.LabelEncoder()
train['air_store_id2'] = lbl.fit_transform(train['air_store_id'])
 test['air_store_id2'] = lbl.transform(test['air_store_id'])
 col = [c for c in train if c not in ['id', 'air_store_id', 'visit_date','visitors']]
train = train.fillna(-1)
 test = test.fillna(-1 )
my_bert_list = os.listdir('.. /input/vanilla-epoch1/')
 bert_full_list = os.listdir('.. /input/bert-full-epoch1/')
 test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=512, shuffle=False )<define_variables>
def RMSLE(y, pred):
return metrics.mean_squared_error(y, pred)**0.5
MY_BERT_PATH = Path('.. /input/vanilla-epoch1/')
 BERT_FULL_PATH = Path('.. /input/bert-full-epoch1/')
 bert_test_preds = []<choose_model_class>
value_col = ['holiday_flg','min_visitors','mean_visitors','median_visitors',
'count_observations',
'rs1_x','rv1_x','rs2_x','rv2_x','rs1_y','rv1_y','rs2_y','rv2_y','total_reserv_sum','total_reserv_mean',
'total_reserv_dt_diff_mean','date_int','var_max_lat','var_max_long','lon_plus_lat']
nn_col = value_col + ['dow', 'year', 'month', 'air_store_id2', 'air_area_name', 'air_genre_name',
'air_area_name0', 'air_area_name1', 'air_area_name2', 'air_area_name3', 'air_area_name4',
'air_area_name5', 'air_area_name6', 'air_genre_name0', 'air_genre_name1',
'air_genre_name2', 'air_genre_name3', 'air_genre_name4']
X = train.copy()
X_test = test[nn_col].copy()
value_scaler = preprocessing.MinMaxScaler()
for vcol in value_col:
X[vcol] = value_scaler.fit_transform(X[vcol].values.astype(np.float64 ).reshape(-1, 1))
X_test[vcol] = value_scaler.transform(X_test[vcol].values.astype(np.float64 ).reshape(-1, 1))
X_train = list(X[nn_col].T.as_matrix())
 Y_train = np.log1p(X['visitors'] ).values
nn_train = [X_train, Y_train]
nn_test = [list(X_test[nn_col].T.as_matrix())]
print("Train and test data prepared" )
%%time
for i, weight in enumerate(my_bert_list):
if i == 4:
break
print("{} bert model prediction starts".format(i+1))
model = BertForSequenceClassification(bert_config, num_labels=7)
 model.load_state_dict(torch.load(MY_BERT_PATH / weight))
model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
test_preds = np.zeros(( len(X_test)))
 for i,(x_batch,)in enumerate(test_loader):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 test_preds[i * 512:(i + 1)* 512] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()
bert_test_preds.append(test_pred)
 del test_pred, model
gc.collect()<choose_model_class>
def get_nn_complete_model(train, hidden1_neurons=35, hidden2_neurons=15):

 K.clear_session()
air_store_id = Input(shape=(1,), dtype='int32', name='air_store_id')
 air_store_id_emb = Embedding(len(train['air_store_id2'].unique())+ 1, 15, input_shape=(1,),
name='air_store_id_emb' )(air_store_id)
 air_store_id_emb = keras.layers.Flatten(name='air_store_id_emb_flatten' )(air_store_id_emb)
 dow = Input(shape=(1,), dtype='int32', name='dow')
 dow_emb = Embedding(8, 3, input_shape=(1,), name='dow_emb' )(dow)
 dow_emb = keras.layers.Flatten(name='dow_emb_flatten' )(dow_emb)
 month = Input(shape=(1,), dtype='int32', name='month')
 month_emb = Embedding(13, 3, input_shape=(1,), name='month_emb' )(month)
 month_emb = keras.layers.Flatten(name='month_emb_flatten' )(month_emb)
 air_area_name, air_genre_name = [], []
air_area_name_emb, air_genre_name_emb = [], []
for i in range(7):
area_name_col = 'air_area_name' + str(i)
 air_area_name.append(Input(shape=(1,), dtype='int32', name=area_name_col))
tmp = Embedding(len(train[area_name_col].unique()), 3, input_shape=(1,),
name=area_name_col + '_emb' )(air_area_name[-1])
 tmp = keras.layers.Flatten(name=area_name_col + '_emb_flatten' )(tmp)
 air_area_name_emb.append(tmp)
 if i > 4:
continue
area_genre_col = 'air_genre_name' + str(i)
 air_genre_name.append(Input(shape=(1,), dtype='int32', name=area_genre_col))
tmp = Embedding(len(train[area_genre_col].unique()), 3, input_shape=(1,),
name=area_genre_col + '_emb' )(air_genre_name[-1])
 tmp = keras.layers.Flatten(name=area_genre_col + '_emb_flatten' )(tmp)
 air_genre_name_emb.append(tmp)
 air_genre_name_emb = keras.layers.concatenate(air_genre_name_emb)
 air_genre_name_emb = Dense(4, activation='sigmoid', name='final_air_genre_emb' )(air_genre_name_emb)
 air_area_name_emb = keras.layers.concatenate(air_area_name_emb)
 air_area_name_emb = Dense(4, activation='sigmoid', name='final_air_area_emb' )(air_area_name_emb)
 air_area_code = Input(shape=(1,), dtype='int32', name='air_area_code')
 air_area_code_emb = Embedding(len(train['air_area_name'].unique()), 8, input_shape=(1,), name='air_area_code_emb' )(air_area_code)
 air_area_code_emb = keras.layers.Flatten(name='air_area_code_emb_flatten' )(air_area_code_emb)
 air_genre_code = Input(shape=(1,), dtype='int32', name='air_genre_code')
 air_genre_code_emb = Embedding(len(train['air_genre_name'].unique()), 5, input_shape=(1,),
name='air_genre_code_emb' )(air_genre_code)
 air_genre_code_emb = keras.layers.Flatten(name='air_genre_code_emb_flatten' )(air_genre_code_emb)
 holiday_flg = Input(shape=(1,), dtype='float32', name='holiday_flg')
 year = Input(shape=(1,), dtype='float32', name='year')
 min_visitors = Input(shape=(1,), dtype='float32', name='min_visitors')
 mean_visitors = Input(shape=(1,), dtype='float32', name='mean_visitors')
 median_visitors = Input(shape=(1,), dtype='float32', name='median_visitors')
 count_observations = Input(shape=(1,), dtype='float32', name='count_observations')
 rs1_x = Input(shape=(1,), dtype='float32', name='rs1_x')
 rv1_x = Input(shape=(1,), dtype='float32', name='rv1_x')
 rs2_x = Input(shape=(1,), dtype='float32', name='rs2_x')
 rv2_x = Input(shape=(1,), dtype='float32', name='rv2_x')
 rs1_y = Input(shape=(1,), dtype='float32', name='rs1_y')
 rv1_y = Input(shape=(1,), dtype='float32', name='rv1_y')
 rs2_y = Input(shape=(1,), dtype='float32', name='rs2_y')
 rv2_y = Input(shape=(1,), dtype='float32', name='rv2_y')
 total_reserv_sum = Input(shape=(1,), dtype='float32', name='total_reserv_sum')
 total_reserv_mean = Input(shape=(1,), dtype='float32', name='total_reserv_mean')
 total_reserv_dt_diff_mean = Input(shape=(1,), dtype='float32', name='total_reserv_dt_diff_mean')
 date_int = Input(shape=(1,), dtype='float32', name='date_int')
 var_max_lat = Input(shape=(1,), dtype='float32', name='var_max_lat')
 var_max_long = Input(shape=(1,), dtype='float32', name='var_max_long')
 lon_plus_lat = Input(shape=(1,), dtype='float32', name='lon_plus_lat')
 date_emb = keras.layers.concatenate([dow_emb, month_emb, year, holiday_flg])
 date_emb = Dense(5, activation='sigmoid', name='date_merged_emb' )(date_emb)
 cat_layer = keras.layers.concatenate([holiday_flg, min_visitors, mean_visitors,
median_visitors,
count_observations, rs1_x, rv1_x,
rs2_x, rv2_x, rs1_y, rv1_y, rs2_y, rv2_y,
total_reserv_sum, total_reserv_mean, total_reserv_dt_diff_mean,
date_int, var_max_lat, var_max_long, lon_plus_lat,
date_emb, air_area_name_emb, air_genre_name_emb,
air_area_code_emb, air_genre_code_emb, air_store_id_emb])
 m = Dense(hidden1_neurons, name='hidden1',
kernel_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.05, seed=None))(cat_layer)
 m = keras.layers.PReLU()(m)
 m = keras.layers.BatchNormalization()(m)
 m1 = Dense(hidden2_neurons, name='sub1' )(m)
 m1 = keras.layers.PReLU()(m1)
 m = Dense(1, activation='relu' )(m1)
 inp_ten = [
holiday_flg, min_visitors, mean_visitors, median_visitors,
count_observations,
rs1_x, rv1_x, rs2_x, rv2_x, rs1_y, rv1_y, rs2_y, rv2_y, total_reserv_sum, total_reserv_mean,
total_reserv_dt_diff_mean, date_int, var_max_lat, var_max_long, lon_plus_lat,
dow, year, month, air_store_id, air_area_code, air_genre_code
]
inp_ten += air_area_name
inp_ten += air_genre_name
model = keras.Model(inp_ten, m)
 model.compile(loss='mse', optimizer='rmsprop', metrics=['acc'])
 return model
for i, weight in enumerate(bert_full_list):
print("{} bert model prediction starts".format(i+1))
model = BertForSequenceClassification(bert_config, num_labels=7)
 model.load_state_dict(torch.load(BERT_FULL_PATH / weight))
model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
test_preds = np.zeros(( len(X_test)))
 for i,(x_batch,)in enumerate(test_loader):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 test_preds[i * 512:(i + 1)* 512] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()
bert_test_preds.append(test_pred)
 del test_pred, model
gc.collect()<prepare_output>
model1 = ensemble.GradientBoostingRegressor(learning_rate=0.2, random_state=3,
n_estimators=200, subsample=0.8, max_depth =10)
 model2 = neighbors.KNeighborsRegressor(n_jobs=-1, n_neighbors=4)
 model3 = XGBRegressor(learning_rate=0.02, random_state=2, n_estimators=1000, subsample=0.8,
colsample_bytree=0.75, max_depth =10, reg_lambda=.25)
 model4 = get_nn_complete_model(train, hidden2_neurons=12)
 model1.fit(train[col], np.log1p(train['visitors'].values))
print("Model1 trained")
 model2.fit(train[col], np.log1p(train['visitors'].values))
print("Model2 trained")
 model3.fit(train[col], np.log1p(train['visitors'].values))
print("Model3 trained")
 for i in range(5):
model4.fit(nn_train[0], nn_train[1], epochs=3, verbose=1,
batch_size=256, shuffle=True, validation_split=0.15)
 model4.fit(nn_train[0], nn_train[1], epochs=8, verbose=0,
batch_size=256, shuffle=True)
 print("Model4 trained")
 preds1 = model1.predict(train[col])
 preds2 = model2.predict(train[col])
 preds3 = model3.predict(train[col])
 preds4 = pd.Series(model4.predict(nn_train[0] ).reshape(-1)).clip(0, 6.8 ).values
print('RMSE GradientBoostingRegressor: ', RMSLE(np.log1p(train['visitors'].values), preds1))
print('RMSE KNeighborsRegressor: ', RMSLE(np.log1p(train['visitors'].values), preds2))
print('RMSE XGBRegressor: ', RMSLE(np.log1p(train['visitors'].values), preds3))
print('RMSE NeuralNetwork: ', RMSLE(np.log1p(train['visitors'].values), preds4))
preds1 = model1.predict(test[col])
 preds2 = model2.predict(test[col])
 preds3 = model3.predict(test[col])
 preds4 = pd.Series(model4.predict(nn_test[0] ).reshape(-1)).clip(0, 6.8 ).values
test['visitors'] = 0.15*preds1+0.15*preds2+0.4*preds3+0.3*preds4
test['visitors'] = np.expm1(test['visitors'] ).clip(lower=0.)
sub1 = test[['id','visitors']].copy()
print("Model predictions done.")
bert_test_preds = np.mean(bert_test_preds, axis=0 )<save_to_csv>
dfs = { re.search('/([^/\.]*)\.csv', fn ).group(1):
pd.read_csv(fn)for fn in glob.glob('.. /input/*.csv')}
for k, v in dfs.items() : locals() [k] = v
wkend_holidays = date_info.apply(
(lambda x:(x.day_of_week=='Sunday' or x.day_of_week=='Saturday')and x.holiday_flg==1), axis=1)
 date_info.loc[wkend_holidays, 'holiday_flg'] = 0
date_info['weight'] =(( date_info.index + 1)/ len(date_info)) ** 5
visit_data = air_visit_data.merge(date_info, left_on='visit_date', right_on='calendar_date', how='left')
 visit_data.drop('calendar_date', axis=1, inplace=True)
 visit_data['visitors'] = visit_data.visitors.map(pd.np.log1p)
 wmean = lambda x:(( x.weight * x.visitors ).sum() / x.weight.sum())
 visitors = visit_data.groupby(['air_store_id', 'day_of_week', 'holiday_flg'] ).apply(wmean ).reset_index()
visitors.rename(columns={0:'visitors'}, inplace=True)
 sample_submission['air_store_id'] = sample_submission.id.map(lambda x: '_'.join(x.split('_')[:-1]))
sample_submission['calendar_date'] = sample_submission.id.map(lambda x: x.split('_')[2])
 sample_submission.drop('visitors', axis=1, inplace=True)
 sample_submission = sample_submission.merge(date_info, on='calendar_date', how='left')
 sample_submission = sample_submission.merge(visitors, on=[
'air_store_id', 'day_of_week', 'holiday_flg'], how='left')
 sample_submission['visitors'] = sample_submission.visitors.map(pd.np.expm1)
 sub2 = sample_submission[['id', 'visitors']].copy()
sub2 = sub2.fillna(-1 )
<set_options><EOS> 
def final_visitors(x, alt=False):
visitors_x, visitors_y = x['visitors_x'], x['visitors_y']
if x['visitors_y'] == -1:
return visitors_x
else:
return 0.7*visitors_x + 0.3*visitors_y* 1.1
sub_merge = pd.merge(sub1, sub2, on='id', how='inner')
 sub_merge['visitors'] = sub_merge.apply(lambda x: final_visitors(x), axis=1)
 print("Done")
 sub_merge[['id', 'visitors']].to_csv('submission.csv', index=False )