Kaggle competition: Recruit Restaurant Visitor Forecasting
<SOS> metric: RMSLE Kaggle data source: recruit-restaurant-visitor-forecasting<find_best_params>
%%time
test_lengths = torch.from_numpy(np.array([len(x)for x in test_word_sequences]))
maxlen = test_lengths.max()
print(f"Max len = {maxlen}")
 maxlen = min(maxlen, 400)
 x_test_padded = torch.tensor(pad_sequences(test_word_sequences, maxlen=maxlen)).long()
test_collator = SequenceBucketCollator(torch.max, sequence_index=0, length_index=1)
 del text_list, test_word_sequences, tknzr
gc.collect()<categorify>
data = {
'tra': pd.read_csv('.. /input/air_visit_data.csv'),
'as': pd.read_csv('.. /input/air_store_info.csv'),
'hs': pd.read_csv('.. /input/hpg_store_info.csv'),
'ar': pd.read_csv('.. /input/air_reserve.csv'),
'hr': pd.read_csv('.. /input/hpg_reserve.csv'),
'id': pd.read_csv('.. /input/store_id_relation.csv'),
'tes': pd.read_csv('.. /input/sample_submission.csv'),
'hol': pd.read_csv('.. /input/date_info.csv' ).rename(columns={'calendar_date':'visit_date'})
 }
data['hr'] = pd.merge(data['hr'], data['id'], how='inner', on=['hpg_store_id'])
 for df in ['ar','hr']:
data[df]['visit_datetime'] = pd.to_datetime(data[df]['visit_datetime'])
 data[df]['visit_dow'] = data[df]['visit_datetime'].dt.dayofweek
data[df]['visit_datetime'] = data[df]['visit_datetime'].dt.date
data[df]['reserve_datetime'] = pd.to_datetime(data[df]['reserve_datetime'])
 data[df]['reserve_datetime'] = data[df]['reserve_datetime'].dt.date
data[df]['reserve_datetime_diff'] = data[df].apply(lambda r:(r['visit_datetime'] - r['reserve_datetime'] ).days, axis=1)
 data[df] = data[df][data[df]['reserve_datetime_diff'] > data[df]['visit_dow']]
tmp1 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].sum().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs1', 'reserve_visitors':'rv1'})
 tmp2 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].mean().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs2', 'reserve_visitors':'rv2'})
 data[df] = pd.merge(tmp1, tmp2, how='inner', on=['air_store_id','visit_date'])
 data['tra']['visit_date'] = pd.to_datetime(data['tra']['visit_date'])
 data['tra']['dow'] = data['tra']['visit_date'].dt.dayofweek
data['tra']['doy'] = data['tra']['visit_date'].dt.dayofyear
data['tra']['year'] = data['tra']['visit_date'].dt.year
data['tra']['month'] = data['tra']['visit_date'].dt.month
data['tra']['week'] = data['tra']['visit_date'].dt.week
data['tra']['visit_date'] = data['tra']['visit_date'].dt.date
data['tes']['visit_date'] = data['tes']['id'].map(lambda x: str(x ).split('_')[2])
 data['tes']['air_store_id'] = data['tes']['id'].map(lambda x: '_'.join(x.split('_')[:2]))
data['tes']['visit_date'] = pd.to_datetime(data['tes']['visit_date'])
 data['tes']['dow'] = data['tes']['visit_date'].dt.dayofweek
data['tes']['doy'] = data['tes']['visit_date'].dt.dayofyear
data['tes']['year'] = data['tes']['visit_date'].dt.year
data['tes']['month'] = data['tes']['visit_date'].dt.month
data['tes']['week'] = data['tes']['visit_date'].dt.week
data['tes']['visit_date'] = data['tes']['visit_date'].dt.date
unique_stores = data['tes']['air_store_id'].unique()
stores = pd.concat([pd.DataFrame({'air_store_id': unique_stores, 'dow': [i]*len(unique_stores)})for i in range(7)], axis=0, ignore_index=True ).reset_index(drop=True)
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].min().rename(columns={'visitors':'min_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].mean().rename(columns={'visitors':'mean_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].median().rename(columns={'visitors':'median_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].count().rename(columns={'visitors':'count_observations'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 stores = pd.merge(stores, data['as'], how='left', on=['air_store_id'])
 stores['air_genre_name'] = stores['air_genre_name'].map(lambda x: str(str(x ).replace('/',' ')))
 stores['air_area_name'] = stores['air_area_name'].map(lambda x: str(str(x ).replace('-',' ')))
 lbl = preprocessing.LabelEncoder()
for i in range(10):
stores['air_genre_name'+str(i)] = lbl.fit_transform(stores['air_genre_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_area_name'+str(i)] = lbl.fit_transform(stores['air_area_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_genre_name'] = lbl.fit_transform(stores['air_genre_name'])
 stores['air_area_name'] = lbl.fit_transform(stores['air_area_name'])
 data['hol']['visit_date'] = pd.to_datetime(data['hol']['visit_date'])
 data['hol']['day_of_week'] = lbl.fit_transform(data['hol']['day_of_week'])
 data['hol']['visit_date'] = data['hol']['visit_date'].dt.date
train = pd.merge(data['tra'], data['hol'], how='left', on=['visit_date'])
 test = pd.merge(data['tes'], data['hol'], how='left', on=['visit_date'])
 train = pd.merge(train, stores, how='inner', on=['air_store_id','dow'])
 test = pd.merge(test, stores, how='left', on=['air_store_id','dow'])
 for df in ['ar','hr']:
train = pd.merge(train, data[df], how='left', on=['air_store_id','visit_date'])
 test = pd.merge(test, data[df], how='left', on=['air_store_id','visit_date'])
 train['id'] = train.apply(lambda r: '_'.join([str(r['air_store_id']), str(r['visit_date'])]), axis=1)
 train['total_reserv_sum'] = train['rv1_x'] + train['rv1_y']
train['total_reserv_mean'] =(train['rv2_x'] + train['rv2_y'])/ 2
train['total_reserv_dt_diff_mean'] =(train['rs2_x'] + train['rs2_y'])/ 2
test['total_reserv_sum'] = test['rv1_x'] + test['rv1_y']
test['total_reserv_mean'] =(test['rv2_x'] + test['rv2_y'])/ 2
test['total_reserv_dt_diff_mean'] =(test['rs2_x'] + test['rs2_y'])/ 2
train['date_int'] = train['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 test['date_int'] = test['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 train['var_max_lat'] = train['latitude'].max() - train['latitude']
train['var_max_long'] = train['longitude'].max() - train['longitude']
test['var_max_lat'] = test['latitude'].max() - test['latitude']
test['var_max_long'] = test['longitude'].max() - test['longitude']
train['lon_plus_lat'] = train['longitude'] + train['latitude']
test['lon_plus_lat'] = test['longitude'] + test['latitude']
lbl = preprocessing.LabelEncoder()
train['air_store_id2'] = lbl.fit_transform(train['air_store_id'])
 test['air_store_id2'] = lbl.transform(test['air_store_id'])
 col = [c for c in train if c not in ['id', 'air_store_id', 'visit_date','visitors']]
train = train.fillna(-1)
 test = test.fillna(-1 )
%%time
glove_matrix, _ = gensim_to_embedding_matrix(
 word_dict,
".. /input/gensim-embeddings-dataset/glove.840B.300d.gensim", 
)
 crawl_matrix, _ = gensim_to_embedding_matrix(
 word_dict,
".. /input/gensim-embeddings-dataset/crawl-300d-2M.gensim", 
)
 para_matrix, _ = gensim_to_embedding_matrix(
 word_dict,
".. /input/gensim-embeddings-dataset/paragram_300_sl999.gensim", 
)
 w2v_matrix, _ = gensim_to_embedding_matrix(
 word_dict,
".. /input/gensim-embeddings-dataset/GoogleNews-vectors-negative300.gensim",
)<categorify>
traintest = pd.concat([train, test])
 value_col = ['holiday_flg','min_visitors','mean_visitors','median_visitors',
'count_observations',
'rs1_x','rv1_x','rs2_x','rv2_x','rs1_y','rv1_y','rs2_y','rv2_y','total_reserv_sum','total_reserv_mean',
'total_reserv_dt_diff_mean','date_int','var_max_lat','var_max_long','lon_plus_lat']
cat_col = ['dow', 'year', 'month', 'air_store_id2', 'air_area_name', 'air_genre_name',
'air_area_name0', 'air_area_name1', 'air_area_name2', 'air_area_name3', 'air_area_name4',
'air_area_name5', 'air_area_name6', 'air_genre_name0', 'air_genre_name1',
'air_genre_name2', 'air_genre_name3', 'air_genre_name4']
nn_col = value_col + cat_col
dummys = []
for col in cat_col:
dummy = pd.get_dummies(traintest[col], drop_first=False)
 dummys.append(dummy.as_matrix())
 dummys = np.concatenate(dummys, axis=1)
 value_scaler = preprocessing.MinMaxScaler()
for vcol in value_col:
traintest[vcol] = value_scaler.fit_transform(traintest[vcol].values.astype(np.float64 ).reshape(-1, 1))
X_value = traintest[value_col].as_matrix()
X = np.concatenate([dummys, X_value], axis=1 )
%%time
def one_hot_char_embeddings(word2index, char_vectorizer):
words = [""] *(max(word2index.values())+ 1)
 for word, i in word2index.items() :
words[i] = word
return char_vectorizer.transform(words ).toarray().astype(np.float32)
 char_matrix = one_hot_char_embeddings(
 word_dict,
joblib.load('.. /input/jigsaw-solution-ver-1/char_vectorizer.pkl'),
)<normalization>
def RMSLE(y, pred):
return metrics.mean_squared_error(y, pred)**0.5
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 6 * LSTM_UNITS
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, output_aux_sub=11):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(embedding_matrix.shape[0], embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS + 6 + output_aux_sub, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, 6)
 self.linear_sub_out = nn.Linear(DENSE_HIDDEN_UNITS, output_aux_sub)
 def forward(self, x, lengths=None):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool1 = torch.mean(h_lstm1, 1)
 avg_pool2 = torch.mean(h_lstm2, 1)
 max_pool2, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( avg_pool1, max_pool2, avg_pool2), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
aux_result = self.linear_aux_out(hidden)
 sub_result = self.linear_sub_out(hidden)
 result = self.linear_out(torch.cat(( hidden, aux_result, sub_result), 1))
out = torch.cat([result, aux_result, sub_result], 1)
 return out
def get_lstm_preds(model_name, embedding_matrix):
model = NeuralNet(embedding_matrix)
 temp_dict = torch.load('.. /input/jigsaw-solution-ver-1/' + model_name)
 temp_dict['embedding.weight'] = torch.tensor(embedding_matrix)
 model.load_state_dict(temp_dict)
 model = model.cuda()
for param in model.parameters() :
param.requires_grad=False
model = model.eval()
batch_size = 256
test_dataset = data.TensorDataset(x_test_padded, test_lengths)
 test_loader = data.DataLoader(test_dataset, batch_size=batch_size, collate_fn=test_collator)
 preds = np.zeros(( len(test_dataset), 18), dtype=np.float32)
 with torch.no_grad() :
for i,(x_batch)in enumerate(test_loader):
X_1 = x_batch[0].cuda()
y_pred = torch.sigmoid(model(X_1)).cpu().numpy()
preds[i * batch_size:(i + 1)* batch_size] = y_pred
return preds<define_variables>
def get_DAE() :
inputs = Input(( X.shape[1],))
x = Dense(800, activation='relu',bias=False )(inputs)
 x = BatchNormalization()(x)
 x = Dense(400, activation='relu', name="feature",bias=False )(x)
 x = BatchNormalization()(x)
 x = Dense(800, activation='relu',bias=False )(x)
 x = BatchNormalization()(x)
 outputs = Dense(X.shape[1], activation='relu' )(x)
 model = Model(inputs=inputs, outputs=outputs)
 model.compile(optimizer='adam', loss='mse')
 return model
def x_generator(x, batch_size, shuffle=True):
batch_index = 0
n = x.shape[0]
while True:
if batch_index == 0:
index_array = np.arange(n)
 if shuffle:
index_array = np.random.permutation(n)
 current_index =(batch_index * batch_size)% n
if n >= current_index + batch_size:
current_batch_size = batch_size
batch_index += 1
else:
current_batch_size = n - current_index
batch_index = 0
batch_x = x[index_array[current_index: current_index + current_batch_size]]
yield batch_x
def mix_generator(x, batch_size, swaprate=0.15, shuffle=True):
num_value = X.shape[1]
num_swap = int(num_value * swaprate)
 gen1 = x_generator(x, batch_size, shuffle)
 gen2 = x_generator(x, batch_size, shuffle)
 while True:
batch1 = next(gen1)
 batch2 = next(gen2)
 new_batch = batch1.copy()
for i in range(batch1.shape[0]):
swap_idx = np.random.choice(num_value, num_swap, replace=False)
 new_batch[i, swap_idx] = batch2[i, swap_idx]
yield(new_batch, batch1 )
%%time
lstm_models = ['Notebook_100_5.bin',
'Notebook_100_6.bin',
'Notebook_100_7.bin',
'Notebook_100_8.bin',
'Notebook_100_9.bin',
'Notebook_100_10.bin',
'Notebook_100_11.bin',
'Notebook_100_12.bin']
embedding_matrix = np.concatenate([glove_matrix, crawl_matrix, w2v_matrix, char_matrix], axis=1)
 all_lstm_preds = []
for model_name in lstm_models:
print(model_name)
 preds = get_lstm_preds(model_name, embedding_matrix)
 all_lstm_preds.append(preds)
 gc.collect()<predict_on_test>
batch_size = 128
num_epoch = 50
gen = mix_generator(X, batch_size)
 dae = get_DAE()
dae.fit_generator(generator=gen,
steps_per_epoch=np.ceil(X.shape[0] / batch_size),
epochs=num_epoch,
verbose=3, )
%%time
lstm_models = ['Notebook_100_1.bin']
embedding_matrix = np.concatenate([para_matrix, crawl_matrix, w2v_matrix, char_matrix], axis=1)
 for model_name in lstm_models:
print(model_name)
 preds = get_lstm_preds(model_name, embedding_matrix)
 all_lstm_preds.append(preds)
 lstm_models = ['Notebook_100_2.bin']
embedding_matrix = np.concatenate([glove_matrix, crawl_matrix, w2v_matrix, char_matrix], axis=1)
 for model_name in lstm_models:
print(model_name)
 preds = get_lstm_preds(model_name, embedding_matrix)
 all_lstm_preds.append(preds)
 lstm_models = ['Notebook_100_3.bin']
embedding_matrix = np.concatenate([glove_matrix, para_matrix, w2v_matrix, char_matrix], axis=1)
 for model_name in lstm_models:
print(model_name)
 preds = get_lstm_preds(model_name, embedding_matrix)
 all_lstm_preds.append(preds)
 lstm_models = ['Notebook_100_4.bin']
embedding_matrix = np.concatenate([glove_matrix, para_matrix, crawl_matrix, char_matrix], axis=1)
 for model_name in lstm_models:
print(model_name)
 preds = get_lstm_preds(model_name, embedding_matrix)
 all_lstm_preds.append(preds )<compute_test_metric>
def get_NN(DAE):
l2_loss = l2(0.01)
 DAE.trainable = False
x = dae.get_layer("feature" ).output
x = Dropout(0.1 )(x)
 x = Dense(1500, activation='relu', kernel_regularizer=l2_loss,bias=False )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.4 )(x)
 x = Dense(500, activation='relu', kernel_regularizer=l2_loss,bias=False )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.4 )(x)
 x = Dense(500, activation='relu', kernel_regularizer=l2_loss,bias=False )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.4 )(x)
 predictions = Dense(1, activation='relu', kernel_regularizer=l2_loss,bias=False )(x)
 model = Model(inputs=dae.input, outputs=predictions)
 model.compile(loss='mse',optimizer='adam')
 return model
def train_generator(x, y, batch_size, shuffle=True):
batch_index = 0
n = x.shape[0]
while True:
if batch_index == 0:
index_array = np.arange(n)
 if shuffle:
index_array = np.random.permutation(n)
 current_index =(batch_index * batch_size)% n
if n >= current_index + batch_size:
current_batch_size = batch_size
batch_index += 1
else:
current_batch_size = n - current_index
batch_index = 0
batch_x = x[index_array[current_index: current_index + current_batch_size]]
batch_y = y[index_array[current_index: current_index + current_batch_size]]
yield batch_x, batch_y
def test_generator(x, batch_size, shuffle=False):
batch_index = 0
n = x.shape[0]
while True:
if batch_index == 0:
index_array = np.arange(n)
 if shuffle:
index_array = np.random.permutation(n)
 current_index =(batch_index * batch_size)% n
if n >= current_index + batch_size:
current_batch_size = batch_size
batch_index += 1
else:
current_batch_size = n - current_index
batch_index = 0
batch_x = x[index_array[current_index: current_index + current_batch_size]]
yield batch_x
def get_callbacks(save_path):
save_checkpoint = ModelCheckpoint(filepath=save_path, monitor='val_loss', save_best_only=True)
 early_stopping = EarlyStopping(monitor='val_loss',
patience=4,
verbose=2,
min_delta=1e-4,
mode='min')
 Callbacks = [ save_checkpoint, early_stopping]
return Callbacks
def simple_magic(preds):
return(
 preds[:, 0] + preds[:, 1] * 0.05 - preds[:, -1] * 0.05 - preds[:, 4] * 0.05 
)
 def sophisticated_magic(preds):
return(
 preds[:, 0] + preds[:, 1] * 0.05 - preds[:, -1] * 0.03 - preds[:, 4] * 0.03 
)- preds[:, 14] *(1 - preds[:, 0])* 0.05 - preds[:, 10] *(1 - preds[:, 0])* 0.05<define_variables>
batch_size = 128
num_epoch = 40
num_fold = 3
Y_train = np.log1p(train['visitors'] ).values
X_train = X[:Y_train.shape[0]]
X_test = X[Y_train.shape[0]:]
y_test = np.zeros([num_fold, X_test.shape[0]])
 y_valid = np.zeros([X_train.shape[0]])
 folds = list(KFold(n_splits=num_fold, shuffle=True, random_state=42 ).split(X_train))
for j,(ids_train_split, ids_valid_split)in enumerate(folds):
print("fold", j+1, "==================")
 model = get_NN(dae)
 gen_train = train_generator(X_train[ids_train_split], Y_train[ids_train_split], batch_size)
 gen_val = train_generator(X_train[ids_valid_split], Y_train[ids_valid_split], batch_size, shuffle=False)
 gen_val_pred = test_generator(X_train[ids_valid_split], batch_size, shuffle=False)
 gen_test_pred = test_generator(X_test, batch_size, shuffle=False)
 callbacks = get_callbacks("weight" + str(j)+ ".hdf5")
 model.fit_generator(generator=gen_train,
steps_per_epoch=np.ceil(ids_train_split.shape[0] / batch_size),
epochs=num_epoch,
verbose=3,
validation_data=gen_val,
validation_steps=np.ceil(ids_valid_split.shape[0] / batch_size), 
)
 y_valid[ids_valid_split] = model.predict_generator(generator=gen_val_pred,
steps=np.ceil(ids_valid_split.shape[0] / batch_size)) [:,0]
y_test[j] = model.predict_generator(generator=gen_test_pred,
steps=np.ceil(X_test.shape[0] / batch_size)) [:,0]
score = RMSLE(y_valid, Y_train)
 print("valid score", score)
 y_test_mean = np.mean(y_test, axis=0)
<define_variables><EOS> 
id = pd.read_csv(".. /input/sample_submission.csv")['id']
submission = pd.DataFrame({'id': id, 'visitors': np.expm1(y_test_mean)})
 submission.to_csv('submission_dae.csv', index=False )