Kaggle competition: Recruit Restaurant Visitor Forecasting
tokenizer = text.Tokenizer(num_words = max_features, filters='',lower=False)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))<predict_on_test>
dfs = { re.search('/([^/\.]*)\.csv', fn ).group(1):pd.read_csv(fn)for fn in glob.glob('.. /input/*.csv')}
print('data frames read:{}'.format(list(dfs.keys())))
 print('local variables with the same names are created.')
 for k, v in dfs.items() : locals() [k] = v
print('holidays at weekends are not special, right?')
 wkend_holidays = date_info.apply(( lambda x:(x.day_of_week=='Sunday' or x.day_of_week=='Saturday')and x.holiday_flg==1), axis=1)
 date_info.loc[wkend_holidays, 'holiday_flg'] = 0
print('add decreasing weights from now')
 date_info['weight'] =(( date_info.index + 1)/ len(date_info)) ** 5
def predict_test(model, test_loader):
model.eval()
test_preds = np.zeros(( len(x_test)))
 test_preds_fold = np.zeros(( len(x_test)))
 with torch.no_grad() :
for i, x in enumerate(test_loader):
x[0] = x[0].cuda()
x[1] = x[1].cuda()
x[2] = x[2].cuda()
y_pred = model(x[0], x[1], x[2] ).detach()
test_preds_fold[i * batch_size:(i + 1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
test_preds += test_preds_fold
return test_preds
def fold_load_model(path):
model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 temp_dict = torch.load(path)
 temp_dict['embedding.weight'] = torch.tensor(embedding_matrix)
 model.load_state_dict(temp_dict)
 return model<concatenate>
print('weighted mean visitors for each(air_store_id, day_of_week, holiday_flag)or(air_store_id, day_of_week)')
 visit_data = air_visit_data.merge(date_info, left_on='visit_date', right_on='calendar_date', how='left')
 visit_data.drop('calendar_date', axis=1, inplace=True )
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 print(embedding_matrix.shape)
 del crawl_matrix, unknown_words_crawl
del glove_matrix, unknown_words_glove
gc.collect()<categorify>
visit_data['visit_date'] = pd.to_datetime(visit_data['visit_date'] )
x_test_1 = tokenizer.texts_to_sequences(x_test)
 x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test_1, maxlen=maxlen))
test_lengths = torch.from_numpy(np.array([len(x)for x in x_test_1]))<create_dataframe>
test_data = visit_data[visit_data['visit_date']>='2017-04-01']
batch_size = 512
test_dataset = data.TensorDataset(x_test_padded, test_normal_feats, test_lengths)
 test_collator = SequenceBucketCollator(lambda lengths: lengths.max() , sequence_index=0, length_index=2, normal_feats_idx=1)
 test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_collator )<drop_column>
train_data = visit_data[visit_data['visit_date'] >='2017-03-01']
del test_lengths
del x_test_padded
del test_dataset
del test_collator
gc.collect()<define_variables>
y_train_data = visit_data.loc[visit_data['visit_date']<'2017-04-01','visitors']
y_test_data = visit_data.loc[visit_data['visit_date']>='2017-04-01','visitors']
glove_crawl_seed1 = {
'fold1': Path('.. /input/toxic-glove-crawl/10_1fold.pt5'),
'fold2': Path('.. /input/toxic-glove-crawl/10_2fold.pt5'),
'fold3': Path('.. /input/toxic-glove-crawl/10_3fold.pt5'),
'fold4': Path('.. /input/toxic-glove-crawl/10_4fold.pt5'),
'fold5': Path('.. /input/toxic-glove-crawl/10_5fold.pt5'),
}
glove_crawl_seed2 = {
'fold1': Path('.. /input/toxic-glove-crawl/20_1fold.pt5'),
'fold2': Path('.. /input/toxic-glove-crawl/20_2fold.pt5'),
'fold3': Path('.. /input/toxic-glove-crawl/20_3fold.pt5'),
'fold4': Path('.. /input/toxic-glove-crawl/20_4fold.pt5'),
'fold5': Path('.. /input/toxic-glove-crawl/20_5fold.pt5'),
}
glove_crawl_seed3 = {
'fold1': Path('.. /input/toxic-glove-crawl/30_1fold.pt5'),
'fold2': Path('.. /input/toxic-glove-crawl/30_2fold.pt5'),
'fold3': Path('.. /input/toxic-glove-crawl/30_3fold.pt5'),
'fold4': Path('.. /input/toxic-glove-crawl/30_4fold.pt5'),
'fold5': Path('.. /input/toxic-glove-crawl/30_5fold.pt5'),
}
glove_crawl_seed4 = {
'fold1': Path('.. /input/toxic-glove-crawl/40_1fold.pt5'),
'fold2': Path('.. /input/toxic-glove-crawl/40_2fold.pt5'),
'fold3': Path('.. /input/toxic-glove-crawl/40_3fold.pt5'),
'fold4': Path('.. /input/toxic-glove-crawl/40_4fold.pt5'),
'fold5': Path('.. /input/toxic-glove-crawl/40_5fold.pt5'),
}
glove_crawl_seed5 = {
'fold1': Path('.. /input/toxic-glove-crawl/50_1fold.pt5'),
'fold2': Path('.. /input/toxic-glove-crawl/50_2fold.pt5'),
'fold3': Path('.. /input/toxic-glove-crawl/50_3fold.pt5'),
'fold4': Path('.. /input/toxic-glove-crawl/50_4fold.pt5'),
'fold5': Path('.. /input/toxic-glove-crawl/50_5fold.pt5'),
}
glove_crawl_seed6 = {
'fold1': Path('.. /input/toxic-glove-crawl/60_1fold.pt5'),
'fold2': Path('.. /input/toxic-glove-crawl/60_2fold.pt5'),
'fold3': Path('.. /input/toxic-glove-crawl/60_3fold.pt5'),
'fold4': Path('.. /input/toxic-glove-crawl/60_4fold.pt5'),
'fold5': Path('.. /input/toxic-glove-crawl/60_5fold.pt5'),
}
glove_crawl_seed7 = {
'fold1': Path('.. /input/toxic-glove-crawl/70_1fold.pt5'),
'fold2': Path('.. /input/toxic-glove-crawl/70_2fold.pt5'),
'fold3': Path('.. /input/toxic-glove-crawl/70_3fold.pt5'),
'fold4': Path('.. /input/toxic-glove-crawl/70_4fold.pt5'),
'fold5': Path('.. /input/toxic-glove-crawl/70_5fold.pt5'),
}
glove_crawl_seed8 = {
'fold1': Path('.. /input/toxic-glove-crawl/80_1fold.pt5'),
'fold2': Path('.. /input/toxic-glove-crawl/80_2fold.pt5'),
'fold3': Path('.. /input/toxic-glove-crawl/80_3fold.pt5'),
'fold4': Path('.. /input/toxic-glove-crawl/80_4fold.pt5'),
'fold5': Path('.. /input/toxic-glove-crawl/80_5fold.pt5'),
}
glove_crawl_seed9 = {
'fold1': Path('.. /input/toxic-glove-crawl/90_1fold.pt5'),
'fold2': Path('.. /input/toxic-glove-crawl/90_2fold.pt5'),
'fold3': Path('.. /input/toxic-glove-crawl/90_3fold.pt5'),
'fold4': Path('.. /input/toxic-glove-crawl/90_4fold.pt5'),
'fold5': Path('.. /input/toxic-glove-crawl/90_5fold.pt5'),
}
glove_crawl_seed10 = {
'fold1': Path('.. /input/toxic-glove-crawl/100_1fold.pt5'),
'fold2': Path('.. /input/toxic-glove-crawl/100_2fold.pt5'),
'fold3': Path('.. /input/toxic-glove-crawl/100_3fold.pt5'),
'fold4': Path('.. /input/toxic-glove-crawl/100_4fold.pt5'),
'fold5': Path('.. /input/toxic-glove-crawl/100_5fold.pt5'),
}
seed_list = [glove_crawl_seed1, glove_crawl_seed2, glove_crawl_seed3, glove_crawl_seed4, glove_crawl_seed5,
glove_crawl_seed6, glove_crawl_seed7, glove_crawl_seed8, glove_crawl_seed9, glove_crawl_seed10]<predict_on_test>
train_data['visitors'] = train_data.visitors.map(pd.np.log1p)
 wmean = lambda x:(( x.weight * x.visitors ).sum() / x.weight.sum())
 visitors = train_data.groupby(['air_store_id', 'day_of_week', 'holiday_flg'] ).apply(wmean ).reset_index()
visitors.rename(columns={0:'visitors'}, inplace=True )
%%time
all_test_preds = []
for i, seed in enumerate(seed_list):
print("glove_crawl seed {} prediction starts".format(i+1))
fold_preds = np.zeros(( len(x_test)))
 for fold, path in seed.items() :
model = fold_load_model(path)
 model.cuda()
test_preds = predict_test(model, test_loader)
 fold_preds += test_preds
del test_preds, model
gc.collect()
all_test_preds.append(fold_preds )<define_variables>
test_data['visitors_predict'] = test_data.merge(visitors[visitors.holiday_flg==0], \
on=('air_store_id', 'day_of_week'), how='left')['visitors_y']
submission_predict = np.zeros(( len(x_test)))
 for _ in all_test_preds:
submission_predict += _<create_dataframe>
missings = test_data.visitors_predict.isnull()
test_data.loc[missings, 'visitors_predict'] = test_data[missings].merge(
 visitors[['air_store_id', 'visitors']].groupby('air_store_id' ).\
mean().reset_index() , on='air_store_id', how='left')['visitors_y'].values
submission_chris = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': submission_predict / 10
})
<load_pretrained>
test_data['visitors'] = test_data['visitors'].map(np.log1p )
def load_model(path, lstm_units=128):
model = omer_lstm(embedding_matrix, 6, lstm_units)
 temp_dict = torch.load(path)
 temp_dict['embedding.weight'] = torch.tensor(embedding_matrix)
 model.load_state_dict(temp_dict)
 model.cuda()
model.eval()
return model<predict_on_test>
test_data[test_data['visitors_predict'].isnull() ]
def predict(model):
all_test_preds = []
test_preds = np.zeros(( len(x_test), 7))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
test_preds[i * batch_size:(i+1)* batch_size, :] = sigmoid(model(X ).detach().cpu().numpy())
 all_test_preds.append(test_preds)
 del model
gc.collect()
return(all_test_preds )<categorify>
import sklearn.metrics
def build_matrix_1(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( max_features + 1, 300))
unknown_words = []
for word, i in word_index.items() :
if i <= max_features:
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.lower() ]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.title() ]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words<choose_model_class>
sklearn.metrics.mean_squared_error(
 np.array([2.3]),np.array([3]))
class omer_lstm(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets, lstm_units):
super(omer_lstm, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.DENSE_HIDEN_UNITS = lstm_units * 4
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, lstm_units, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(lstm_units * 2, lstm_units, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(self.DENSE_HIDEN_UNITS, self.DENSE_HIDEN_UNITS)
 self.linear2 = nn.Linear(self.DENSE_HIDEN_UNITS, self.DENSE_HIDEN_UNITS)
 self.linear_out = nn.Linear(self.DENSE_HIDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(self.DENSE_HIDEN_UNITS, num_aux_targets)
 def forward(self, x, lengths=None):
h_embedding = self.embedding(x.long())
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<prepare_x_and_y>
sklearn.metrics.mean_squared_error(\
test_data.loc[~test_data.visitors_predict.isnull() ,'visitors']
,test_data.loc[~test_data.visitors_predict.isnull() ,'visitors_predict'] )
maxlen = 300
batch_size = 128
x_test = tokenizer.texts_to_sequences(x_test)
 test_lengths = torch.from_numpy(np.array([len(x)for x in x_test]))
x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test, maxlen=maxlen))
test_dataset = data.TensorDataset(x_test_padded, test_lengths)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 del test_lengths
del x_test_padded
del test_dataset
gc.collect()<define_variables>
sklearn.metrics.mean_squared_error(\
test_data['visitors']
,test_data['visitors_predict'] )
crawl_matrix, unknown_words_crawl = build_matrix_1(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
glove_matrix, unknown_words_glove = build_matrix_1(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
max_features = max_features or len(tokenizer.word_index)+ 1
print("max features :",max_features)
 embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()<predict_on_test>
print('prepare to merge with date_info and visitors')
 sample_submission['air_store_id'] = sample_submission.id.map(lambda x: '_'.join(x.split('_')[:-1]))
sample_submission['calendar_date'] = sample_submission.id.map(lambda x: x.split('_')[2])
 sample_submission.drop('visitors', axis=1, inplace=True)
 sample_submission = sample_submission.merge(date_info, on='calendar_date', how='left')
 sample_submission = sample_submission.merge(visitors, on=['air_store_id', 'day_of_week', 'holiday_flg'], how='left')
 missings = sample_submission.visitors.isnull()
sample_submission.loc[missings, 'visitors'] = sample_submission[missings].merge(
 visitors[visitors.holiday_flg==0], on=('air_store_id', 'day_of_week'), how='left')['visitors_y'].values
missings = sample_submission.visitors.isnull()
sample_submission.loc[missings, 'visitors'] = sample_submission[missings].merge(
 visitors[['air_store_id', 'visitors']].groupby('air_store_id' ).mean().reset_index() , on='air_store_id', how='left')['visitors_y'].values
sample_submission['visitors'] = sample_submission.visitors.map(pd.np.expm1)
 sample_submission[['id', 'visitors']].to_csv('dumb_result.csv', float_format='%.4f', index=None)
 print("done" )