Kaggle competition: Recruit Restaurant Visitor Forecasting
<SOS> metric: RMSLE Kaggle data source: recruit-restaurant-visitor-forecasting<drop_column>
NWEEKS_TO_EXCLUDE = 2
del crawl_matrix, glove_matrix, para_matrix, w2v_matrix, embedding_matrix
del test_lengths, maxlen, x_test_padded, test_collator
del lstm_models, all_lstm_preds
del word_dict, word_index
gc.collect()<load_pretrained>
UNCASED_BERT_MODEL_PATH = ".. /input/transformer-tokenizers/bert-base-uncased/"
CASED_BERT_MODEL_PATH = ".. /input/transformer-tokenizers/bert-base-cased/"
def clip_to_max_len(batch):
X, lengths = map(torch.stack, zip(*batch))
max_len = torch.max(lengths ).item()
return X[:, :max_len]
def prepare_bert(bin_file, is_cased):
bert_path = CASED_BERT_MODEL_PATH if is_cased else UNCASED_BERT_MODEL_PATH
config = BertConfig(os.path.join(bert_path, "bert_config.json"))
model = BertForSequenceClassification(config, num_labels=18)
 model.load_state_dict(torch.load(bin_file, map_location="cpu"))
for p in model.parameters() :
p.requires_grad = False
model = model.eval()
model = model.cuda()
return model
def prepare_tokenizer(is_cased):
if is_cased:
return BertTokenizer.from_pretrained(CASED_BERT_MODEL_PATH, do_lower_case=False)
 else:
return BertTokenizer.from_pretrained(UNCASED_BERT_MODEL_PATH, do_lower_case=True)
 def apply_bert(model, loader):
preds = np.zeros(( len(loader.dataset), 18), dtype=np.float32)
 for i, X in enumerate(loader):
X = X.cuda()
p = torch.sigmoid(model(X, attention_mask=(X > 0)))
 preds[i * loader.batch_size :(i + 1)* loader.batch_size] = p.cpu().numpy()
return preds<feature_engineering>
data = {
'tra': pd.read_csv('.. /input/air_visit_data.csv'),
'as': pd.read_csv('.. /input/air_store_info.csv'),
'hs': pd.read_csv('.. /input/hpg_store_info.csv'),
'ar': pd.read_csv('.. /input/air_reserve.csv'),
'hr': pd.read_csv('.. /input/hpg_reserve.csv'),
'id': pd.read_csv('.. /input/store_id_relation.csv'),
'tes': pd.read_csv('.. /input/sample_submission.csv'),
'hol': pd.read_csv('.. /input/date_info.csv' ).rename(columns={'calendar_date':'visit_date'})
 }
data['hr'] = pd.merge(data['hr'], data['id'], how='inner', on=['hpg_store_id'])
 for df in ['ar','hr']:
data[df]['visit_datetime'] = pd.to_datetime(data[df]['visit_datetime'])
 data[df]['visit_dow'] = data[df]['visit_datetime'].dt.dayofweek
data[df]['visit_datetime'] = data[df]['visit_datetime'].dt.date
data[df]['reserve_datetime'] = pd.to_datetime(data[df]['reserve_datetime'])
 data[df]['reserve_datetime'] = data[df]['reserve_datetime'].dt.date
data[df]['reserve_datetime_diff'] = data[df].apply(lambda r:(r['visit_datetime'] - r['reserve_datetime'] ).days, axis=1)
 ne = 7*(NWEEKS_TO_EXCLUDE - 1)
 data[df] = data[df][data[df]['reserve_datetime_diff'] > data[df]['visit_dow']+ne]
tmp1 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].sum().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs1', 'reserve_visitors':'rv1'})
 tmp2 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].mean().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs2', 'reserve_visitors':'rv2'})
 data[df] = pd.merge(tmp1, tmp2, how='inner', on=['air_store_id','visit_date'])
 data['tra']['visit_date'] = pd.to_datetime(data['tra']['visit_date'])
 data['tra']['dow'] = data['tra']['visit_date'].dt.dayofweek
data['tra']['year'] = data['tra']['visit_date'].dt.year
data['tra']['month'] = data['tra']['visit_date'].dt.month
data['tra']['visit_date'] = data['tra']['visit_date'].dt.date
data['tes']['visit_date'] = data['tes']['id'].map(lambda x: str(x ).split('_')[2])
 data['tes']['air_store_id'] = data['tes']['id'].map(lambda x: '_'.join(x.split('_')[:2]))
data['tes']['visit_date'] = pd.to_datetime(data['tes']['visit_date'])
 data['tes']['dow'] = data['tes']['visit_date'].dt.dayofweek
data['tes']['year'] = data['tes']['visit_date'].dt.year
data['tes']['month'] = data['tes']['visit_date'].dt.month
data['tes']['visit_date'] = data['tes']['visit_date'].dt.date
unique_stores = data['tes']['air_store_id'].unique()
stores = pd.concat([pd.DataFrame({'air_store_id': unique_stores, 'dow': [i]*len(unique_stores)})for i in range(7)], axis=0, ignore_index=True ).reset_index(drop=True)
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].min().rename(columns={'visitors':'min_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].mean().rename(columns={'visitors':'mean_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].median().rename(columns={'visitors':'median_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].max().rename(columns={'visitors':'max_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].count().rename(columns={'visitors':'count_observations'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 stores = pd.merge(stores, data['as'], how='left', on=['air_store_id'])
 stores['air_genre_name'] = stores['air_genre_name'].map(lambda x: str(str(x ).replace('/',' ')))
 stores['air_area_name'] = stores['air_area_name'].map(lambda x: str(str(x ).replace('-',' ')))
 lbl = preprocessing.LabelEncoder()
for i in range(10):
stores['air_genre_name'+str(i)] = lbl.fit_transform(stores['air_genre_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_area_name'+str(i)] = lbl.fit_transform(stores['air_area_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_genre_name'] = lbl.fit_transform(stores['air_genre_name'])
 stores['air_area_name'] = lbl.fit_transform(stores['air_area_name'])
 data['hol']['visit_date'] = pd.to_datetime(data['hol']['visit_date'])
 data['hol']['day_of_week'] = lbl.fit_transform(data['hol']['day_of_week'])
 data['hol']['visit_date'] = data['hol']['visit_date'].dt.date
train = pd.merge(data['tra'], data['hol'], how='left', on=['visit_date'])
 test = pd.merge(data['tes'], data['hol'], how='left', on=['visit_date'])
 train = pd.merge(train, stores, how='inner', on=['air_store_id','dow'])
 test = pd.merge(test, stores, how='left', on=['air_store_id','dow'])
 for df in ['ar','hr']:
train = pd.merge(train, data[df], how='left', on=['air_store_id','visit_date'])
 test = pd.merge(test, data[df], how='left', on=['air_store_id','visit_date'])
 train['id'] = train.apply(lambda r: '_'.join([str(r['air_store_id']), str(r['visit_date'])]), axis=1)
 train['total_reserv_sum'] = train['rv1_x'] + train['rv1_y']
train['total_reserv_mean'] =(train['rv2_x'] + train['rv2_y'])/ 2
train['total_reserv_dt_diff_mean'] =(train['rs2_x'] + train['rs2_y'])/ 2
test['total_reserv_sum'] = test['rv1_x'] + test['rv1_y']
test['total_reserv_mean'] =(test['rv2_x'] + test['rv2_y'])/ 2
test['total_reserv_dt_diff_mean'] =(test['rs2_x'] + test['rs2_y'])/ 2
train['date_int'] = train['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 test['date_int'] = test['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 train['var_max_lat'] = train['latitude'].max() - train['latitude']
train['var_max_long'] = train['longitude'].max() - train['longitude']
test['var_max_lat'] = test['latitude'].max() - test['latitude']
test['var_max_long'] = test['longitude'].max() - test['longitude']
train['lon_plus_lat'] = train['longitude'] + train['latitude']
test['lon_plus_lat'] = test['longitude'] + test['latitude']
lbl = preprocessing.LabelEncoder()
train['air_store_id2'] = lbl.fit_transform(train['air_store_id'])
 test['air_store_id2'] = lbl.transform(test['air_store_id'])
 col = [c for c in train if c not in ['id', 'air_store_id', 'visit_date','visitors']]
train = train.fillna(-1)
 test = test.fillna(-1 )
%%time
tokenizer = prepare_tokenizer(is_cased=False)
 MAX_LEN = 400 - 2
def convert_line(text):
tokens_a = tokenizer.tokenize(text)[:MAX_LEN]
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"] + tokens_a + ["[SEP]"])
 one_token += [0] *(MAX_LEN - len(tokens_a))
return one_token
with multiprocessing.Pool(processes=2)as pool:
sequences = pool.map(convert_line, test.comment_text)
 sequences = np.array(sequences)
 lengths = np.argmax(sequences == 0, axis=1)
 lengths[lengths == 0] = sequences.shape[1]
ids = lengths.argsort(kind="stable")
 inverse_ids = test.id.values[ids].argsort(kind="stable")
 sequences = torch.from_numpy(sequences)
 lengths = torch.from_numpy(lengths)
 test_dataset = data.TensorDataset(sequences, lengths)
 test_loader = data.DataLoader(data.Subset(test_dataset, ids), batch_size=128, collate_fn=clip_to_max_len )<categorify>
def RMSLE(y, pred):
return metrics.mean_squared_error(y, pred)**0.5
%%time
UNCASED_BERT_MODELS = [
"BERT_exp_BERT_3_decay_5_epoch_1",
"final-pipe1-raw_1",
"final-pipe1-raw_2",
"final-pipe1-raw_3",
"final-pipe2-raw_1",
"final-pipe2-raw_2",
"final-pipe2-raw_3",
"final-pipe4-wiki_raw_1",
"final-pipe4-wiki_raw_2",
"final-pipe4-wiki_raw_3",
]
for path in UNCASED_BERT_MODELS:
model = prepare_bert(f".. /input/toxic-models-zoo/{path}.bin", is_cased=False)
 preds = apply_bert(model, test_loader)
 STORE[path] = sophisticated_magic(preds)[inverse_ids]
for path in ["final-pipe2-raw_4", "BERT_exp_BERT_3_epoch_1"]:
model = prepare_bert(f".. /input/jigsaw-solution-ver-1/{path}.bin", is_cased=False)
 preds = apply_bert(model, test_loader)
 STORE[path] = sophisticated_magic(preds)[inverse_ids]<feature_engineering>
value_col = ['holiday_flg','min_visitors','mean_visitors','median_visitors','max_visitors','count_observations',
'rs1_x','rv1_x','rs2_x','rv2_x','rs1_y','rv1_y','rs2_y','rv2_y','total_reserv_sum','total_reserv_mean',
'total_reserv_dt_diff_mean','date_int','var_max_lat','var_max_long','lon_plus_lat']
nn_col = value_col + ['dow', 'year', 'month', 'air_store_id2', 'air_area_name', 'air_genre_name',
'air_area_name0', 'air_area_name1', 'air_area_name2', 'air_area_name3', 'air_area_name4',
'air_area_name5', 'air_area_name6', 'air_genre_name0', 'air_genre_name1',
'air_genre_name2', 'air_genre_name3', 'air_genre_name4']
X = train.copy()
X_test = test[nn_col].copy()
value_scaler = preprocessing.MinMaxScaler()
for vcol in value_col:
X[vcol] = value_scaler.fit_transform(X[vcol].values.astype(np.float64 ).reshape(-1, 1))
X_test[vcol] = value_scaler.transform(X_test[vcol].values.astype(np.float64 ).reshape(-1, 1))
X_train = list(X[nn_col].T.as_matrix())
 Y_train = np.log1p(X['visitors'] ).values
nn_train = [X_train, Y_train]
nn_test = [list(X_test[nn_col].T.as_matrix())]
print("Train and test data prepared" )
%%time
tokenizer = prepare_tokenizer(is_cased=True)
 def convert_line(text):
tokens_a = tokenizer.tokenize(text)[:MAX_LEN]
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"] + tokens_a + ["[SEP]"])
 one_token += [0] *(MAX_LEN - len(tokens_a))
return one_token
with multiprocessing.Pool(processes=2)as pool:
sequences = pool.map(convert_line, test.comment_text)
 sequences = np.array(sequences)
 lengths = np.argmax(sequences == 0, axis=1)
 lengths[lengths == 0] = sequences.shape[1]
ids = lengths.argsort(kind="stable")
 inverse_ids = test.id.values[ids].argsort(kind="stable")
 sequences = torch.from_numpy(sequences)
 lengths = torch.from_numpy(lengths)
 test_dataset = data.TensorDataset(sequences, lengths)
 test_loader = data.DataLoader(data.Subset(test_dataset, ids), batch_size=128, collate_fn=clip_to_max_len )<load_pretrained>
def get_nn_complete_model(train, hidden1_neurons=35, hidden2_neurons=15):

 K.clear_session()
air_store_id = Input(shape=(1,), dtype='int32', name='air_store_id')
 air_store_id_emb = Embedding(len(train['air_store_id2'].unique())+ 1, 15, input_shape=(1,),
name='air_store_id_emb' )(air_store_id)
 air_store_id_emb = keras.layers.Flatten(name='air_store_id_emb_flatten' )(air_store_id_emb)
 dow = Input(shape=(1,), dtype='int32', name='dow')
 dow_emb = Embedding(8, 3, input_shape=(1,), name='dow_emb' )(dow)
 dow_emb = keras.layers.Flatten(name='dow_emb_flatten' )(dow_emb)
 month = Input(shape=(1,), dtype='int32', name='month')
 month_emb = Embedding(13, 3, input_shape=(1,), name='month_emb' )(month)
 month_emb = keras.layers.Flatten(name='month_emb_flatten' )(month_emb)
 air_area_name, air_genre_name = [], []
air_area_name_emb, air_genre_name_emb = [], []
for i in range(7):
area_name_col = 'air_area_name' + str(i)
 air_area_name.append(Input(shape=(1,), dtype='int32', name=area_name_col))
tmp = Embedding(len(train[area_name_col].unique()), 3, input_shape=(1,),
name=area_name_col + '_emb' )(air_area_name[-1])
 tmp = keras.layers.Flatten(name=area_name_col + '_emb_flatten' )(tmp)
 air_area_name_emb.append(tmp)
 if i > 4:
continue
area_genre_col = 'air_genre_name' + str(i)
 air_genre_name.append(Input(shape=(1,), dtype='int32', name=area_genre_col))
tmp = Embedding(len(train[area_genre_col].unique()), 3, input_shape=(1,),
name=area_genre_col + '_emb' )(air_genre_name[-1])
 tmp = keras.layers.Flatten(name=area_genre_col + '_emb_flatten' )(tmp)
 air_genre_name_emb.append(tmp)
 air_genre_name_emb = keras.layers.concatenate(air_genre_name_emb)
 air_genre_name_emb = Dense(4, activation='sigmoid', name='final_air_genre_emb' )(air_genre_name_emb)
 air_area_name_emb = keras.layers.concatenate(air_area_name_emb)
 air_area_name_emb = Dense(4, activation='sigmoid', name='final_air_area_emb' )(air_area_name_emb)
 air_area_code = Input(shape=(1,), dtype='int32', name='air_area_code')
 air_area_code_emb = Embedding(len(train['air_area_name'].unique()), 8, input_shape=(1,), name='air_area_code_emb' )(air_area_code)
 air_area_code_emb = keras.layers.Flatten(name='air_area_code_emb_flatten' )(air_area_code_emb)
 air_genre_code = Input(shape=(1,), dtype='int32', name='air_genre_code')
 air_genre_code_emb = Embedding(len(train['air_genre_name'].unique()), 5, input_shape=(1,),
name='air_genre_code_emb' )(air_genre_code)
 air_genre_code_emb = keras.layers.Flatten(name='air_genre_code_emb_flatten' )(air_genre_code_emb)
 holiday_flg = Input(shape=(1,), dtype='float32', name='holiday_flg')
 year = Input(shape=(1,), dtype='float32', name='year')
 min_visitors = Input(shape=(1,), dtype='float32', name='min_visitors')
 mean_visitors = Input(shape=(1,), dtype='float32', name='mean_visitors')
 median_visitors = Input(shape=(1,), dtype='float32', name='median_visitors')
 max_visitors = Input(shape=(1,), dtype='float32', name='max_visitors')
 count_observations = Input(shape=(1,), dtype='float32', name='count_observations')
 rs1_x = Input(shape=(1,), dtype='float32', name='rs1_x')
 rv1_x = Input(shape=(1,), dtype='float32', name='rv1_x')
 rs2_x = Input(shape=(1,), dtype='float32', name='rs2_x')
 rv2_x = Input(shape=(1,), dtype='float32', name='rv2_x')
 rs1_y = Input(shape=(1,), dtype='float32', name='rs1_y')
 rv1_y = Input(shape=(1,), dtype='float32', name='rv1_y')
 rs2_y = Input(shape=(1,), dtype='float32', name='rs2_y')
 rv2_y = Input(shape=(1,), dtype='float32', name='rv2_y')
 total_reserv_sum = Input(shape=(1,), dtype='float32', name='total_reserv_sum')
 total_reserv_mean = Input(shape=(1,), dtype='float32', name='total_reserv_mean')
 total_reserv_dt_diff_mean = Input(shape=(1,), dtype='float32', name='total_reserv_dt_diff_mean')
 date_int = Input(shape=(1,), dtype='float32', name='date_int')
 var_max_lat = Input(shape=(1,), dtype='float32', name='var_max_lat')
 var_max_long = Input(shape=(1,), dtype='float32', name='var_max_long')
 lon_plus_lat = Input(shape=(1,), dtype='float32', name='lon_plus_lat')
 date_emb = keras.layers.concatenate([dow_emb, month_emb, year, holiday_flg])
 date_emb = Dense(5, activation='sigmoid', name='date_merged_emb' )(date_emb)
 cat_layer = keras.layers.concatenate([holiday_flg, min_visitors, mean_visitors,
median_visitors, max_visitors, count_observations, rs1_x, rv1_x,
rs2_x, rv2_x, rs1_y, rv1_y, rs2_y, rv2_y,
total_reserv_sum, total_reserv_mean, total_reserv_dt_diff_mean,
date_int, var_max_lat, var_max_long, lon_plus_lat,
date_emb, air_area_name_emb, air_genre_name_emb,
air_area_code_emb, air_genre_code_emb, air_store_id_emb])
 m = Dense(hidden1_neurons, name='hidden1',
kernel_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.05, seed=None))(cat_layer)
 m = keras.layers.PReLU()(m)
 m = keras.layers.BatchNormalization()(m)
 m1 = Dense(hidden2_neurons, name='sub1' )(m)
 m1 = keras.layers.PReLU()(m1)
 m = Dense(1, activation='relu' )(m1)
 inp_ten = [
holiday_flg, min_visitors, mean_visitors, median_visitors, max_visitors, count_observations,
rs1_x, rv1_x, rs2_x, rv2_x, rs1_y, rv1_y, rs2_y, rv2_y, total_reserv_sum, total_reserv_mean,
total_reserv_dt_diff_mean, date_int, var_max_lat, var_max_long, lon_plus_lat,
dow, year, month, air_store_id, air_area_code, air_genre_code
]
inp_ten += air_area_name
inp_ten += air_genre_name
model = keras.Model(inp_ten, m)
 model.compile(loss='mse', optimizer='rmsprop', metrics=['acc'])
 return model
%%time
CASED_BERT_MODELS = [
"BERT_exp_BERT_3_cased_decay_epoch_1",
"final-pipe2-cased_1",
"final-pipe2-cased_2",
"final-pipe2-cased_3",
"final-pipe5-wiki_cased_1",
"final-pipe5-wiki_cased_2",
"final-pipe5-wiki_cased_3",
]
for path in CASED_BERT_MODELS:
model = prepare_bert(f".. /input/toxic-models-zoo/{path}.bin", is_cased=True)
 preds = apply_bert(model, test_loader)
 STORE[path] = sophisticated_magic(preds)[inverse_ids]
for path in ["final-pipe2-cased_10"]:
model = prepare_bert(f".. /input/jigsaw-solution-ver-1/{path}.bin", is_cased=True)
 preds = apply_bert(model, test_loader)
 STORE[path] = sophisticated_magic(preds)[inverse_ids]<categorify>
model1 = ensemble.GradientBoostingRegressor(learning_rate=0.2, random_state=3,
n_estimators=200, subsample=0.8, max_depth =10)
 model2 = neighbors.KNeighborsRegressor(n_jobs=-1, n_neighbors=4)
 model3 = XGBRegressor(learning_rate=0.02, random_state=3, n_estimators=800, subsample=0.8,
colsample_bytree=0.8, max_depth =10)
 model4 = get_nn_complete_model(train, hidden2_neurons=12)
 model1.fit(train[col], np.log1p(train['visitors'].values))
print("Model1 trained")
 model2.fit(train[col], np.log1p(train['visitors'].values))
print("Model2 trained")
 model3.fit(train[col], np.log1p(train['visitors'].values))
print("Model3 trained")
 for i in range(5):
model4.fit(nn_train[0], nn_train[1], epochs=3, verbose=1,
batch_size=256, shuffle=True, validation_split=0.15)
 model4.fit(nn_train[0], nn_train[1], epochs=8, verbose=0,
batch_size=256, shuffle=True)
 print("Model4 trained")
 preds1 = model1.predict(train[col])
 preds2 = model2.predict(train[col])
 preds3 = model3.predict(train[col])
 preds4 = pd.Series(model4.predict(nn_train[0] ).reshape(-1)).clip(0, 6.8 ).values
print('RMSE GradientBoostingRegressor: ', RMSLE(np.log1p(train['visitors'].values), preds1))
print('RMSE KNeighborsRegressor: ', RMSLE(np.log1p(train['visitors'].values), preds2))
print('RMSE XGBRegressor: ', RMSLE(np.log1p(train['visitors'].values), preds3))
print('RMSE NeuralNetwork: ', RMSLE(np.log1p(train['visitors'].values), preds4))
preds1 = model1.predict(test[col])
 preds2 = model2.predict(test[col])
 preds3 = model3.predict(test[col])
 preds4 = pd.Series(model4.predict(nn_test[0] ).reshape(-1)).clip(0, 6.8 ).values
test['visitors'] = 0.15*preds1+0.15*preds2+0.4*preds3+0.3*preds4
test['visitors'] = np.expm1(test['visitors'] ).clip(lower=0.)
sub1 = test[['id','visitors']].copy()
print("Model predictions done.")
@lru_cache()
def bytes_to_unicode() :
bs = list(range(ord("!"), ord("~")+1)) +list(range(ord("¡"), ord("¬")+1)) +list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
 cs.append(2**8+n)
 n += 1
cs = [chr(n)for n in cs]
return dict(zip(bs, cs))
def get_pairs(word):
pairs = set()
prev_char = word[0]
for char in word[1:]:
pairs.add(( prev_char, char))
prev_char = char
return pairs
class MonkeyPatchedGPT2Tokenizer(object):
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path, cache_dir=None, *inputs, **kwargs):
vocab_file = os.path.join(pretrained_model_name_or_path, "vocab.json")
 merges_file = os.path.join(pretrained_model_name_or_path, "merges.txt")
 max_len = 1024
kwargs['max_len'] = min(kwargs.get('max_len', int(1e12)) , max_len)
 special_tokens = kwargs.pop('special_tokens', [])
 tokenizer = cls(vocab_file, merges_file, special_tokens=special_tokens, *inputs, **kwargs)
 return tokenizer
def __init__(self, vocab_file, merges_file, errors='replace', special_tokens=None, max_len=None):
self.max_len = max_len if max_len is not None else int(1e12)
 self.encoder = json.load(open(vocab_file))
self.decoder = {v:k for k,v in self.encoder.items() }
self.errors = errors
self.byte_encoder = bytes_to_unicode()
self.byte_decoder = {v:k for k, v in self.byte_encoder.items() }
bpe_data = open(merges_file, encoding='utf-8' ).read().split('
')[1:-1]
bpe_merges = [tuple(merge.split())for merge in bpe_data]
self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
self.cache = {}
self.pat = re.compile(r)
 self.special_tokens = {}
self.special_tokens_decoder = {}
self.set_special_tokens(special_tokens)
 def __len__(self):
return len(self.encoder)+ len(self.special_tokens)
 def set_special_tokens(self, special_tokens):
if not special_tokens:
self.special_tokens = {}
self.special_tokens_decoder = {}
return
self.special_tokens = dict(( tok, len(self.encoder)+ i)for i, tok in enumerate(special_tokens))
self.special_tokens_decoder = {v:k for k, v in self.special_tokens.items() }
def bpe(self, token):
if token in self.cache:
return self.cache[token]
word = tuple(token)
 pairs = get_pairs(word)
 if not pairs:
return token
while True:
bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
 if bigram not in self.bpe_ranks:
break
first, second = bigram
new_word = []
i = 0
while i < len(word):
try:
j = word.index(first, i)
 new_word.extend(word[i:j])
 i = j
except:
new_word.extend(word[i:])
 break
if word[i] == first and i < len(word)-1 and word[i+1] == second:
new_word.append(first+second)
 i += 2
else:
new_word.append(word[i])
 i += 1
new_word = tuple(new_word)
 word = new_word
if len(word)== 1:
break
else:
pairs = get_pairs(word)
 word = ' '.join(word)
 self.cache[token] = word
return word
def tokenize(self, text):
bpe_tokens = []
for token in re.findall(self.pat, text):
token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token ).split(' '))
return bpe_tokens
def convert_tokens_to_ids(self, tokens):
ids = []
if isinstance(tokens, str):
if tokens in self.special_tokens:
return self.special_tokens[tokens]
else:
return self.encoder.get(tokens, 0)
 for token in tokens:
if token in self.special_tokens:
ids.append(self.special_tokens[token])
 else:
ids.append(self.encoder.get(token, 0))
if len(ids)> self.max_len:
print(
 "Token indices sequence length is longer than the specified maximum "
" sequence length for this OpenAI GPT model({} > {} ).Running this"
" sequence through the model will result in indexing errors".format(len(ids), self.max_len)
)
 return ids
def convert_ids_to_tokens(self, ids, skip_special_tokens=False):

 tokens = []
for i in ids:
if i in self.special_tokens_decoder:
if not skip_special_tokens:
tokens.append(self.special_tokens_decoder[i])
 else:
tokens.append(self.decoder[i])
 return tokens
def encode(self, text):
return self.convert_tokens_to_ids(self.tokenize(text))
def decode(self, tokens, skip_special_tokens=False, clean_up_tokenization_spaces=True):
text = ''.join(self.convert_ids_to_tokens(tokens, skip_special_tokens=skip_special_tokens))
text = bytearray([self.byte_decoder[c] for c in text] ).decode('utf-8', errors=self.errors)
 if clean_up_tokenization_spaces:
text = text.replace('<unk>', '')
 text = text.replace('.', '.' ).replace(' ?', '?' ).replace(' !', '!' ).replace(' ,', ','
).replace(" ' ", "'" ).replace(" n't", "n't" ).replace(" 'm", "'m" ).replace(" do not", " don't"
).replace(" 's", "'s" ).replace(" 've", "'ve" ).replace(" 're", "'re")
 return text 
<categorify>
dfs = { re.search('/([^/\.]*)\.csv', fn ).group(1):
pd.read_csv(fn)for fn in glob.glob('.. /input/*.csv')}
for k, v in dfs.items() : locals() [k] = v
wkend_holidays = date_info.apply(
(lambda x:(x.day_of_week=='Sunday' or x.day_of_week=='Saturday')and x.holiday_flg==1), axis=1)
 date_info.loc[wkend_holidays, 'holiday_flg'] = 0
date_info['weight'] =(( date_info.index + 1)/ len(date_info)) ** 5
visit_data = air_visit_data.merge(date_info, left_on='visit_date', right_on='calendar_date', how='left')
 visit_data.drop('calendar_date', axis=1, inplace=True)
 visit_data['visitors'] = visit_data.visitors.map(pd.np.log1p)
 wmean = lambda x:(( x.weight * x.visitors ).sum() / x.weight.sum())
 visitors = visit_data.groupby(['air_store_id', 'day_of_week', 'holiday_flg'] ).apply(wmean ).reset_index()
visitors.rename(columns={0:'visitors'}, inplace=True)
 sample_submission['air_store_id'] = sample_submission.id.map(lambda x: '_'.join(x.split('_')[:-1]))
sample_submission['calendar_date'] = sample_submission.id.map(lambda x: x.split('_')[2])
 sample_submission.drop('visitors', axis=1, inplace=True)
 sample_submission = sample_submission.merge(date_info, on='calendar_date', how='left')
 sample_submission = sample_submission.merge(visitors, on=[
'air_store_id', 'day_of_week', 'holiday_flg'], how='left')
 sample_submission['visitors'] = sample_submission.visitors.map(pd.np.expm1)
 sub2 = sample_submission[['id', 'visitors']].copy()
sub2 = sub2.fillna(-1 )
MAX_LEN = 250
GPT2_TOKENIZER = MonkeyPatchedGPT2Tokenizer.from_pretrained(".. /input/transformer-tokenizers/gpt2/")
 def convert_line_gpt2(text):
tokens_a = GPT2_TOKENIZER.tokenize(text)[:MAX_LEN]
one_token = GPT2_TOKENIZER.convert_tokens_to_ids(tokens_a)
 one_token += [0] *(MAX_LEN - len(tokens_a))
return one_token
def apply_gpt(model, loader):
preds = np.zeros(( len(loader.dataset), 18), dtype=np.float32)
 for i, X in enumerate(loader):
p = torch.sigmoid(model(X.cuda()))
preds[i * loader.batch_size :(i + 1)* loader.batch_size] = p.cpu().numpy()
return preds
class GPT2CNN(GPT2PreTrainedModel):
def __init__(self, config, num_labels):
super().__init__(config)
 self.transformer = GPT2Model(config)
 self.cnn1 = nn.Conv1d(768, 256, kernel_size=3, padding=1)
 self.cnn2 = nn.Conv1d(256, num_labels, kernel_size=3, padding=1)
 self.apply(self.init_weights)
 def forward(self, input_ids, position_ids=None, token_type_ids=None, lm_labels=None, past=None):
x, _ = self.transformer(input_ids, position_ids, token_type_ids, past)
 x = x.permute(0, 2, 1)
 x = F.relu(self.cnn1(x))
x = self.cnn2(x)
 output, _ = torch.max(x, 2)
 return output<create_dataframe>
def final_visitors(x, alt=False):
visitors_x, visitors_y = x['visitors_x'], x['visitors_y']
if x['visitors_y'] == -1:
return visitors_x
else:
return 0.7*visitors_x + 0.3*visitors_y* 1.1
sub_merge = pd.merge(sub1, sub2, on='id', how='inner')
 sub_merge['visitors'] = sub_merge.apply(lambda x: final_visitors(x), axis=1)
 print("Done")
 sub_merge[['id', 'visitors']].to_csv('submission.csv', index=False )