Kaggle competition: Recruit Restaurant Visitor Forecasting
warnings.filterwarnings(action='once')
 device = torch.device('cuda' )<string_transform>
train_prepared = pd.read_csv('.. /input/agg-data-2/air_train_agg8000.csv')
 test_prepared = pd.read_csv('.. /input/agg-data-3/air_test_14000.csv' )
def convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"])+[0] *(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens )<load_pretrained>
lbl = preprocessing.LabelEncoder()
train_prepared['air_genre_name'] = lbl.fit_transform(train_prepared['air_genre_name'])
 train_prepared['air_area_name'] = lbl.fit_transform(train_prepared['air_area_name'])
 train_prepared['holiday_flg']= lbl.fit_transform(train_prepared['holiday_flg'])
 train_prepared['day_of_week']= lbl.fit_transform(train_prepared['day_of_week'])
 train_prepared.head()
MAX_SEQUENCE_LENGTH = 220
SEED = 1234
BATCH_SIZE = 32
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
bert_config = BertConfig('.. /input/bert-inference/bert/bert_config.json')
 tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True )<load_from_csv>
train_x = train_prepared.drop(['air_store_id', 'visit_date', 'visitors','total_air_reservations', 'total_hpg_reservations', 'total_reservations'], axis=1 )
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv")
 test_df['comment_text'] = test_df['comment_text'].astype(str)
 X_test = convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), MAX_SEQUENCE_LENGTH, tokenizer )<load_pretrained>
train_y = np.log1p(train_prepared['visitors'].values )
model = BertForSequenceClassification(bert_config, num_labels=1)
 model.load_state_dict(torch.load(".. /input/bertnew/bert_pytorch.bin"))
model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()<categorify>
test_prepared['air_genre_name'] = lbl.fit_transform(test_prepared['air_genre_name'])
 test_prepared['air_area_name'] = lbl.fit_transform(test_prepared['air_area_name'])
 test_prepared['holiday_flg']= lbl.fit_transform(test_prepared['holiday_flg'])
 test_prepared['day_of_week']= lbl.fit_transform(test_prepared['day_of_week'])
 train_prepared.head()
test_preds = np.zeros(( len(X_test)))
 test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)
 tk0 = tqdm(test_loader)
 for i,(x_batch,)in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 test_preds[i * 32:(i + 1)* 32] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()<save_to_csv>
test_prepared['id'] = test_prepared[['air_store_id', 'visit_date']].apply(lambda x: '_'.join(x.astype(str)) , axis=1 )
submission = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': test_pred
})
 submission.to_csv('submission.csv', index=False )<define_variables>
test_x = test_prepared.drop([ 'id','air_store_id', 'visit_date', 'visitors'], axis=1 )
package_dir = ".. /input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT"
sys.path.append(package_dir )<set_options>
train_x['hpg_store_id'] = pd.to_numeric(train_x['hpg_store_id'], errors='coerce')
 train_x['hpg_genre_name'] = pd.to_numeric(train_x['hpg_genre_name'], errors='coerce')
 train_x['hpg_area_name'] = pd.to_numeric(train_x['hpg_genre_name'], errors='coerce')
 test_x['hpg_store_id'] = pd.to_numeric(test_x['hpg_store_id'], errors='coerce')
 test_x['hpg_genre_name'] = pd.to_numeric(test_x['hpg_genre_name'], errors='coerce')
 test_x['hpg_area_name'] = pd.to_numeric(test_x['hpg_area_name'], errors='coerce' )
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<feature_engineering>
train_x['var_max_lat'] = train_x['air_latitude'].max() - train_x['air_latitude']
train_x['var_max_long'] = train_x['air_longitude'].max() - train_x['air_longitude']
test_x['var_max_lat'] = test_x['air_latitude'].max() - test_x['air_latitude']
test_x['var_max_long'] = test_x['air_longitude'].max() - test_x['air_longitude']
train_x['lon_plus_lat'] = train_x['air_longitude'] + train_x['air_latitude']
test_x['lon_plus_lat'] = test_x['air_longitude'] + test_x['air_latitude']
def count_regexp_occ(regexp="", text=None):

 return len(re.findall(regexp, text))
def normal_feature_engineering(text_list):
add_feats_matrix = np.zeros(( len(text_list), 11))
for idx in range(len(text_list)) :
target_text = text_list[idx]
add_feats_matrix[idx, 0] = len(target_text.split(' '))
add_feats_matrix[idx, 1] = len(set(target_text.split(' ')))
 add_feats_matrix[idx, 2] = count_regexp_occ(r'http[s]{0,1}://\S+', target_text)
 add_feats_matrix[idx, 3] = count_regexp_occ(r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+', target_text)
 add_feats_matrix[idx, 4] = count_regexp_occ(r'^[A-Z]+[A-Z]$', target_text)
 symbol_cnt = 0
for symbol in symbols_to_isolate:
if symbol in target_text:
symbol_cnt += 1
add_feats_matrix[idx, 5] = symbol_cnt
symbol_delete_cnt = 0
for symbol_del in symbols_to_delete:
if symbol_del in target_text:
symbol_delete_cnt += 1
add_feats_matrix[idx, 6] = symbol_delete_cnt
add_feats_matrix[idx, 7] = add_feats_matrix[idx, 1] / add_feats_matrix[idx, 0]
add_feats_matrix[idx, 8] = add_feats_matrix[idx, 4] / add_feats_matrix[idx, 0]
add_feats_matrix[idx, 9] = add_feats_matrix[idx, 5] / add_feats_matrix[idx, 0]
add_feats_matrix[idx, 10] = add_feats_matrix[idx, 6] / add_feats_matrix[idx, 0]
print('normal feature enginnering done: {}'.format(add_feats_matrix.shape))
return add_feats_matrix<string_transform>
train_x = train_x.fillna(-1)
 test_x = test_x.fillna(-1 )
def convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"])+[0] *(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens)
 def is_interactive() :
return 'SHLVL' not in os.environ
def seed_everything(seed=123):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path,'rb')as f:
emb_arr = pickle.load(f)
 return emb_arr
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( max_features + 1, 300))
unknown_words = []
for word, i in word_index.items() :
if i <= max_features:
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.lower() ]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.title() ]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.upper() ]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
class SequenceBucketCollator() :
def __init__(self, choose_length, normal_feats_idx, sequence_index, length_index, label_index=None):
self.choose_length = choose_length
self.sequence_index = sequence_index
self.length_index = length_index
self.label_index = label_index
self.normal_feats_idx = normal_feats_idx
def __call__(self, batch):
batch = [torch.stack(x)for x in list(zip(*batch)) ]
sequences = batch[self.sequence_index]
lengths = batch[self.length_index]
normal_feats = batch[self.normal_feats_idx]
length = self.choose_length(lengths)
 mask = torch.arange(start=maxlen, end=0, step=-1)< length
padded_sequences = sequences[:, mask]
batch[self.sequence_index] = padded_sequences
batch[self.normal_feats_idx] = normal_feats
if self.label_index is not None:
return [x for i, x in enumerate(batch)if i != self.label_index], batch[self.label_index]
return batch
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.normal_linear = nn.Linear(ADD_FEATS_UNITS, DENSE_HIDDEN_UNITS)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x, normal_feats, lengths=None):
h_embedding = self.embedding(x.long())
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
normal_linear = F.relu(self.normal_linear(normal_feats.float()))
hidden = h_conc + h_conc_linear1 + h_conc_linear2 + normal_linear
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
def reduce_mem_usage(df):
start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<set_options>
validation = 0.1
mask = np.random.rand(len(train_x)) < validation
X_train = train_x[~mask]
y_train = train_y[~mask]
X_validation = train_x[mask]
y_validation = train_y[mask]
tqdm.pandas()
warnings.filterwarnings(action='once')
 device = torch.device('cuda')
 MAX_SEQUENCE_LENGTH = 220
SEED = 1234
batch_size = 512
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
bert_config = BertConfig('.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'+'bert_config.json')
 tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True)
 if not is_interactive() :
def nop(it, *a, **k):
return it
tqdm = nop
fastprogress.fastprogress.NO_BAR = True
master_bar, progress_bar = force_console_behavior()
fastai.basic_train.master_bar, fastai.basic_train.progress_bar = master_bar, progress_bar
seed_everything()<load_from_csv>
xgb0 = xgb.XGBRegressor()
xgb0.fit(X_train, y_train)
 print('done' )
<prepare_x_and_y><EOS> 
boost_params = {'eval_metric': 'rmse'}
xgb0 = xgb.XGBRegressor(max_depth=10,
learning_rate=0.01,
n_estimators=1000,
objective='reg:linear',
gamma=0,
min_child_weight=1,
subsample=0.8,
colsample_bytree=0.8,
scale_pos_weight=1,
seed=27,
**boost_params)
 xgb0.fit(train_x, train_y)
 predict_y = xgb0.predict(test_x)
 test_prepared['visitors'] = np.expm1(predict_y)
 print('done' )