Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<categorify>
%matplotlib inline
%%time
stage1_test_bert_emb = embed_by_bert(stage1_test_df )<prepare_x_and_y>
df_train = pd.read_csv('.. /input/train.csv', index_col='id')
 df_test = pd.read_csv('.. /input/test.csv', index_col='id' )
X_test, y_test = featurize(stage1_test_bert_emb )<predict_on_test>
from geopy.distance import geodesic
from haversine import haversine
logit_test_pred = logit.predict_proba(X_test)
 log_loss(y_test, logit_test_pred )<save_to_csv>
def calculate_distance(coords1, coords2):
return haversine(coords1, coords2 )
submission = pd.read_csv(".. /input/sample_submission_stage_1.csv", index_col = "ID")
 submission["A"] = logit_test_pred[:, 0]
submission["B"] = logit_test_pred[:, 1]
submission["NEITHER"] = logit_test_pred[:, 2]
submission.to_csv("submission.csv" )<load_from_url>
periods = {
'morning': {
'min': 6,
'max': 12
},
'afternoon': {
'min': 12,
'max': 18
},
'evening': {
'min': 18,
'max': 24
},
'night': {
'min': 0,
'max': 6
},
}
rush_hours = [ 
(7, 9), 
(16, 18)
 ]
def get_time_period(hour):
for p in periods:
if hour >= periods[p]['min'] and hour < periods[p]['max']:
return p
def is_during_rush(hour):
for h in rush_hours:
if hour >= h[0] and hour < h[1]:
return True
return False
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-development.tsv -q
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-test.tsv -q
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-validation.tsv -q<install_modules>
def extract_date_features(df, col):

 df[col + '_year'] = df[col].dt.year
df[col + '_month'] = df[col].dt.month
df[col + '_week'] = df[col].dt.week
df[col + '_day'] = df[col].dt.day
df[col + '_dow'] = df[col].dt.dayofweek
df[col + '_hour'] = df[col].dt.hour
df[col + '_minute'] = df[col].dt.minute
df[col + '_weekday'] = df[col].dt.weekday
df[col + '_days_in_month'] = df[col].dt.days_in_month
df[col + '_is_month_start'] = df[col].dt.is_month_start
df[col + '_is_month_end'] = df[col].dt.is_month_end
df[col + '_period'] = df[col].dt.hour.apply(get_time_period)
 df['is_during_rush_hour'] = df[col].dt.hour.apply(is_during_rush)
 return df
df_train['pickup_datetime_dt'] = df_train.pickup_datetime.apply(lambda row: datetime.strptime(row, '%Y-%m-%d %H:%M:%S'))
df_test['pickup_datetime_dt'] = df_test.pickup_datetime.apply(lambda row: datetime.strptime(row, '%Y-%m-%d %H:%M:%S'))
df_train = extract_date_features(df_train, 'pickup_datetime_dt')
 df_test = extract_date_features(df_test, 'pickup_datetime_dt' )
!pip install pytorch-pretrained-bert
!pip install https://github.com/ceshine/pytorch_helper_bot/archive/0.0.4.zip<feature_engineering>
os.environ["SEED"] = "420" 
<define_variables>
train = df_train
test = df_test
BERT_MODEL = 'bert-large-uncased'
CASED = False<string_transform>
train.store_and_fwd_flag = [1 if flag == 'N' else 2 for flag in train.store_and_fwd_flag]
def insert_tag(row):

 to_be_inserted = sorted([ 
(row["A-offset"], " [A] "), 
(row["B-offset"], " [B] "), 
(row["Pronoun-offset"], " [P] ")
 ], key=lambda x: x[0], reverse=True)
 text = row["Text"]
for offset, tag in to_be_inserted:
text = text[:offset] + tag + text[offset:]
return text
def tokenize(text, tokenizer):

 entries = {}
final_tokens = []
for token in tokenizer.tokenize(text):
if token in("[A]", "[B]", "[P]"):
entries[token] = len(final_tokens)
 continue
final_tokens.append(token)
 return final_tokens,(entries["[A]"], entries["[B]"], entries["[P]"])
 class GAPDataset(Dataset):

 def __init__(self, df, tokenizer, labeled=True):
self.labeled = labeled
if labeled:
tmp = df[["A-coref", "B-coref"]].copy()
tmp["Neither"] = ~(df["A-coref"] | df["B-coref"])
 self.y = tmp.values.astype("bool")
 self.offsets, self.tokens = [], []
for _, row in df.iterrows() :
text = insert_tag(row)
 tokens, offsets = tokenize(text, tokenizer)
 self.offsets.append(offsets)
 self.tokens.append(tokenizer.convert_tokens_to_ids(
 ["[CLS]"] + tokens + ["[SEP]"]))
def __len__(self):
return len(self.tokens)
 def __getitem__(self, idx):
if self.labeled:
return self.tokens[idx], self.offsets[idx], self.y[idx]
return self.tokens[idx], self.offsets[idx], None
def collate_examples(batch, truncate_len=500):

 transposed = list(zip(*batch))
max_len = min(
 max(( len(x)for x in transposed[0])) ,
truncate_len 
)
 tokens = np.zeros(( len(batch), max_len), dtype=np.int64)
 for i, row in enumerate(transposed[0]):
row = np.array(row[:truncate_len])
 tokens[i, :len(row)] = row
token_tensor = torch.from_numpy(tokens)
 offsets = torch.stack([
torch.LongTensor(x)for x in transposed[1]
], dim=0)+ 1
if len(transposed)== 2:
return token_tensor, offsets, None
one_hot_labels = torch.stack([
torch.from_numpy(x.astype("uint8")) for x in transposed[2]
], dim=0)
 _, labels = one_hot_labels.max(dim=1)
 return token_tensor, offsets, labels
class Head(nn.Module):

 def __init__(self, bert_hidden_size: int, head_hidden_size:int = 3^6, num_blocks:int = 5):
super().__init__()
self.bert_hidden_size = bert_hidden_size
fca =([
nn.Linear(self.bert_hidden_size * 3, head_hidden_size),
nn.BatchNorm1d(head_hidden_size),
nn.ReLU() ,
])
 for block in range(num_blocks):
fca.append(nn.Linear(head_hidden_size, head_hidden_size))
fca.append(nn.BatchNorm1d(head_hidden_size))
fca.append(nn.ReLU())
 fca.append(nn.Linear(head_hidden_size, 3))
self.fc = nn.Sequential(*fca)
 for i, module in enumerate(self.fc):
if isinstance(module,(nn.BatchNorm1d, nn.BatchNorm2d)) :
nn.init.constant_(module.weight, 1)
 nn.init.constant_(module.bias, 0)
 print("Initing batchnorm")
 elif isinstance(module, nn.Linear):
if getattr(module, "weight_v", None)is not None:
nn.init.uniform_(module.weight_g, 0, 1)
 nn.init.kaiming_normal_(module.weight_v)
 print("Initing linear with weight normalization")
 assert model[i].weight_g is not None
else:
nn.init.kaiming_normal_(module.weight)
 print("Initing linear")
 nn.init.constant_(module.bias, 0)
 def forward(self, bert_outputs, offsets):
assert bert_outputs.size(2)== self.bert_hidden_size
extracted_outputs = bert_outputs.gather(
 1, offsets.unsqueeze(2 ).expand(-1, -1, bert_outputs.size(2))
).view(bert_outputs.size(0), -1)
 return self.fc(extracted_outputs)
 class GAPModel(nn.Module):

 def __init__(self, bert_model: str, device: torch.device):
super().__init__()
self.device = device
if bert_model in("bert-base-uncased", "bert-base-cased"):
self.bert_hidden_size = 768
elif bert_model in("bert-large-uncased", "bert-large-cased"):
self.bert_hidden_size = 1024
else:
raise ValueError("Unsupported BERT model.")
 self.bert = BertModel.from_pretrained(bert_model ).to(device)
 self.head = Head(self.bert_hidden_size ).to(device)
 def forward(self, token_tensor, offsets):
token_tensor = token_tensor.to(self.device)
 bert_outputs, _ = self.bert(
 token_tensor, attention_mask=(token_tensor > 0 ).long() ,
token_type_ids=None, output_all_encoded_layers=False)
 head_outputs = self.head(bert_outputs, offsets.to(self.device))
return head_outputs
def children(m):
return m if isinstance(m,(list, tuple)) else list(m.children())
 def set_trainable_attr(m, b):
m.trainable = b
for p in m.parameters() :
p.requires_grad = b
def apply_leaf(m, f):
c = children(m)
 if isinstance(m, nn.Module):
f(m)
 if len(c)> 0:
for l in c:
apply_leaf(l, f)
 def set_trainable(l, b):
apply_leaf(l, lambda m: set_trainable_attr(m, b))
class GAPBot(BaseBot):
def __init__(self, model, train_loader, val_loader, *, optimizer, clip_grad=0,
avg_window=100, log_dir="./cache/logs/", log_level=logging.INFO,
checkpoint_dir="./cache/model_cache/", batch_idx=0, echo=False,
device="cuda:0", use_tensorboard=False):
super().__init__(
 model, train_loader, val_loader,
optimizer=optimizer, clip_grad=clip_grad,
log_dir=log_dir, checkpoint_dir=checkpoint_dir,
batch_idx=batch_idx, echo=echo,
device=device, use_tensorboard=use_tensorboard 
)
 self.criterion = torch.nn.CrossEntropyLoss()
self.loss_format = "%.6f"
def extract_prediction(self, tensor):
return tensor
def snapshot(self):

 loss = self.eval(self.val_loader)
 loss_str = self.loss_format % loss
self.logger.info("Snapshot loss %s", loss_str)
 self.logger.tb_scalars(
 "losses", {"val": loss}, self.step)
 target_path =(
 self.checkpoint_dir / "best.pth")
 if not self.best_performers or(self.best_performers[0][0] > loss):
torch.save(self.model.state_dict() , target_path)
 self.best_performers = [(loss, target_path, self.step)]
self.logger.info("Saving checkpoint %s...", target_path)
 else:
new_loss_str = self.loss_format % self.best_performers[0][0]
self.logger.info("This performance:%s is not as a good as our previously saved:%s", loss_str,new_loss_str)
 assert Path(target_path ).exists()
return loss<load_from_csv>
test.store_and_fwd_flag = [1 if flag == 'N' else 2 for flag in test.store_and_fwd_flag]
df_train = pd.read_csv("gap-test.tsv", delimiter="\t")
 df_val = pd.read_csv("gap-validation.tsv", delimiter="\t")
 df_test = pd.read_csv("gap-development.tsv", delimiter="\t")
 sample_sub = pd.read_csv(".. /input/sample_submission_stage_1.csv")
 assert sample_sub.shape[0] == df_test.shape[0]<feature_engineering>
def to_category(keys):
for key in keys:
train[key] = train[key].astype('category')
 test[key] = test[key].astype('category')
 CATEGORIES = [
'vendor_id',
'store_and_fwd_flag',
'passenger_count'
]
to_category(CATEGORIES )
tokenizer = BertTokenizer.from_pretrained(
 BERT_MODEL,
do_lower_case=CASED,
never_split =("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]", "[A]", "[B]", "[P]")
)
 tokenizer.vocab["[A]"] = -1
tokenizer.vocab["[B]"] = -1
tokenizer.vocab["[P]"] = -1<create_dataframe>
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
train_ds = GAPDataset(df_train, tokenizer)
 val_ds = GAPDataset(df_val, tokenizer)
 test_ds = GAPDataset(df_test, tokenizer)
 train_loader = DataLoader(
 train_ds,
collate_fn = collate_examples,
batch_size=20,
num_workers=2,
pin_memory=True,
shuffle=True,
drop_last=True 
)
 val_loader = DataLoader(
 val_ds,
collate_fn = collate_examples,
batch_size=128,
num_workers=2,
pin_memory=True,
shuffle=False 
)
 test_loader = DataLoader(
 test_ds,
collate_fn = collate_examples,
batch_size=128,
num_workers=2,
pin_memory=True,
shuffle=False
)<choose_model_class>
rf = RandomForestRegressor(n_estimators=10)
 COLUMNS = [
'pickup_datetime_dt_month',
'pickup_datetime_dt_day',
'pickup_datetime_dt_dow',
'pickup_datetime_dt_hour',
'pickup_longitude',
'pickup_latitude',
'dropoff_longitude',
'dropoff_latitude',
'distance',
'passenger_count',
'vendor_id'
]
X_train = train[COLUMNS]
y_train = train.trip_duration
X_test = test[COLUMNS]
model = GAPModel(BERT_MODEL, torch.device("cuda:0"))
set_trainable(model.bert, False)
 set_trainable(model.head, True )<choose_model_class>
from math import log, exp
lr=1e-4
weight_decay=5e-5
optimizer = torch.optim.Adam(model.parameters() , lr=lr, weight_decay=weight_decay)
 bot = GAPBot(
 model, train_loader, val_loader,
optimizer=optimizer, echo=True,
avg_window=25
)<train_model>
rf_X_train = X_train
rf_y_train = y_train
rf_y_train = [log(y)for y in rf_y_train]
rf.fit(rf_X_train, rf_y_train)
 y_hat = rf.predict(X_test )
steps_per_epoch = len(train_loader)
 n_steps = steps_per_epoch * 27
bot.train(
 n_steps,
log_interval=steps_per_epoch // 4,
snapshot_interval=steps_per_epoch,
scheduler=TriangularLR(
 optimizer, 20, ratio=3, steps_per_cycle=n_steps)
 )<load_pretrained>
y_hat = [exp(y)for y in y_hat]
bot.load_model(bot.best_performers[0][1] )<predict_on_test>
np.mean(y_hat )
preds = bot.predict(test_loader )<import_modules>
sample = pd.read_csv('.. /input/sample_submission.csv' )
<feature_engineering><EOS> 
sample.trip_duration = y_hat
sample.to_csv('results.csv', index=False )