Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<predict_on_test>
%matplotlib inline
snb.set({'figure.figsize':(16,8), 'axes.titlesize':30, 'axes.labelsize':20})
y_subm['Target'] = clf_final.predict(test)+ 1<save_to_csv>
TRAINFILEPATH = os.path.join('.. ', 'input', 'train.csv' )
now = datetime.now()
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm.to_csv(sub_file, index=False )<load_from_url>
data = pd.read_csv(TRAINFILEPATH, index_col=0 )
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-development.tsv -q
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-test.tsv -q
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-validation.tsv -q<install_modules>
data.isna().sum()
!pip install pytorch-pretrained-bert
!pip install https://github.com/ceshine/pytorch_helper_bot/archive/0.0.4.zip<feature_engineering>
data['pickup_datetime'] = pd.to_datetime(data['pickup_datetime'])
 data['dropoff_datetime'] = pd.to_datetime(data['dropoff_datetime'] )
os.environ["SEED"] = "2524" 
<define_variables>
data = data[data["trip_duration"]<21600]
BERT_MODEL = 'bert-large-uncased'
CASED = False<string_transform>
print("{} trips have been removed.It means {:0.2f}% of the total trips in the dataset.".format(1458644-1456583,(1458644-1456583)*100/1458644))
def insert_tag(row):

 to_be_inserted = sorted([ 
(row["A-offset"], " [A] "), 
(row["B-offset"], " [B] "), 
(row["Pronoun-offset"], " [P] ")
 ], key=lambda x: x[0], reverse=True)
 text = row["Text"]
for offset, tag in to_be_inserted:
text = text[:offset] + tag + text[offset:]
return text
def tokenize(text, tokenizer):

 entries = {}
final_tokens = []
for token in tokenizer.tokenize(text):
if token in("[A]", "[B]", "[P]"):
entries[token] = len(final_tokens)
 continue
final_tokens.append(token)
 return final_tokens,(entries["[A]"], entries["[B]"], entries["[P]"])
 class GAPDataset(Dataset):

 def __init__(self, df, tokenizer, labeled=True):
self.labeled = labeled
if labeled:
tmp = df[["A-coref", "B-coref"]].copy()
tmp["Neither"] = ~(df["A-coref"] | df["B-coref"])
 self.y = tmp.values.astype("bool")
 self.offsets, self.tokens = [], []
for _, row in df.iterrows() :
text = insert_tag(row)
 tokens, offsets = tokenize(text, tokenizer)
 self.offsets.append(offsets)
 self.tokens.append(tokenizer.convert_tokens_to_ids(
 ["[CLS]"] + tokens + ["[SEP]"]))
def __len__(self):
return len(self.tokens)
 def __getitem__(self, idx):
if self.labeled:
return self.tokens[idx], self.offsets[idx], self.y[idx]
return self.tokens[idx], self.offsets[idx]
def collate_examples(batch, truncate_len=500):

 transposed = list(zip(*batch))
max_len = min(
 max(( len(x)for x in transposed[0])) ,
truncate_len 
)
 tokens = np.zeros(( len(batch), max_len), dtype=np.int64)
 for i, row in enumerate(transposed[0]):
row = np.array(row[:truncate_len])
 tokens[i, :len(row)] = row
token_tensor = torch.from_numpy(tokens)
 offsets = torch.stack([
torch.LongTensor(x)for x in transposed[1]
], dim=0)+ 1
if len(transposed)== 2:
return token_tensor, offsets, None
one_hot_labels = torch.stack([
torch.from_numpy(x.astype("uint8")) for x in transposed[2]
], dim=0)
 _, labels = one_hot_labels.max(dim=1)
 return token_tensor, offsets, labels
def collate_examples_no_labels(batch, truncate_len=500):

 transposed = list(zip(*batch))
print("transposed size is", transposed.size())
 max_len = min(
 max(( len(x)for x in transposed[0])) ,
truncate_len 
)
 tokens = np.zeros(( len(batch), max_len), dtype=np.int64)
 for i, row in enumerate(transposed[0]):
row = np.array(row[:truncate_len])
 tokens[i, :len(row)] = row
token_tensor = torch.from_numpy(tokens)
 offsets = torch.stack([
torch.LongTensor(x)for x in transposed[1]
], dim=0)+ 1
if len(transposed)== 2:
return token_tensor, offsets, None
one_hot_labels = torch.stack([
torch.from_numpy(x.astype("uint8")) for x in transposed[2]
], dim=0)
 _, labels = one_hot_labels.max(dim=1)
 return token_tensor, offsets, labels
class Head(nn.Module):

 def __init__(self, bert_hidden_size: int):
super().__init__()
self.head_hidden_size = 1024
self.bert_hidden_size = bert_hidden_size
self.fc = nn.Sequential(
 nn.BatchNorm1d(bert_hidden_size * 3),
nn.Dropout(0.2),
nn.Linear(bert_hidden_size * 3, self.head_hidden_size),
nn.ReLU() ,
nn.BatchNorm1d(self.head_hidden_size),
nn.Dropout(0.2),
nn.Linear(self.head_hidden_size, self.head_hidden_size),
nn.ReLU() ,
nn.BatchNorm1d(self.head_hidden_size),
nn.Dropout(0.3),
nn.Linear(self.head_hidden_size, self.head_hidden_size),
nn.ReLU() ,
nn.BatchNorm1d(self.head_hidden_size),
nn.Dropout(0.5),
nn.Linear(self.head_hidden_size, 3)
)
 for i, module in enumerate(self.fc):
if isinstance(module,(nn.BatchNorm1d, nn.BatchNorm2d)) :
nn.init.constant_(module.weight, 1)
 nn.init.constant_(module.bias, 0)
 print("Initing batchnorm")
 elif isinstance(module, nn.Linear):
if getattr(module, "weight_v", None)is not None:
nn.init.uniform_(module.weight_g, 0, 1)
 nn.init.kaiming_normal_(module.weight_v)
 print("Initing linear with weight normalization")
 assert model[i].weight_g is not None
else:
nn.init.kaiming_normal_(module.weight)
 print("Initing linear")
 nn.init.constant_(module.bias, 0)
 def forward(self, bert_outputs, offsets):
assert bert_outputs.size(2)== self.bert_hidden_size
extracted_outputs = bert_outputs.gather(
 1, offsets.unsqueeze(2 ).expand(-1, -1, bert_outputs.size(2))
).view(bert_outputs.size(0), -1)
 return self.fc(extracted_outputs)
 class GAPModel(nn.Module):

 def __init__(self, bert_model: str, device: torch.device):
super().__init__()
self.device = device
if bert_model in("bert-base-uncased", "bert-base-cased"):
self.bert_hidden_size = 768
elif bert_model in("bert-large-uncased", "bert-large-cased"):
self.bert_hidden_size = 1024
else:
raise ValueError("Unsupported BERT model.")
 self.bert = BertModel.from_pretrained(bert_model ).to(device)
 self.head = Head(self.bert_hidden_size ).to(device)
 def forward(self, token_tensor, offsets):
token_tensor = token_tensor.to(self.device)
 bert_outputs, _ = self.bert(
 token_tensor, attention_mask=(token_tensor > 0 ).long() ,
token_type_ids=None, output_all_encoded_layers=False)
 head_outputs = self.head(bert_outputs, offsets.to(self.device))
return head_outputs
def children(m):
return m if isinstance(m,(list, tuple)) else list(m.children())
 def set_trainable_attr(m, b):
m.trainable = b
for p in m.parameters() :
p.requires_grad = b
def apply_leaf(m, f):
c = children(m)
 if isinstance(m, nn.Module):
f(m)
 if len(c)> 0:
for l in c:
apply_leaf(l, f)
 def set_trainable(l, b):
apply_leaf(l, lambda m: set_trainable_attr(m, b))
class GAPBot(BaseBot):
def __init__(self, model, train_loader, val_loader, *, optimizer, clip_grad=0,
avg_window=100, log_dir="./cache/logs/", log_level=logging.INFO,
checkpoint_dir="./cache/model_cache/", batch_idx=0, echo=False,
device="cuda:0", use_tensorboard=False):
super().__init__(
 model, train_loader, val_loader,
optimizer=optimizer, clip_grad=clip_grad,
log_dir=log_dir, checkpoint_dir=checkpoint_dir,
batch_idx=batch_idx, echo=echo,
device=device, use_tensorboard=use_tensorboard 
)
 self.criterion = torch.nn.CrossEntropyLoss()
self.loss_format = "%.6f"
def extract_prediction(self, tensor):
return tensor
def snapshot(self):

 loss = self.eval(self.val_loader)
 loss_str = self.loss_format % loss
self.logger.info("Snapshot loss %s", loss_str)
 self.logger.tb_scalars(
 "losses", {"val": loss}, self.step)
 target_path =(
 self.checkpoint_dir / "best.pth")
 if not self.best_performers or(self.best_performers[0][0] > loss):
torch.save(self.model.state_dict() , target_path)
 self.best_performers = [(loss, target_path, self.step)]
self.logger.info("Saving checkpoint %s...", target_path)
 else:
new_loss_str = self.loss_format % self.best_performers[0][0]
self.logger.info("This performance:%s is not as a good as our previously saved:%s", loss_str,new_loss_str)
 assert Path(target_path ).exists()
return loss<load_from_csv>
train = data
df_train = pd.read_csv("gap-test.tsv", delimiter="\t")
 df_val = pd.read_csv("gap-validation.tsv", delimiter="\t")
 df_test = pd.read_csv("/kaggle/input/gendered-pronoun-resolution/test_stage_2.tsv", delimiter="\t")
 sample_sub = pd.read_csv("/kaggle/input/gendered-pronoun-resolution/sample_submission_stage_2.csv")
 assert sample_sub.shape[0] == df_test.shape[0]<feature_engineering>
train['month_pickup']=data['pickup_datetime'].dt.month
train['day_pickup']=data['pickup_datetime'].dt.dayofweek
train['hour_pickup']=data['pickup_datetime'].dt.hour
train['minute_pickup']=data['pickup_datetime'].dt.minute
train['second_pickup']=data['pickup_datetime'].dt.second
train = train.drop(columns=['pickup_datetime'] )
tokenizer = BertTokenizer.from_pretrained(
 BERT_MODEL,
do_lower_case=CASED,
never_split =("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]", "[A]", "[B]", "[P]")
)
 tokenizer.vocab["[A]"] = -1
tokenizer.vocab["[B]"] = -1
tokenizer.vocab["[P]"] = -1<create_dataframe>
def cosrad(n):
"Return the cosine of ``n`` degrees in radians."
return math.cos(math.radians(n))
def distance(row):

 lat1 = row['pickup_latitude']
long1 = row['pickup_longitude']
lat2 = row['dropoff_latitude']
long2 = row['dropoff_longitude']
earth_radius = 6371
dLat = math.radians(lat2 - lat1)
 dLong = math.radians(long2 - long1)
 a =(math.sin(dLat / 2)** 2 +
cosrad(lat1)* cosrad(lat2)* math.sin(dLong / 2)** 2)
 c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
d = earth_radius * c
return d
train_ds = GAPDataset(df_train, tokenizer)
 val_ds = GAPDataset(df_val, tokenizer)
 test_ds = GAPDataset(df_test, tokenizer, labeled=False)
 train_loader = DataLoader(
 train_ds,
collate_fn = collate_examples,
batch_size=20,
num_workers=2,
pin_memory=True,
shuffle=True,
drop_last=True 
)
 val_loader = DataLoader(
 val_ds,
collate_fn = collate_examples,
batch_size=128,
num_workers=2,
pin_memory=True,
shuffle=False 
)
 test_loader = DataLoader(
 test_ds,
collate_fn = collate_examples,
batch_size=128,
num_workers=2,
pin_memory=True,
shuffle=False 
)
<choose_model_class>
train['trip_distance']=data.apply(distance, axis=1 )
model = GAPModel(BERT_MODEL, torch.device("cuda:0"))
set_trainable(model.bert, False)
 set_trainable(model.head, True )<choose_model_class>
train['trip_duration_log']=data['trip_duration'].apply(np.log )
lr=1e-4
weight_decay=5e-3
optimizer = torch.optim.Adam(model.parameters() , lr=lr, weight_decay=weight_decay)
 bot = GAPBot(
 model, train_loader, val_loader,
optimizer=optimizer, echo=True,
avg_window=25
)<train_model>
features = ['trip_distance', 'day_pickup', 'hour_pickup', 'minute_pickup', "pickup_longitude", "dropoff_longitude", "pickup_latitude", "dropoff_latitude"]
target = 'trip_duration_log'
steps_per_epoch = len(train_loader)
 n_steps = steps_per_epoch * 27
bot.train(
 n_steps,
log_interval=steps_per_epoch // 4,
snapshot_interval=steps_per_epoch,
scheduler=TriangularLR(
 optimizer, 20, ratio=2, steps_per_cycle=n_steps)
 )<load_pretrained>
X = train[features]
y = train[target]
X.shape, y.shape
bot.load_model(bot.best_performers[0][1] )<predict_on_test>
from sklearn.metrics import make_scorer
from sklearn.model_selection import cross_val_score
def predict(loader, *, return_y=False):
model.eval()
outputs, y_global = [], []
with torch.set_grad_enabled(False):
for input_tensors in loader:
input_tensors = [x.to(model.device)for x in input_tensors if x is not None]
outputs.append(bot.predict_batch(input_tensors ).cpu())
 outputs = torch.cat(outputs, dim=0)
 return outputs<predict_on_test>
def rmsle_func(ypred, ytest):
assert len(ytest)== len(ypred)
 return np.sqrt(np.mean(( np.log1p(ypred)- np.log1p(ytest)) **2))
preds = predict(test_loader )<import_modules>
rmsle = make_scorer(rmsle_func )
from torch.autograd import Variable 
<feature_engineering>
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import SGDRegressor
from sklearn.linear_model import LinearRegression
def exx(row):
if(row.A > row.B and row.A > row.NEITHER):
row.A = 1
row.B = 0
row.NEITHER = 0
elif(row.B > row.A and row.B > row.NEITHER):
row.A = 0
row.B = 1
row.NEITHER = 0
else:
row.A = 0
row.B = 0
row.NEITHER = 1
return row<save_to_csv>
rfr = RandomForestRegressor(n_estimators=15)
 reg = LinearRegression()
sgdr = SGDRegressor()
df_sub = pd.DataFrame(torch.softmax(preds, -1 ).cpu().numpy().clip(1e-3, 1-1e-3), columns=["A", "B", "NEITHER"])
 df_sub["ID"] = df_test.ID
df_sub.to_csv("submission.csv", index=False)
 df_sub.head()<load_from_url>
scores_rfr = cross_val_score(rfr, X, y, cv=5, scoring=rmsle )
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-development.tsv -q
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-test.tsv -q
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-validation.tsv -q<install_modules>
print("RMSLE: %0.2f(+/- %0.4f)" %(scores_rfr.mean() , scores_rfr.std() * 2))
!pip install pytorch-pretrained-bert
!pip install https://github.com/ceshine/pytorch_helper_bot/archive/0.0.4.zip<feature_engineering>
scores_reg = cross_val_score(reg, X, y, cv=5, scoring=rmsle)
 print("RMSLE: %0.2f(+/- %0.4f)" %(scores_reg.mean() , scores_reg.std() * 2))
os.environ["SEED"] = "420" 
<define_variables>
scores_sgdr= cross_val_score(sgdr, X, y, cv=5, scoring=rmsle )
BERT_MODEL = 'bert-large-uncased'
CASED = False<string_transform>
print("RMSLE: %0.2f(+/- %0.4f)" %(scores_sgdr.mean() , scores_sgdr.std() * 2))
def insert_tag(row):

 to_be_inserted = sorted([ 
(row["A-offset"], " [A] "), 
(row["B-offset"], " [B] "), 
(row["Pronoun-offset"], " [P] ")
 ], key=lambda x: x[0], reverse=True)
 text = row["Text"]
for offset, tag in to_be_inserted:
text = text[:offset] + tag + text[offset:]
return text
def tokenize(text, tokenizer):

 entries = {}
final_tokens = []
for token in tokenizer.tokenize(text):
if token in("[A]", "[B]", "[P]"):
entries[token] = len(final_tokens)
 continue
final_tokens.append(token)
 return final_tokens,(entries["[A]"], entries["[B]"], entries["[P]"])
 class GAPDataset(Dataset):

 def __init__(self, df, tokenizer, labeled=True):
self.labeled = labeled
if labeled:
tmp = df[["A-coref", "B-coref"]].copy()
tmp["Neither"] = ~(df["A-coref"] | df["B-coref"])
 self.y = tmp.values.astype("bool")
 self.offsets, self.tokens = [], []
for _, row in df.iterrows() :
text = insert_tag(row)
 tokens, offsets = tokenize(text, tokenizer)
 self.offsets.append(offsets)
 self.tokens.append(tokenizer.convert_tokens_to_ids(
 ["[CLS]"] + tokens + ["[SEP]"]))
def __len__(self):
return len(self.tokens)
 def __getitem__(self, idx):
if self.labeled:
return self.tokens[idx], self.offsets[idx], self.y[idx]
return self.tokens[idx], self.offsets[idx]
def collate_examples(batch, truncate_len=500):

 transposed = list(zip(*batch))
max_len = min(
 max(( len(x)for x in transposed[0])) ,
truncate_len 
)
 tokens = np.zeros(( len(batch), max_len), dtype=np.int64)
 for i, row in enumerate(transposed[0]):
row = np.array(row[:truncate_len])
 tokens[i, :len(row)] = row
token_tensor = torch.from_numpy(tokens)
 offsets = torch.stack([
torch.LongTensor(x)for x in transposed[1]
], dim=0)+ 1
if len(transposed)== 2:
return token_tensor, offsets, None
one_hot_labels = torch.stack([
torch.from_numpy(x.astype("uint8")) for x in transposed[2]
], dim=0)
 _, labels = one_hot_labels.max(dim=1)
 return token_tensor, offsets, labels
def collate_examples_no_labels(batch, truncate_len=500):

 transposed = list(zip(*batch))
print("transposed size is", transposed.size())
 max_len = min(
 max(( len(x)for x in transposed[0])) ,
truncate_len 
)
 tokens = np.zeros(( len(batch), max_len), dtype=np.int64)
 for i, row in enumerate(transposed[0]):
row = np.array(row[:truncate_len])
 tokens[i, :len(row)] = row
token_tensor = torch.from_numpy(tokens)
 offsets = torch.stack([
torch.LongTensor(x)for x in transposed[1]
], dim=0)+ 1
if len(transposed)== 2:
return token_tensor, offsets, None
one_hot_labels = torch.stack([
torch.from_numpy(x.astype("uint8")) for x in transposed[2]
], dim=0)
 _, labels = one_hot_labels.max(dim=1)
 return token_tensor, offsets, labels
class Head(nn.Module):

 def __init__(self, bert_hidden_size: int):
super().__init__()
self.head_hidden_size = 1024
self.bert_hidden_size = bert_hidden_size
self.fc = nn.Sequential(
 nn.BatchNorm1d(bert_hidden_size * 3),
nn.Dropout(0.5),
nn.Linear(bert_hidden_size * 3, self.head_hidden_size),
nn.ReLU() ,
nn.BatchNorm1d(self.head_hidden_size),
nn.Dropout(0.5),
nn.Linear(self.head_hidden_size, self.head_hidden_size),
nn.ReLU() ,
nn.BatchNorm1d(self.head_hidden_size),
nn.Dropout(0.5),
nn.Linear(self.head_hidden_size, self.head_hidden_size),
nn.ReLU() ,
nn.BatchNorm1d(self.head_hidden_size),
nn.Dropout(0.5),
nn.Linear(self.head_hidden_size, 3)
)
 for i, module in enumerate(self.fc):
if isinstance(module,(nn.BatchNorm1d, nn.BatchNorm2d)) :
nn.init.constant_(module.weight, 1)
 nn.init.constant_(module.bias, 0)
 print("Initing batchnorm")
 elif isinstance(module, nn.Linear):
if getattr(module, "weight_v", None)is not None:
nn.init.uniform_(module.weight_g, 0, 1)
 nn.init.kaiming_normal_(module.weight_v)
 print("Initing linear with weight normalization")
 assert model[i].weight_g is not None
else:
nn.init.kaiming_normal_(module.weight)
 print("Initing linear")
 nn.init.constant_(module.bias, 0)
 def forward(self, bert_outputs, offsets):
assert bert_outputs.size(2)== self.bert_hidden_size
extracted_outputs = bert_outputs.gather(
 1, offsets.unsqueeze(2 ).expand(-1, -1, bert_outputs.size(2))
).view(bert_outputs.size(0), -1)
 return self.fc(extracted_outputs)
 class GAPModel(nn.Module):

 def __init__(self, bert_model: str, device: torch.device):
super().__init__()
self.device = device
if bert_model in("bert-base-uncased", "bert-base-cased"):
self.bert_hidden_size = 768
elif bert_model in("bert-large-uncased", "bert-large-cased"):
self.bert_hidden_size = 1024
else:
raise ValueError("Unsupported BERT model.")
 self.bert = BertModel.from_pretrained(bert_model ).to(device)
 self.head = Head(self.bert_hidden_size ).to(device)
 def forward(self, token_tensor, offsets):
token_tensor = token_tensor.to(self.device)
 bert_outputs, _ = self.bert(
 token_tensor, attention_mask=(token_tensor > 0 ).long() ,
token_type_ids=None, output_all_encoded_layers=False)
 head_outputs = self.head(bert_outputs, offsets.to(self.device))
return head_outputs
def children(m):
return m if isinstance(m,(list, tuple)) else list(m.children())
 def set_trainable_attr(m, b):
m.trainable = b
for p in m.parameters() :
p.requires_grad = b
def apply_leaf(m, f):
c = children(m)
 if isinstance(m, nn.Module):
f(m)
 if len(c)> 0:
for l in c:
apply_leaf(l, f)
 def set_trainable(l, b):
apply_leaf(l, lambda m: set_trainable_attr(m, b))
class GAPBot(BaseBot):
def __init__(self, model, train_loader, val_loader, *, optimizer, clip_grad=0,
avg_window=100, log_dir="./cache/logs/", log_level=logging.INFO,
checkpoint_dir="./cache/model_cache/", batch_idx=0, echo=False,
device="cuda:0", use_tensorboard=False):
super().__init__(
 model, train_loader, val_loader,
optimizer=optimizer, clip_grad=clip_grad,
log_dir=log_dir, checkpoint_dir=checkpoint_dir,
batch_idx=batch_idx, echo=echo,
device=device, use_tensorboard=use_tensorboard 
)
 self.criterion = torch.nn.CrossEntropyLoss()
self.loss_format = "%.6f"
def extract_prediction(self, tensor):
return tensor
def snapshot(self):

 loss = self.eval(self.val_loader)
 loss_str = self.loss_format % loss
self.logger.info("Snapshot loss %s", loss_str)
 self.logger.tb_scalars(
 "losses", {"val": loss}, self.step)
 target_path =(
 self.checkpoint_dir / "best.pth")
 if not self.best_performers or(self.best_performers[0][0] > loss):
torch.save(self.model.state_dict() , target_path)
 self.best_performers = [(loss, target_path, self.step)]
self.logger.info("Saving checkpoint %s...", target_path)
 else:
new_loss_str = self.loss_format % self.best_performers[0][0]
self.logger.info("This performance:%s is not as a good as our previously saved:%s", loss_str,new_loss_str)
 assert Path(target_path ).exists()
return loss<load_from_csv>
rfr.fit(X, y )
df_train = pd.read_csv("gap-test.tsv", delimiter="\t")
 df_val = pd.read_csv("gap-validation.tsv", delimiter="\t")
 df_test = pd.read_csv(".. /input/test_stage_2.tsv", delimiter="\t")
 sample_sub = pd.read_csv(".. /input/sample_submission_stage_2.csv")
 assert sample_sub.shape[0] == df_test.shape[0]<feature_engineering>
TESTFILEPATH = os.path.join('.. ', 'input', 'test.csv')
 test = pd.read_csv(TESTFILEPATH )
tokenizer = BertTokenizer.from_pretrained(
 BERT_MODEL,
do_lower_case=CASED,
never_split =("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]", "[A]", "[B]", "[P]")
)
 tokenizer.vocab["[A]"] = -1
tokenizer.vocab["[B]"] = -1
tokenizer.vocab["[P]"] = -1<create_dataframe>
test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'])
 test['month_pickup']=test['pickup_datetime'].dt.month
test['day_pickup']=test['pickup_datetime'].dt.dayofweek
test['hour_pickup']=test['pickup_datetime'].dt.hour
test['minute_pickup']=test['pickup_datetime'].dt.minute
test['second_pickup']=test['pickup_datetime'].dt.second
test = test.drop(columns=['pickup_datetime'])
 test['trip_distance']=test.apply(distance, axis=1 )
train_ds = GAPDataset(df_train, tokenizer)
 val_ds = GAPDataset(df_val, tokenizer)
 test_ds = GAPDataset(df_test, tokenizer, labeled=False)
 train_loader = DataLoader(
 train_ds,
collate_fn = collate_examples,
batch_size=20,
num_workers=2,
pin_memory=True,
shuffle=True,
drop_last=True 
)
 val_loader = DataLoader(
 val_ds,
collate_fn = collate_examples,
batch_size=128,
num_workers=2,
pin_memory=True,
shuffle=False 
)
 test_loader = DataLoader(
 test_ds,
collate_fn = collate_examples,
batch_size=128,
num_workers=2,
pin_memory=True,
shuffle=False
)<choose_model_class>
test_X = test[features]
model = GAPModel(BERT_MODEL, torch.device("cuda:0"))
set_trainable(model.bert, False)
 set_trainable(model.head, True )<choose_model_class>
predicted_duration_log = rfr.predict(test_X )
lr=1e-3
weight_decay=5e-3
optimizer = torch.optim.Adam(model.parameters() , lr=lr, weight_decay=weight_decay)
 bot = GAPBot(
 model, train_loader, val_loader,
optimizer=optimizer, echo=True,
avg_window=25
)<train_model>
predicted_duration = np.exp(predicted_duration_log)
 predicted_duration
steps_per_epoch = len(train_loader)
 n_steps = steps_per_epoch * 27
bot.train(
 n_steps,
log_interval=steps_per_epoch // 4,
snapshot_interval=steps_per_epoch,
scheduler=TriangularLR(
 optimizer, 20, ratio=2, steps_per_cycle=n_steps)
 )<load_pretrained>
my_submission = pd.DataFrame({'id': test['id'], 'trip_duration': predicted_duration})
 WORKINGFILEPATH = os.path.join('.. ', 'working', 'submission.csv')
 my_submission.to_csv(WORKINGFILEPATH, index=False )