Kaggle competition: Quora Question Pairs
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']<compute_test_metric>
import torch
import torch.nn as nn
import seaborn as sns
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.model_selection import train_test_split
import transformers
from transformers import BertModel, BertTokenizer, AdamW, get_linear_schedule_with_warmup
del merged, site1
print(f'RMSLE of buildings 105-155: {rmsle(ytrue, pred):.4f}' )<load_from_disk>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
site2 = pd.read_feather('.. /input/asu-feather/site2.ft')
 site2 = site2.query('timestamp >= 2017' )<merge>
SEED = 1234
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
merged = df_test.merge(site2, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )<prepare_x_and_y>
train_df = pd.read_csv("/kaggle/input/quora-question-pairs/train.csv.zip")
 train_df.info()
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']<compute_test_metric>
train_df.dropna(inplace=True )
del site2, merged
print(f'RMSLE of buildings 156-290: {rmsle(ytrue, pred):.4f}' )<load_from_disk>
train_sentences_lens = train_df['question1'].apply(lambda x: len(x.split(' ')) ).tolist()
train_sentences_lens.extend(train_df['question2'].apply(lambda x: len(x.split(' ')) ).tolist())
 sns.distplot(train_sentences_lens )
site4 = pd.read_feather('.. /input/ucb-feather/site4.ft')
 site4 = site4.query('timestamp >= 2017' )<merge>
MAX_LEN = 40
merged = df_test.merge(site4, left_on=['building_id', 'timestamp'],
right_on=['building_id', 'timestamp'], how='left' )<prepare_x_and_y>
qids = pd.Series(list(train_df['qid1'])+ list(train_df['qid2']))
print('Unique Questions number: {}
'.format(len(np.unique(qids))))
q_vals=qids.value_counts() [0:5]
print('Top 5 most frequently asked questions: ')
 for pair in q_vals.iteritems() :
print(train_df.loc[train_df['qid2']==pair[0]]['question1'].head(1 ).values + " count: " + str(pair[1]))
q_vals=q_vals.values
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']<compute_test_metric>
duplicate_rows = train_df[train_df.duplicated(['qid1','qid2'])]
print("Number of duplicate questions : ", len(duplicate_rows))
del site4, merged
print(f'RMSLE of 74/91 buildings : {rmsle(ytrue, pred):.4f}' )<load_from_disk>
def common_words(row):
q1_word_set = set(map(lambda word: word.lower().strip() , row['question1'].split(" ")))
 q2_word_set = set(map(lambda word: word.lower().strip() , row['question2'].split(" ")))
 return 1.0 * len(q1_word_set & q2_word_set)
 train_df['common_words'] = train_df.apply(common_words, axis=1 )
site15 = pd.read_feather('.. /input/cornell-feather/site15.ft')
 site15 = site15.query('timestamp >= 2017')
 site15 = site15.drop_duplicates()<merge>
def shared_words(row):
q1_word_set = set(map(lambda word: word.lower().strip() , row['question1'].split(" ")))
 q2_word_set = set(map(lambda word: word.lower().strip() , row['question2'].split(" ")))
 return 1.0 * len(q1_word_set & q2_word_set)/(len(q1_word_set)+ len(q2_word_set))
train_df['shared_words'] = train_df.apply(shared_words, axis=1 )
merged = df_test.merge(site15, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )<prepare_x_and_y>
BERT_VERSION = 'bert-base-uncased'
POOLED_OUTPUT_DIM = 768
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']<compute_test_metric>
tokenizer = BertTokenizer.from_pretrained(BERT_VERSION )
del site15, merged
print(f'RMSLE of buildings 1325-1448: {rmsle(ytrue, pred):.4f}' )<load_from_disk>
train_df, val_df = train_test_split(train_df, test_size=0.1)
 train_df = train_df.reset_index(drop=True)
 val_df = val_df.reset_index(drop=True )
site012 = pd.read_feather('.. /input/comb-leaked-dataset/site012.ft')
 site012 = site012.query('timestamp >= 2017' )<merge>
class BertDataSet:
def __init__(self, first_questions, second_questions, targets, tokenizer):
self.first_questions = first_questions
self.second_questions = second_questions
self.targets = targets
self.tokenizer = tokenizer
self.length = len(first_questions)
 def __len__(self):
return self.length
def __getitem__(self, item):
first_question = str(self.first_questions[item])
 second_question = str(self.second_questions[item])
 first_question = " ".join(first_question.split())
 second_question = " ".join(second_question.split())
 inputs = self.tokenizer.encode_plus(
 first_question,
second_question,
add_special_tokens=True,
padding='max_length',
max_length=2 * MAX_LEN + 3,
truncation=True 
)
 return {
"ids": torch.tensor(inputs["input_ids"], dtype=torch.long),
"mask": torch.tensor(inputs["attention_mask"], dtype=torch.long),
"token_type_ids": torch.tensor(inputs["token_type_ids"], dtype=torch.long),
"targets": torch.tensor(int(self.targets[item]), dtype=torch.long)if self.targets is not None else 0
}
merged = df_test.merge(site012, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )<prepare_x_and_y>
def get_data_loader(df, targets, batch_size, shuffle, tokenizer):
dataset = BertDataSet(
 first_questions=df["question1"].values,
second_questions=df["question2"].values,
targets=targets,
tokenizer=tokenizer 
)
 data_loader = torch.utils.data.DataLoader(
 dataset,
batch_size = batch_size,
shuffle=shuffle 
)
 return data_loader
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']<compute_test_metric>
BS = 128
del site012, merged
print(f'RMSLE of buildings 0-290: {rmsle(ytrue, pred):.4f}' )<set_options>
train_data_loader = get_data_loader(
 df=train_df,
targets=train_df["is_duplicate"].values,
batch_size=BS,
shuffle=True,
tokenizer=tokenizer 
)
 val_data_loader = get_data_loader(
 df=val_df,
targets=val_df["is_duplicate"].values,
batch_size=4 * BS,
shuffle=True,
tokenizer=tokenizer
)
py.init_notebook_mode(connected=True)
 pd.set_option('max_columns', 150)
 py.init_notebook_mode(connected=True)
 init_notebook_mode(connected=True)
<define_variables>
class BertModel(nn.Module):
def __init__(self, bert_path):
super(BertModel, self ).__init__()
self.bert_path = bert_path
self.bert = transformers.BertModel.from_pretrained(self.bert_path)
 self.dropout = nn.Dropout(0.3)
 self.out = nn.Linear(POOLED_OUTPUT_DIM, 1)
 def forward(self, ids, mask, token_type_ids):
_, pooled = self.bert(ids, attention_mask=mask,token_type_ids=token_type_ids)
 pooled = self.dropout(pooled)
 return self.out(pooled)
 model = BertModel(BERT_VERSION ).to(device )
metadata_dtype = {'site_id':"uint8",'building_id':'uint16','square_feet':'float32','year_built':'float32','floor_count':"float16"}
weather_dtype = {"site_id":"uint8",'air_temperature':"float16",'cloud_coverage':"float16",'dew_temperature':"float16",'precip_depth_1_hr':"float16",
'sea_level_pressure':"float32",'wind_direction':"float16",'wind_speed':"float16"}
train_dtype = {'meter':"uint8",'building_id':'uint16'}<load_from_csv>
def loss_fn(outputs, targets):
outputs = torch.squeeze(outputs)
 return nn.BCELoss()(nn.Sigmoid()(outputs), targets )
%%time
weather_train = pd.read_csv(".. /input/ashrae-energy-prediction/weather_train.csv", parse_dates=['timestamp'], dtype=weather_dtype)
 weather_test = pd.read_csv(".. /input/ashrae-energy-prediction/weather_test.csv", parse_dates=['timestamp'], dtype=weather_dtype)
 metadata = pd.read_csv(".. /input/ashrae-energy-prediction/building_metadata.csv", dtype=metadata_dtype)
 train = pd.read_csv(".. /input/ashrae-energy-prediction/train.csv", parse_dates=['timestamp'], dtype=train_dtype)
 test = pd.read_csv(".. /input/ashrae-energy-prediction/test.csv", parse_dates=['timestamp'], usecols=['building_id','meter','timestamp'], dtype=train_dtype)
 print('Size of train_df data', train.shape)
 print('Size of weather_train_df data', weather_train.shape)
 print('Size of weather_test_df data', weather_test.shape)
 print('Size of building_meta_df data', metadata.shape )<categorify>
def calculate_perplexity(data_loader, model, device):
model.eval()
with torch.no_grad() :
total_loss = 0
for batch in data_loader:
ids = batch["ids"].to(device, dtype=torch.long)
 mask = batch["mask"].to(device, dtype=torch.long)
 token_type_ids = batch["token_type_ids"].to(device, dtype=torch.long)
 targets = batch["targets"].to(device, dtype=torch.float)
 outputs = model(ids=ids, mask=mask, token_type_ids=token_type_ids)
 total_loss += loss_fn(outputs, targets ).item()
model.train()
return np.exp(total_loss / len(data_loader))
train['meter'].replace({0:"Electricity",1:"ChilledWater",2:"Steam",3:"HotWater"},inplace=True)
 test['meter'].replace({0:"Electricity",1:"ChilledWater",2:"Steam",3:"HotWater"},inplace=True )<drop_column>
def train_loop(epochs, train_data_loader, val_data_loader, model, optimizer, device, scheduler=None):
it = 1
total_loss = 0
curr_perplexity = None
perplexity = None
model.train()
for epoch in range(epochs):
print('Epoch: ', epoch + 1)
 for batch in train_data_loader:
ids = batch["ids"].to(device, dtype=torch.long)
 mask = batch["mask"].to(device, dtype=torch.long)
 token_type_ids = batch["token_type_ids"].to(device, dtype=torch.long)
 targets = batch["targets"].to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(ids, mask=mask, token_type_ids=token_type_ids)
 loss = loss_fn(outputs, targets)
 total_loss += loss.item()
loss.backward()
optimizer.step()
if it % 100 == 0:
if it % 500 == 0:
curr_perplexity = calculate_perplexity(val_data_loader, model, device)
 if scheduler is not None:
scheduler.step()
if not perplexity or curr_perplexity < perplexity:
torch.save(model.state_dict() , 'saved_model')
 perplexity = curr_perplexity
print('| Iter', it, '| Avg Train Loss', total_loss / 100, '| Dev Perplexity', curr_perplexity)
 total_loss = 0
it += 1
metadata.drop('floor_count',axis=1,inplace=True )<data_type_conversions>
def run(model, train_df, device, train_data_loader, val_data_loader):
EPOCHS = 1
lr = 3e-5
num_training_steps = int(len(train_data_loader)* EPOCHS)
 optimizer = AdamW(model.parameters() , lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_training_steps 
)
 train_loop(EPOCHS, train_data_loader, val_data_loader, model, optimizer, device, scheduler )
for df in [train, test]:
df['Month'] = df['timestamp'].dt.month.astype("uint8")
 df['DayOfMonth'] = df['timestamp'].dt.day.astype("uint8")
 df['DayOfWeek'] = df['timestamp'].dt.dayofweek.astype("uint8")
 df['Hour'] = df['timestamp'].dt.hour.astype("uint8" )<feature_engineering>
run(model, train_df, device, train_data_loader, val_data_loader )
train['meter_reading'] = np.log1p(train['meter_reading'] )<data_type_conversions>
test_df = pd.read_csv("/kaggle/input/quora-question-pairs/test.csv")
 test_df.head()
metadata['primary_use'].replace({"Healthcare":"Other","Parking":"Other","Warehouse/storage":"Other","Manufacturing/industrial":"Other",
"Retail":"Other","Services":"Other","Technology/science":"Other","Food sales and service":"Other",
"Utility":"Other","Religious worship":"Other"},inplace=True)
 metadata['square_feet'] = np.log1p(metadata['square_feet'])
 metadata['year_built'].fillna(-999, inplace=True)
 metadata['year_built'] = metadata['year_built'].astype('int16' )<merge>
def test(model, test_df, device):
predictions = torch.empty(0 ).to(device, dtype=torch.float)
 test_dataset = BertDataSet(
 first_questions=test_df["question1"].values,
second_questions=test_df["question2"].values,
targets=None,
tokenizer=tokenizer 
)
 test_data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=512 
)
 with torch.no_grad() :
model.eval()
for batch in tqdm(test_data_loader):
ids = batch["ids"]
mask = batch["mask"]
token_type_ids = batch["token_type_ids"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 outputs = model(ids=ids, mask=mask, token_type_ids=token_type_ids)
 predictions = torch.cat(( predictions, nn.Sigmoid()(outputs)))
 return predictions.cpu().numpy().squeeze()
predictions = test(model, test_df, device)
 len(predictions )
%%time
train = pd.merge(train,metadata,on='building_id',how='left')
 test = pd.merge(test,metadata,on='building_id',how='left')
 print("Training Data+Metadata Shape {}".format(train.shape))
print("Testing Data+Metadata Shape {}".format(test.shape))
gc.collect()
train = pd.merge(train,weather_train,on=['site_id','timestamp'],how='left')
 test = pd.merge(test,weather_test,on=['site_id','timestamp'],how='left')
 print("Training Data+Metadata+Weather Shape {}".format(train.shape))
print("Testing Data+Metadata+Weather Shape {}".format(test.shape))
gc.collect()<data_type_conversions>
test_df['is_duplicate'] = predictions
for df in [train,test]:
df['square_feet'] = df['square_feet'].astype('float16')
 cols = ['air_temperature','cloud_coverage','dew_temperature','precip_depth_1_hr','sea_level_pressure','wind_direction','wind_speed']
for col in cols:
train[col].fillna(train[col].mean() ,inplace=True)
 test[col].fillna(test[col].mean() ,inplace=True)
 idx_to_drop = list(( train[(train['site_id'] == 0)&(train['timestamp'] < "2016-05-21 00:00:00")] ).index)
 print(len(idx_to_drop))
train.drop(idx_to_drop,axis='rows',inplace=True)
 idx_to_drop = list(train[(train['meter'] == "Electricity")&(train['meter_reading'] == 0)].index)
 print(len(idx_to_drop))
train.drop(idx_to_drop,axis='rows',inplace=True )<feature_engineering>
test_df[['test_id', 'is_duplicate']].to_csv('submission.csv', index=False )
%%time
number_unique_meter_per_building = train.groupby('building_id')['meter'].nunique()
train['number_unique_meter_per_building'] = train['building_id'].map(number_unique_meter_per_building)
 mean_meter_reading_per_building = train.groupby('building_id')['meter_reading'].mean()
train['mean_meter_reading_per_building'] = train['building_id'].map(mean_meter_reading_per_building)
 median_meter_reading_per_building = train.groupby('building_id')['meter_reading'].median()
train['median_meter_reading_per_building'] = train['building_id'].map(median_meter_reading_per_building)
 std_meter_reading_per_building = train.groupby('building_id')['meter_reading'].std()
train['std_meter_reading_per_building'] = train['building_id'].map(std_meter_reading_per_building)
 mean_meter_reading_on_year_built = train.groupby('year_built')['meter_reading'].mean()
train['mean_meter_reading_on_year_built'] = train['year_built'].map(mean_meter_reading_on_year_built)
 median_meter_reading_on_year_built = train.groupby('year_built')['meter_reading'].median()
train['median_meter_reading_on_year_built'] = train['year_built'].map(median_meter_reading_on_year_built)
 std_meter_reading_on_year_built = train.groupby('year_built')['meter_reading'].std()
train['std_meter_reading_on_year_built'] = train['year_built'].map(std_meter_reading_on_year_built)
 mean_meter_reading_per_meter = train.groupby('meter')['meter_reading'].mean()
train['mean_meter_reading_per_meter'] = train['meter'].map(mean_meter_reading_per_meter)
 median_meter_reading_per_meter = train.groupby('meter')['meter_reading'].median()
train['median_meter_reading_per_meter'] = train['meter'].map(median_meter_reading_per_meter)
 std_meter_reading_per_meter = train.groupby('meter')['meter_reading'].std()
train['std_meter_reading_per_meter'] = train['meter'].map(std_meter_reading_per_meter)
 mean_meter_reading_per_primary_usage = train.groupby('primary_use')['meter_reading'].mean()
train['mean_meter_reading_per_primary_usage'] = train['primary_use'].map(mean_meter_reading_per_primary_usage)
 median_meter_reading_per_primary_usage = train.groupby('primary_use')['meter_reading'].median()
train['median_meter_reading_per_primary_usage'] = train['primary_use'].map(median_meter_reading_per_primary_usage)
 std_meter_reading_per_primary_usage = train.groupby('primary_use')['meter_reading'].std()
train['std_meter_reading_per_primary_usage'] = train['primary_use'].map(std_meter_reading_per_primary_usage)
 mean_meter_reading_per_site_id = train.groupby('site_id')['meter_reading'].mean()
train['mean_meter_reading_per_site_id'] = train['site_id'].map(mean_meter_reading_per_site_id)
 median_meter_reading_per_site_id = train.groupby('site_id')['meter_reading'].median()
train['median_meter_reading_per_site_id'] = train['site_id'].map(median_meter_reading_per_site_id)
 std_meter_reading_per_site_id = train.groupby('site_id')['meter_reading'].std()
train['std_meter_reading_per_site_id'] = train['site_id'].map(std_meter_reading_per_site_id)
 test['number_unique_meter_per_building'] = test['building_id'].map(number_unique_meter_per_building)
 test['mean_meter_reading_per_building'] = test['building_id'].map(mean_meter_reading_per_building)
 test['median_meter_reading_per_building'] = test['building_id'].map(median_meter_reading_per_building)
 test['std_meter_reading_per_building'] = test['building_id'].map(std_meter_reading_per_building)
 test['mean_meter_reading_on_year_built'] = test['year_built'].map(mean_meter_reading_on_year_built)
 test['median_meter_reading_on_year_built'] = test['year_built'].map(median_meter_reading_on_year_built)
 test['std_meter_reading_on_year_built'] = test['year_built'].map(std_meter_reading_on_year_built)
 test['mean_meter_reading_per_meter'] = test['meter'].map(mean_meter_reading_per_meter)
 test['median_meter_reading_per_meter'] = test['meter'].map(median_meter_reading_per_meter)
 test['std_meter_reading_per_meter'] = test['meter'].map(std_meter_reading_per_meter)
 test['mean_meter_reading_per_primary_usage'] = test['primary_use'].map(mean_meter_reading_per_primary_usage)
 test['median_meter_reading_per_primary_usage'] = test['primary_use'].map(median_meter_reading_per_primary_usage)
 test['std_meter_reading_per_primary_usage'] = test['primary_use'].map(std_meter_reading_per_primary_usage)
 test['mean_meter_reading_per_site_id'] = test['site_id'].map(mean_meter_reading_per_site_id)
 test['median_meter_reading_per_site_id'] = test['site_id'].map(median_meter_reading_per_site_id)
 test['std_meter_reading_per_site_id'] = test['site_id'].map(std_meter_reading_per_site_id )<data_type_conversions>
def eval(model, tokenizer, first_question, second_question, device):
inputs = tokenizer.encode_plus(
 first_question,
second_question,
add_special_tokens=True, 
)
 ids = torch.tensor([inputs["input_ids"]], dtype=torch.long ).to(device, dtype=torch.long)
 mask = torch.tensor([inputs["attention_mask"]], dtype=torch.long ).to(device, dtype=torch.long)
 token_type_ids = torch.tensor([inputs["token_type_ids"]], dtype=torch.long ).to(device, dtype=torch.long)
 with torch.no_grad() :
model.eval()
output = model(ids=ids, mask=mask, token_type_ids=token_type_ids)
 prob = nn.Sigmoid()(output ).item()
print("questions [{}] and [{}] are {} with score {}".format(first_question, second_question, 'similar' if prob > 0.5 else 'not similar', prob))