Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<normalization>
import numpy as np
import pandas as pd
import os
def pca_analyse(columns, num_comp, scaler=None):
pca = PCA(n_components=num_comp)
 if scaler is not None:
columns = scaler.fit_transform(columns)
 pca.fit(columns)
 return np.cumsum(pca.explained_variance_ratio_ )<statistical_test>
!pip install.. /input/sacremoses/sacremoses-master/
!pip install.. /input/transformers/transformers-master/
pca_nonscaled = pca_analyse(X_full.iloc[:, 2:774], 700)
 pca_stand = pca_analyse(X_full.iloc[:, 2:774], 700, StandardScaler())
 pca_quant = pca_analyse(X_full.iloc[:, 2:774], 700, QuantileTransformer(output_distribution="normal"))<statistical_test>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm
import tensorflow_hub as hub
import tensorflow as tf
import tensorflow.keras.backend as K
import gc
import time
import random
import os
import torch
from scipy.stats import spearmanr
from sklearn.model_selection import KFold,StratifiedKFold
from math import floor, ceil
from transformers import AdamW,BertForSequenceClassification, get_cosine_with_hard_restarts_schedule_with_warmup
pca_nonscaled = pca_analyse(X_full.iloc[:, 774:874], 100)
 pca_stand = pca_analyse(X_full.iloc[:, 774:874], 100, StandardScaler())
 pca_quant = pca_analyse(X_full.iloc[:, 774:874], 100, QuantileTransformer(output_distribution="normal"))<normalization>
sample_submission = pd.read_csv(".. /input/google-quest-challenge/sample_submission.csv")
 test = pd.read_csv(".. /input/google-quest-challenge/test.csv")
 train = pd.read_csv(".. /input/google-quest-challenge/train.csv")
 MAX_SEQUENCE_LENGTH = 512
def add_pca(df, columns, num_comp, prefix, scaler):
pca = PCA(n_components=num_comp)
 if scaler is not None:
columns = scaler.fit_transform(columns)
 comps = pca.fit_transform(columns)
 pca_df = pd.DataFrame(comps,
columns=[prefix + f"_pca_{i}".format(i)for i in range(num_comp)])
 return pd.concat(( df.reset_index(drop=True), pca_df.reset_index(drop=True)) , axis=1)
<compute_train_metric>

 def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 if t_len > t_new_len:
ind1 = floor(t_new_len/2)
 ind2 = ceil(t_new_len/2)
 t = t[:ind1]+t[-ind2:]
else:
t = t[:t_new_len]
if q_len > q_new_len:
ind1 = floor(q_new_len/2)
 ind2 = ceil(q_new_len/2)
 q = q[:ind1]+q[-ind2:]
else:
q = q[:q_new_len]
if a_len > a_new_len:
ind1 = floor(a_new_len/2)
 ind2 = ceil(a_new_len/2)
 a = a[:ind1]+a[-ind2:]
else:
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
def variance_threshold(df, columns, thr=0.85):
vt = VarianceThreshold(threshold=thr)
 var = vt.fit(columns ).variances_
drop_cols = columns.columns[var < thr]
return df.drop(columns=drop_cols )<merge>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
train_full = train_features.merge(train_targets_scored, on='sig_id')
 X_full = add_pca(X_full, X_full.iloc[:, 2:774], 600, 'g',
QuantileTransformer(output_distribution="normal"))
X_full = add_pca(X_full, X_full.iloc[:, 774:874], 80, 'c',
QuantileTransformer(output_distribution="normal"))
X_full = variance_threshold(X_full, X_full.drop(columns=['cp_time', 'cp_dose']), thr=0.9)
 X_train = X_full[:X_train.shape[0]]
X_test = X_full[-X_test.shape[0]:]<prepare_x_and_y>
class TextDataset(torch.utils.data.TensorDataset):
def __init__(self, x_train, idxs, targets=None):
self.input_ids = x_train[0][idxs]
self.input_masks = x_train[1][idxs]
self.input_segments = x_train[2][idxs]
self.targets = targets[idxs] if targets is not None else np.zeros(( x_train[0].shape[0], 30))
def __getitem__(self, idx):
input_ids = self.input_ids[idx]
input_masks = self.input_masks[idx]
input_segments = self.input_segments[idx]
target = self.targets[idx]
return input_ids, input_masks, input_segments, target
def __len__(self):
return len(self.input_ids )
class MoaDataset(Dataset):
def __init__(self, X, y=None):
self.X = X
self.y = y
self.device = device
def __getitem__(self, idx):
if self.y is None:
return torch.tensor(self.X[idx], dtype=torch.float)
 return torch.tensor(self.X[idx], dtype=torch.float), \
torch.tensor(self.y[idx], dtype=torch.float)
 def __len__(self):
return self.X.shape[0] 
<normalization>
from transformers import BertTokenizer,BertConfig
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<train_model>
tokenizer = BertTokenizer.from_pretrained('.. /input/bertbaseuncased/bert-base-uncased-vocab.txt' )
def valid_epoch(model, criterion, dataloader, device):
model.eval()
with torch.no_grad() :
total_loss = 0.
 total_num = 0
for X, y in dataloader:
X = X.to(device)
 y = y.to(device)
 outputs = model(X)
 loss = criterion(outputs, y)
 total_loss += loss.item() * X.shape[0]
total_num += X.shape[0]
return total_loss / total_num
def train_epoch(model, criterion, optimizer, scheduler, train_dataloader, valid_dataloader, device):
model.train()
total_loss = 0.
 total_num = 0
for X, y in train_dataloader:
X = X.to(device)
 y = y.to(device)
 optimizer.zero_grad()
outputs = model(X)
 loss = criterion(outputs, y)
 loss.backward()
optimizer.step()
scheduler.step()
total_loss += loss.detach().item() * X.shape[0]
total_num += X.shape[0]
train_loss = total_loss / total_num
val_loss = valid_epoch(model, criterion, val_dataloader, device)
 return train_loss, val_loss
def predict(model, dataset, device):
model.eval()
result = []
with torch.no_grad() :
for elem in dataset:
elem = elem.unsqueeze(0 ).to(device)
 output = model(elem)
 result.append(torch.sigmoid(output ).cpu().numpy())
 return np.concatenate(result )<train_on_grid>
x_train = compute_input_arays(train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 y_train = compute_output_arrays(train, output_categories)
 x_test = compute_input_arays(test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )
device = 'cuda'
train_loss_list = []
val_loss_list = []
pred_list = []
test_dataset = MoaDataset(X_test.values)
 num_features = X_train.shape[1]
num_targets = y_train.shape[1]
criterion = torch.nn.BCEWithLogitsLoss()
seed_list = np.arange(7)
 for seed in seed_list:
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7, shuffle=True, random_state=seed)
 for fold_ind,(train_idx, val_idx)in enumerate(mskf.split(X_train, y_train)) :
curr_train_loss_list = []
curr_val_loss_list = []
train_dataset = MoaDataset(X_train.values[train_idx], y_train.values[train_idx])
 train_dataloader = DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=1)
 val_dataset = MoaDataset(X_train.values[val_idx], y_train.values[val_idx])
 val_dataloader = DataLoader(val_dataset, batch_size=128, shuffle=True, num_workers=1)
 model = Model(num_features, num_targets, 1024 ).to(device)
 optimizer = optim.Adam(model.parameters() , lr=1e-3, weight_decay=1e-5)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=25, steps_per_epoch=len(train_dataloader))
for epoch_num in range(25):
train_loss, val_loss = train_epoch(model, criterion, optimizer, scheduler,
train_dataloader, val_dataloader, device)
 curr_train_loss_list.append(train_loss)
 curr_val_loss_list.append(val_loss)
 print('seed: ', seed, ' fold: ', fold_ind, ' epoch_num: ', epoch_num, ' train loss: ', train_loss)
 print('seed: ', seed, ' fold: ', fold_ind, ' epoch_num: ', epoch_num, ' val loss: ', val_loss)
 train_loss_list.append(curr_train_loss_list)
 val_loss_list.append(curr_val_loss_list)
 pred_list.append(predict(model, test_dataset, device)) 
<compute_test_metric>
bert_config = BertConfig.from_pretrained('.. /input/bertbaseuncased/bert-base-uncased-config.json')
 bert_config.num_labels = 30
print("curr_train_loss: ", sum([elem[-1] for elem in train_loss_list])/ len(train_loss_list))
print("curr_val_loss: ", sum([elem[-1] for elem in val_loss_list])/ len(val_loss_list))<feature_engineering>
class CustomizedBert(BertPreTrainedModel):
r
def __init__(self, config):
super(CustomizedBert, self ).__init__(config)
 self.num_labels = config.num_labels
self.bert = BertModel(config)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.classifier = nn.Linear(config.hidden_size*2, self.config.num_labels)
 self.init_weights()
def forward(
 self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None,
labels=None, 
):
outputs = self.bert(
 input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 avg_pool = torch.mean(outputs[0], 1)
 max_pool, _ = torch.max(outputs[0], 1)
 pooled_output = torch.cat(( max_pool, avg_pool), 1)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
if labels is not None:
if self.num_labels == 1:
loss_fct = MSELoss()
loss = loss_fct(logits.view(-1), labels.view(-1))
else:
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
outputs =(loss,)+ outputs
return outputs
y_test = np.zeros(( sample_submission.shape[0], y_train.shape[1]))
y_test[test_features.cp_type != 'ctl_vehicle'] = sum(pred_list)/ len(pred_list)
<create_dataframe>
class callback:
def __init__(self):
self.score = list()
self.model = list()
self.data = list()
def put(self, model,data, score):
self.score.append(score)
 self.model.append(model)
 self.data.append(data)
 def get_model(self):
ind = np.argmin(self.score)
 return self.model[ind]
def get_data(self):
ind = np.argmin(self.score)
 return self.data[ind]
submission = pd.DataFrame(y_test, columns=train_targets_scored.columns[1:])
 submission['sig_id'] = test_features['sig_id']<save_to_csv>
NFOLDS = 5
BATCH_SIZE = 4
EPOCHS = 4
SEED = 8516
num_warmup_steps = 100
lr = 5e-5
gradient_accumulation_steps = 1
seed_everything(SEED)
 model_list = list()
y_oof = np.zeros(( len(train), 30))
test_pred = np.zeros(( len(test), 30))
kf = KFold(n_splits=NFOLDS, shuffle=True)
 test_loader = torch.utils.data.DataLoader(TextDataset(x_test, test.index),batch_size=BATCH_SIZE, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(kf.split(x_train[0])) :
print(f'fold {i+1}')
 gc.collect()
train_loader = torch.utils.data.DataLoader(TextDataset(x_train, train_idx, y_train),batch_size=BATCH_SIZE, shuffle=True)
 val_loader = torch.utils.data.DataLoader(TextDataset(x_train, valid_idx, y_train),batch_size=BATCH_SIZE, shuffle=False)
 t_total = len(train_loader)//gradient_accumulation_steps*EPOCHS
net = CustomizedBert.from_pretrained('.. /input/bertbaseuncased/', config=bert_config)
 net.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss()
optimizer = AdamW(net.parameters() , lr = lr)
 scheduler = get_cosine_with_hard_restarts_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=t_total)
 cb = callback()
for epoch in range(EPOCHS):
start_time = time.time()
avg_loss = 0.0
net.train()
for step, data in enumerate(train_loader):
input_ids, input_masks, input_segments, labels = data
pred = net(input_ids = input_ids.long().cuda() ,
labels = None,
attention_mask = input_masks.cuda() ,
token_type_ids = input_segments.long().cuda() 
)[0]
loss = loss_fn(pred, labels.cuda())
 avg_loss += loss.item()
loss = loss / gradient_accumulation_steps
loss.backward()
if(step + 1)% gradient_accumulation_steps == 0:
optimizer.step()
scheduler.step()
optimizer.zero_grad()
avg_val_loss = 0.0
valid_preds = np.zeros(( len(valid_idx), 30))
true_label = np.zeros(( len(valid_idx), 30))
net.eval()
for j,data in enumerate(val_loader):
input_ids, input_masks, input_segments, labels = data
pred = net(input_ids = input_ids.long().cuda() ,
labels = None,
attention_mask = input_masks.cuda() ,
token_type_ids = input_segments.long().cuda() 
)[0]
loss_val = loss_fn(pred, labels.cuda())
 avg_val_loss += loss_val.item()
valid_preds[j * BATCH_SIZE:(j+1)* BATCH_SIZE] = torch.sigmoid(pred ).cpu().detach().numpy()
true_label[j * BATCH_SIZE:(j+1)* BATCH_SIZE] = labels
elapsed_time = time.time() - start_time
score = 0
for i in range(30):
s = np.nan_to_num(
 spearmanr(true_label[:, i], valid_preds[:, i] ).correlation / 30)
 score += s
print('Epoch {}/{} \t loss={:.4f}\t val_loss={:.4f}\t spearmanr={:.4f}\t time={:.2f}s'.format(epoch+1, EPOCHS, avg_loss/len(train_loader),avg_val_loss/len(val_loader),score, elapsed_time))
cb.put(net,valid_preds,avg_val_loss/len(val_loader))
model_list.append(cb.get_model())
 y_oof[valid_idx] = cb.get_data()
result = list()
net.eval()
with torch.no_grad() :
for data in test_loader:
input_ids, input_masks, input_segments, labels = data
y_pred = net(input_ids = input_ids.long().cuda() ,
labels = None,
attention_mask = input_masks.cuda() ,
token_type_ids = input_segments.long().cuda() , 
)[0]
result.extend(torch.sigmoid(y_pred ).cpu().detach().numpy())
 test_pred += np.array(result)/NFOLDS
<save_to_csv><EOS> 
sample_submission.loc[:, output_categories] = test_pred
sample_submission.to_csv('submission.csv', index=False )