Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<define_variables>
warnings.filterwarnings('ignore' )
text_cols += ['project_essay_1', 'project_essay_2', 'project_resource_summary', 'description_concatenate']<feature_engineering>
def seed_all(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
%%time
subj = lambda x: TextBlob(x ).sentiment.subjectivity
polar = lambda x: TextBlob(x ).sentiment.polarity
test_train['description_subjectivity'] = test_train['description_concatenate'].apply(subj)
 test_train['description_polarity'] = test_train['description_concatenate'].apply(polar)
 test_train['project_resource_summary_subjectivity'] = test_train['project_resource_summary'].apply(subj)
 test_train['project_resource_summary_polarity'] = test_train['project_resource_summary'].apply(polar)
 test_train['project_essay_2_polarity'] = test_train['project_essay_2'].apply(polar)
 test_train['project_essay_1_polarity'] = test_train['project_essay_1'].apply(polar )<define_variables>
SEED = 42
seed_all(SEED )
text_cols += ['project_title']
print(text_cols )<feature_engineering>
sub = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 TARGET_COLS = list(sub.columns[1:].values)
 for label in TARGET_COLS: print(label )
%%time
vectorizer = CountVectorizer()
for text_col in text_cols:
X = vectorizer.fit_transform(test_train[text_col].values)
 col_new_name = text_col.replace('project_', '')
 unique_words =(X > 0 ).sum(axis=1)
 num_words =(X ).sum(axis=1)
 test_train[col_new_name + '_unique_words'] = unique_words
test_train[col_new_name + '_num_words'] = num_words
test_train[col_new_name + '_vocab'] = np.exp(unique_words /(num_words + 10e-10))<save_to_csv>
def decontract(text):
text = re.sub(r"(W|w)on('|\’)t ", "will not ", text)
 text = re.sub(r"(C|c)an('|\’)t ", "can not ", text)
 text = re.sub(r"(Y|y )('|\’)all ", "you all ", text)
 text = re.sub(r"(Y|y)a('|\’)ll ", "you all ", text)
 text = re.sub(r"(I|i )('|\’)m ", "i am ", text)
 text = re.sub(r"(A|a)isn('|\’)t ", "is not ", text)
 text = re.sub(r"n('|\’)t ", " not ", text)
 text = re.sub(r"('|\’)re ", " are ", text)
 text = re.sub(r"('|\’)d ", " would ", text)
 text = re.sub(r"('|\’)ll ", " will ", text)
 text = re.sub(r"('|\’)t ", " not ", text)
 text = re.sub(r"('|\’)ve ", " have ", text)
 return text
for text_col in text_cols:
test_train[text_col] = test_train[text_col].str.replace('"', ' ')
<load_from_csv>
MAX_LEN = 512
class QuestDatasetBert(torch.utils.data.Dataset):
def __init__(self, df, train_mode=True, labeled=True):
self.df = df
self.train_mode = train_mode
self.labeled = labeled
self.tokenizer = BertTokenizer.from_pretrained('.. /input/btp-uncased-base/')
 def __getitem__(self, index):
row = self.df.iloc[index]
token_ids, seg_ids, masks = self.get_token_ids(row)
 if self.labeled:
labels = self.get_label(row)
 return token_ids, seg_ids, masks, labels
else:
return token_ids, seg_ids, masks
def __len__(self):
return len(self.df)
 def select_tokens(self, tokens, max_num):
if len(tokens)<= max_num:
return tokens
if self.train_mode:
num_remove = len(tokens)- max_num
remove_start = random.randint(0, len(tokens)-num_remove-1)
 return tokens[:remove_start] + tokens[remove_start + num_remove:]
else:
return tokens[:max_num//2] + tokens[-(max_num - max_num//2):]
def preprocessing_text(self, text):
text = re.sub('\t|
|\r', '', text)
 text = decontract(text)
 return text
def trim_input(self, title, question, answer, max_sequence_length=MAX_LEN,
t_max_len=30, q_max_len=239, a_max_len=239):
t = self.tokenizer.tokenize(self.preprocessing_text(title))
q = self.tokenizer.tokenize(self.preprocessing_text(question))
a = self.tokenizer.tokenize(self.preprocessing_text(answer))
t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = self.select_tokens(t, t_new_len)
 q = self.select_tokens(q, q_new_len)
 a = self.select_tokens(a, a_new_len)
 return t, q, a
def get_token_ids(self, row):
t_tokens, q_tokens, a_tokens = self.trim_input(row.question_title, row.question_body, row.answer)
 tokens = ['[CLS]'] + t_tokens + ['[SEP]'] + q_tokens + ['[SEP]'] + a_tokens + ['[SEP]']
token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
 if len(token_ids)< MAX_LEN:
token_ids += [0] *(MAX_LEN - len(token_ids))
ids = torch.tensor(token_ids)
 seg_ids = self.get_seg_ids(ids)
 masks = self.get_masks(ids)
 return ids, seg_ids, masks
def get_seg_ids(self, ids):
seg_ids = torch.zeros_like(ids)
 seg_idx = 0
first_sep = True
for i, e in enumerate(ids):
seg_ids[i] = seg_idx
if e == 102:
if first_sep:
first_sep = False
else:
seg_idx = 1
return seg_ids
def get_masks(self, ids):
masks = torch.where(ids != 0, torch.tensor(1), torch.tensor(0))
return masks
def get_label(self, row):
return torch.tensor(row[TARGET_COLS].values.astype(np.float32))
class QuestDatasetXLNet(torch.utils.data.Dataset):
def __init__(self, df, train_mode=True, labeled=True):
self.df = df
self.train_mode = train_mode
self.labeled = labeled
self.tokenizer = XLNetTokenizer.from_pretrained('.. /input/xlnet-base-pytorch/xlnet-base-cased-spiece.model')
 def __getitem__(self, index):
row = self.df.iloc[index]
token_ids, seg_ids, masks, cls_index = self.get_token_ids(row)
 if self.labeled:
labels = self.get_label(row)
 return token_ids, seg_ids, masks, cls_index, labels
else:
return token_ids, seg_ids, masks, cls_index
def __len__(self):
return len(self.df)
 def select_tokens(self, tokens, max_num):
if len(tokens)<= max_num:
return tokens
if self.train_mode:
num_remove = len(tokens)- max_num
remove_start = random.randint(0, len(tokens)-num_remove-1)
 return tokens[:remove_start] + tokens[remove_start + num_remove:]
else:
return tokens[:max_num//2] + tokens[-(max_num - max_num//2):]
def preprocessing_text(self, text):
text = re.sub('\t|
|\r', '', text)
 text = decontract(text)
 return text
def trim_input(self, title, question, answer, max_sequence_length=MAX_LEN,
t_max_len=30, q_max_len=239, a_max_len=239):
t = self.tokenizer.tokenize(self.preprocessing_text(title))
q = self.tokenizer.tokenize(self.preprocessing_text(question))
a = self.tokenizer.tokenize(self.preprocessing_text(answer))
t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = self.select_tokens(t, t_new_len)
 q = self.select_tokens(q, q_new_len)
 a = self.select_tokens(a, a_new_len)
 return t, q, a
def get_token_ids(self, row):
t_tokens, q_tokens, a_tokens = self.trim_input(row.question_title, row.question_body, row.answer)
 tokens = t_tokens + ['<sep>'] + q_tokens + ['<sep>'] + a_tokens + ['<sep>'] + ['<cls>']
token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
 cls_index = torch.tensor(len(token_ids)- 1)
 if len(token_ids)< MAX_LEN:
token_ids += [5] *(MAX_LEN - len(token_ids))
ids = torch.tensor(token_ids)
 seg_ids = self.get_seg_ids(ids)
 masks = self.get_masks(ids)
 return ids, seg_ids, masks, cls_index
def get_seg_ids(self, ids):
seg_ids = torch.zeros_like(ids)
 seg_idx = 0
first_sep = True
for i, e in enumerate(ids):
seg_ids[i] = seg_idx
if e == 4:
if first_sep:
first_sep = False
else:
seg_idx = 1
return seg_ids
def get_masks(self, ids):
masks = torch.where(ids != 5, torch.tensor(1), torch.tensor(0))
return masks
def get_label(self, row):
return torch.tensor(row[TARGET_COLS].values.astype(np.float32))
def get_test_loader(batch_size=32, model='bert'):
df = pd.read_csv('.. /input/google-quest-challenge/test.csv')
 if model == 'bert':
ds_test = QuestDatasetBert(df, train_mode=False, labeled=False)
 else:
ds_test = QuestDatasetXLNet(df, train_mode=False, labeled=False)
 loader = torch.utils.data.DataLoader(ds_test, batch_size=batch_size, shuffle=False,
num_workers=2, drop_last=False)
 return loader
<define_variables>
class BertForQA(nn.Module):
def __init__(self, net_bert, n_classes):
super(BertForQA, self ).__init__()
self.bert = net_bert
self.dropout = nn.Dropout(0.5)
 self.cls = nn.Linear(in_features=768, out_features=n_classes)
 nn.init.normal_(self.cls.weight, std=0.02)
 nn.init.normal_(self.cls.bias, 0)
 def forward(self, input_ids, token_type_ids=None, attention_mask=None):
_, _, hidden_states = self.bert(input_ids, attention_mask, token_type_ids)
 all_h = torch.cat([hidden_states[-i][:, 0].reshape(( -1, 1, 768)) for i in range(1, 5)], 1)
 mean_pool = torch.mean(all_h, 1)
 pooled_output = self.dropout(mean_pool)
 output = self.cls(pooled_output)
 return output
numeric_cols = [
'teacher_number_of_previously_posted_projects',
'description_nunique',
'quantity_sum',
'price_sum',
'price_mean',
'p_desc_len_mean',
'p_desc_len_amin',
'p_desc_len_amax',
'p_total_price_mean',
'p_total_price_amin',
'p_total_price_amax',
'project_resource_summary_subjectivity',
'project_resource_summary_polarity',
'project_essay_2_polarity',
'project_essay_1_polarity',
'essay_1_unique_words',
'essay_1_num_words',
'essay_1_vocab',
'essay_2_unique_words',
'essay_2_num_words',
'essay_2_vocab',
'resource_summary_unique_words',
'resource_summary_num_words',
'resource_summary_vocab',
'description_concatenate_unique_words',
'description_concatenate_num_words',
'description_concatenate_vocab',
'title_unique_words',
'title_num_words',
'title_vocab']<import_modules>
class XLNetForQA(nn.Module):
def __init__(self, xlnet, n_classes):
super(XLNetForQA, self ).__init__()
self.xlnet = xlnet
self.dropout = nn.Dropout(0.5)
 self.cls = nn.Linear(in_features=768, out_features=n_classes)
 nn.init.normal_(self.cls.weight, std=0.02)
 nn.init.normal_(self.cls.bias, 0)
 def forward(self, input_ids, token_type_ids, attention_mask, cls_index):
_, hidden_states = self.xlnet(
 input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids)
 cls_index = cls_index[:, None, None].expand(-1, -1, 768)
 all_h = torch.cat([hidden_states[-i].gather(-2, cls_index)for i in range(1, 5)], 1)
 mean_pool = torch.mean(all_h, 1)
 pooled_output = self.dropout(mean_pool)
 output = self.cls(pooled_output)
 return output
from tempfile import mkdtemp
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.decomposition import TruncatedSVD
from sklearn.metrics import roc_auc_score, roc_curve, auc
from scipy.sparse import coo_matrix, hstack
from sklearn.preprocessing import RobustScaler<train_model>
def build_model(model='bert'):
if model == 'bert':
config = BertConfig.from_pretrained('.. /input/btp-uncased-base/bert_config.json', output_hidden_states=True)
 net_bert = BertModel.from_pretrained('.. /input/btp-uncased-base/pytorch_model.bin', config=config)
 net = BertForQA(net_bert, 30)
 else:
config = XLNetConfig.from_pretrained('.. /input/xlnet-base-pytorch/xlnet-base-cased-config.json', output_hidden_states=True)
 xlnet = XLNetModel.from_pretrained('.. /input/xlnet-base-pytorch/xlnet-base-cased-pytorch_model.bin', config=config)
 net = XLNetForQA(xlnet, 30)
 return net
class ColSplitter(BaseEstimator, TransformerMixin):

 def __init__(self, cols, ravel=False):
self.cols = cols
self.ravel = ravel
def fit(self, x, y=None):
return self
def transform(self, x):
return(x.loc[:, self.cols].values.ravel() if self.ravel
else x.loc[:, self.cols].values)
 class LogTransform(BaseEstimator, TransformerMixin):

 def __init__(self, alpha):

 self.alpha = alpha
def fit(self, x, y=None):
return self
def transform(self, x):
return self.alpha * np.log(np.log(x + 2.001)+ 1.001)+(1 - self.alpha)* x
class ParallelPipe(BaseEstimator, TransformerMixin):

 def __init__(self, pipes, *args, **kwargs):
self.pipes = pipes
def fit(self, x, y=None):
[p.fit(x)for p in self.pipes]
return self
def transform(self, x):
try:
return hstack([p.transform(x)for p in self.pipes])
 except:
return np.concatenate(tuple([p.transform(x)for p in self.pipes]), axis=1)
 def _get_param_names(self, *args, **kwargs):
return ['pipes']
def set_params(self, *args, **kwargs):
[p.set_params(*args, **kwargs)for p in self.pipes]
return None 
<compute_test_metric>
improved_cols = [
'question_conversational',
'question_has_commonly_accepted_answer',
'question_interestingness_self',
'question_not_really_a_question',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'answer_plausible',
'answer_relevance',
]
bestcutting = [
[],
[],
[0.07143090909002797, 0.12004401497150083, 0.1848436328636153, 0.393906661243123],
[],
[0.5079822811938757, 0.6784007255713459, 0.8821817039873604, 0.9433617221024944],
[0.3371636921591598, 0.5581504204597405, 0.7077497707969119, 0.7828705361913083],
[],
[0.4956358187787945, 0.5224040768530395, 0.5255965284962879, 0.5793678554651578, 0.609287211890339, 0.616010882506188, 0.6196306504341735, 0.6676136621794896],
[],
[0.006889458941723597, 0.3260944763731216, 0.41788196092069185, 0.8651878898710491],
[],
[0.11795989772505418, 0.24754185187571828, 0.4745327419425606, 0.7234122358154191],
[0.09410719980735521, 0.15601674060996373, 0.282369801654087, 0.5313848085789701],
[0.0004399460062622443, 0.03194983660092762, 0.07003471591529131, 0.1503817577814426],
[0.14375842154645047, 0.2616577902050483, 0.2625648734101294, 0.396704338334258],
[0.09127758816465084, 0.17097449682745675, 0.4003532557591638, 0.9950764536726562],
[0.16265486023189857, 0.32208644678446063, 0.5417455427354888, 0.7751835127308364],
[],
[],
[],
[],
[],
[],
[0.05277303723757072, 0.08296633934736848, 0.2892567828157349, 0.8733971791156417, 0.9508973759039053, 0.9536983669289478, 0.9693328391611057, 0.9815554213789836],
[0.675877531406042, 0.7222527113500082, 0.8276284368386688, 0.9215102556176764, 0.939746161126185, 0.9528639928103086, 0.9671258882221032, 0.978652006112756],
[0.06008624214559094, 0.08468818485434321, 0.17139508646282006, 0.42753923139658206, 0.47748050195355696, 0.5020362112065385, 0.6191407747833618, 0.6221662654090617, 0.7453504487470863, 0.8057253638493509, 0.8163920374283427, 0.8472975110653058, 0.8670318719402241, 0.8748592375242844, 0.9165127791415058, 0.9982222527210189],
[],
[],
[],
[]
]
assert np.all(np.isfinite(test_train[numeric_cols].values))<train_on_grid>
def postprocess(pred):
df = pd.read_csv('.. /input/google-quest-challenge/train.csv')
 for i, col in enumerate(TARGET_COLS):
if not col in improved_cols:
continue
labels = np.sort(df[col].unique())
 pred[: , i] = pd.cut(pred[:, i], [-np.inf] + bestcutting[i] + [np.inf], labels=labels)
 return pred
param_grid = {"estimator__C": np.linspace(0.24285-0.1, 0.24285+0.1, num=6)
 }
grid_search = GridSearchCV(pipeline_logreg, param_grid=param_grid,
scoring='roc_auc',
n_jobs=1,
verbose=1,
cv=3 )<prepare_x_and_y>
def predict_bert(net_trained, test_dl):
net_trained.eval()
net_trained.cuda()
preds = []
for batch in test_dl:
token_ids = batch[0].cuda()
seg_ids = batch[1].cuda()
masks = batch[2].cuda()
with torch.set_grad_enabled(False):
outputs = net_trained(token_ids, token_type_ids=seg_ids, attention_mask=masks)
 preds.append(torch.sigmoid(outputs ).cpu())
 return torch.cat(preds, 0 ).numpy()
def predict_xlnet(net_trained, test_dl):
net_trained.eval()
net_trained.cuda()
preds = []
for batch in test_dl:
token_ids = batch[0].cuda()
seg_ids = batch[1].cuda()
masks = batch[2].cuda()
cls_idx = batch[3].cuda()
with torch.set_grad_enabled(False):
outputs = net_trained(token_ids, token_type_ids=seg_ids, attention_mask=masks, cls_index=cls_idx)
 preds.append(torch.sigmoid(outputs ).cpu())
 return torch.cat(preds, 0 ).numpy()
n = 25000
subset_A = test_train.loc[lambda df:(df.project_is_approved == 1)].sample(n)
 subset_B = test_train.loc[lambda df:(df.project_is_approved == 0)].sample(n)
 test_train_subset = pd.concat(( subset_A, subset_B))
test_X = test_train_subset
test_y = test_X.project_is_approved<train_on_grid>
predictions = []
perform_grid_search = True
if perform_grid_search:
with warnings.catch_warnings() :
warnings.simplefilter("ignore", category=UserWarning)
 grid_search.fit(test_X, test_y.values.ravel())
 best_estimator = grid_search.best_estimator_
print('Best score:', grid_search.best_score_)
 print('Best params:
', grid_search.best_params_)
 else:
best_estimator = pipeline_logreg<train_model>
%%time
for i in range(10):
net = build_model(model='bert')
 net.load_state_dict(torch.load(f'.. /input/google-quest-bert-base-train/bert_fold{i+1}.pth'))
test_dl = get_test_loader(model='bert')
 predictions.append(predict_bert(net, test_dl))
for i in range(10):
net = build_model(model='xlnet')
 net.load_state_dict(torch.load(f'.. /input/google-quest-xlnet-train/xlnet_fold{i+1}.pth'))
test_dl = get_test_loader(model='xlnet')
 predictions.append(predict_xlnet(net, test_dl))
full_train = test_train.loc[lambda df: df.source == 'train']
y_pred = best_estimator.fit(full_train, full_train.project_is_approved.ravel() )<predict_on_test>
df = pd.read_csv('.. /input/google-quest-challenge/test.csv')
 n = df['url'].apply(lambda x:(( 'ell.stackexchange.com' in x)or('english.stackexchange.com' in x)) ).tolist()
spelling = []
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.)
<save_to_csv><EOS> 
sub[TARGET_COLS] = postprocess(np.mean(predictions, axis=0))
sub['question_type_spelling'] = spelling
sub.to_csv('submission.csv', index=False)
 sub.head()