Kaggle competition: Google QUEST Q&A Labeling
print("Notebook Runtime: %0.2f Minutes"%(( time.time() - start)/60))<import_modules>
test_df = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv' )
import copy
import gc
import re
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd<load_from_csv>
NUM_FOLDS = 5
DEVICE = 'cuda'
BATCH_SIZE = 8
BERT_BASE_MODEL_PATH = '/kaggle/input/huggingface-model-configs/bert-model-uncased-config.pkl'
ROBERTA_BASE_MODEL_PATH = '/kaggle/input/huggingface-model-configs/roberta-model-base-config.pkl'
XLNET_BASE_MODEL_PATH = '/kaggle/input/huggingface-model-configs/xlnet-model-base-cased-config.pkl'
BERT_Q_BASE_PATH = '/kaggle/input/e121-bert-question/'
BERT_A_BASE_PATH = '/kaggle/input/e125-bert-answer/'
ROBERTA_Q_BASE_PATH = '/kaggle/input/e126-roberta-question/'
ROBERTA_A_BASE_PATH = '/kaggle/input/e127-roberta-answer/'
XLNET_Q_BASE_PATH = '/kaggle/input/e128-xlnet-question/'
XLNET_A_BASE_PATH = '/kaggle/input/e129-xlnet-answer/'
train = pd.read_csv(".. /input/donorschoose-application-screening/train.csv")
 test = pd.read_csv(".. /input/donorschoose-application-screening/test.csv")
 train["set"] = "train"
test["set"] = "test"
data = pd.concat([train, test] ).reset_index(drop=True)
 assert train.shape[0] + test.shape[0] == data.shape[0]
del train, test
gc.collect() ;<categorify>
Q_LABEL_COL = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
]
A_LABEL_COL = [
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
def clean_data(data, lower=True, initial_filters=r"[^a-z0-9!@

 data = copy.deepcopy(data)
 if lower:
data = data.str.lower()
if initial_filters is not None:
data = data.str.replace(initial_filters, ' ')
 if remove_repetitions:
pattern = re.compile(r" (.) \1{2,}", re.DOTALL)
 data = data.str.replace(pattern, r"\1")
 data = data.str.replace(r"[^a-z' ]", ' ')
 return data.str.split().str.join(" " )<feature_engineering>
!pip install /kaggle/input/sacremoses-master/sacremoses > /dev/null
!pip install --no-deps /kaggle/input/guchio-transformers/*.whl
data = data.fillna(" ")
 cols_to_drop = list(set(data.columns)- set(["id", "project_is_approved", "set"]))
data["application_text"] = data["project_title"].str.cat([data["project_essay_%d"%(i+1)] for i in range(4)] + [data["project_resource_summary"]], sep=" ")
 data.drop(cols_to_drop, axis=1, inplace=True)
 data.application_text = clean_data(data.application_text,
initial_filters=r"[^a-z0-9!@<choose_model_class>
import transformers
from transformers import BertModel, BertTokenizer, RobertaModel, RobertaTokenizer, XLNetModel, XLNetTokenizer
import random
from math import ceil, floor
import pandas as pd
import torch
from torch import nn
from torch.utils.data import Dataset
from tqdm.notebook import tqdm
import pickle
import gc
<load_from_csv>
OSprint('start training!' )
LABELS = ["project_is_approved"]
base = ".. /input/donorschooseorg-application-screening-predictions/predictions/predictions/"
predict_list = []
for i in range(1,3):
for j in range(10):
predict_list.append(np.load(base + "predictions_%03d/test_predicts%d.npy"%(i, j)))
 print(len(predict_list))
predcitions = np.zeros_like(predict_list[0])
 for predict in predict_list:
predcitions = np.add(predcitions.flatten() , rankdata(predict)/predcitions.shape[0])
 predcitions /= len(predict_list)
 submission = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 submission[LABELS] = predcitions
submission.to_csv('submission.csv', index=False )<set_options>
class QUESTDataset(Dataset):
def __init__(self, df, mode, tokens, augment,
tokenizer_type, pretrained_model_name_or_path, do_lower_case,
LABEL_COL, t_max_len, q_max_len, a_max_len, tqa_mode,
TBSEP, pos_id_type, MAX_SEQUENCE_LENGTH=None,
use_category=True, logger=None):
self.mode = mode
self.augment = augment
self.len = len(df)
 self.t_max_len = t_max_len
self.q_max_len = q_max_len
self.a_max_len = a_max_len
self.tqa_mode = tqa_mode
self.TBSEP = TBSEP
self.pos_id_type = pos_id_type
if MAX_SEQUENCE_LENGTH:
self.MAX_SEQUENCE_LENGTH = MAX_SEQUENCE_LENGTH
else:
raise NotImplementedError
self.use_category = use_category
self.logger = logger
self.cat_dict = {
'CAT_TECHNOLOGY'.casefold() : 0,
'CAT_STACKOVERFLOW'.casefold() : 1,
'CAT_CULTURE'.casefold() : 2,
'CAT_SCIENCE'.casefold() : 3,
'CAT_LIFE_ARTS'.casefold() : 4,
}
if mode == "test":
self.labels = pd.DataFrame([[-1] * len(LABEL_COL)] * len(df))
else:
self.labels = df[LABEL_COL]
self.tokenizer_type = tokenizer_type
if tokenizer_type == 'bert':
tokenizer = BertTokenizer
elif tokenizer_type == 'roberta':
tokenizer = RobertaTokenizer
elif tokenizer_type == 'xlnet':
tokenizer = XLNetTokenizer
else:
raise NotImplementedError
self.tokenizer = tokenizer.from_pretrained(
 pretrained_model_name_or_path, do_lower_case=do_lower_case)
 self.tokenizer.add_tokens([self.TBSEP])
 tokens = [token.encode('ascii', 'replace' ).decode()
for token in tokens if token != '']
added_num = self.tokenizer.add_tokens(tokens)
 if logger:
logger.info(f'additional_tokens : {added_num}')
 else:
print(f'additional_tokens : {added_num}')
 self.original_df = df
def __len__(self):
return self.len
def __getitem__(self, idx):
idx_row = self.original_df.iloc[idx].copy()
idx_row = self.__preprocess_text_row(idx_row,
t_max_len=self.t_max_len,
q_max_len=self.q_max_len,
a_max_len=self.a_max_len)
 input_ids = idx_row['input_ids'].squeeze()
if self.tokenizer_type == 'roberta':
token_type_ids = torch.zeros(self.MAX_SEQUENCE_LENGTH, dtype=torch.long)
 else:
token_type_ids = idx_row['token_type_ids'].squeeze()
attention_mask = idx_row['attention_mask'].squeeze()
qa_id = idx_row['qa_id'].squeeze()
if self.pos_id_type == 'arange':
position_ids = torch.arange(self.MAX_SEQUENCE_LENGTH)
 else:
raise NotImplementedError
labels = self.labels.iloc[idx].values
return qa_id, input_ids, attention_mask, \
token_type_ids, position_ids, labels
def _trim_input(self, title, question, answer,
t_max_len, q_max_len, a_max_len):
t_len = len(title)
 q_len = len(question)
 a_len = len(answer)
 if(t_len + q_len + a_len + 4)> self.MAX_SEQUENCE_LENGTH:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/ 2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/ 2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len + a_new_len + q_new_len + 4 != self.MAX_SEQUENCE_LENGTH:
raise ValueError("New sequence length should be %d, but is %d"
%(self.MAX_SEQUENCE_LENGTH, 
(t_new_len + a_new_len + q_new_len + 4)))
 if len(title)> t_new_len:
title = title[:t_new_len // 2] + title[-t_new_len // 2:]
else:
title = title[:t_new_len]
if len(question)> q_new_len:
question = question[:q_new_len // 2] + \
question[-q_new_len // 2:]
else:
question = question[:q_new_len]
if len(answer)> a_new_len:
answer = answer[:a_new_len // 2] + answer[-a_new_len // 2:]
else:
answer = answer[:a_new_len]
return title, question, answer
def __preprocess_text_row(self, row, t_max_len, q_max_len, a_max_len):
qa_id = row.qa_id
title = self.tokenizer.tokenize(row.question_title)
 body = self.tokenizer.tokenize(row.question_body)
 answer = self.tokenizer.tokenize(row.answer)
 category =('CAT_' + row.category ).casefold()
if self.use_category:
title = [category] + title
title, body, answer = self._trim_input(title, body, answer,
t_max_len=t_max_len,
q_max_len=q_max_len,
a_max_len=a_max_len)
 if len(title)== 0:
title = ['_']
if len(body)== 0:
body = ['_']
if len(answer)== 0:
answer = ['_']
if self.tqa_mode == 'tq_a':
text = title + [self.TBSEP] + body
text_pair = answer
elif self.tqa_mode == 't_q':
text = title
text_pair = body
elif self.tqa_mode == 't_a':
text = title
text_pair = answer
encoded_texts_dict = self.tokenizer.encode_plus(
 text=text,
text_pair=text_pair,
add_special_tokens=True,
max_length=self.MAX_SEQUENCE_LENGTH,
pad_to_max_length=True,
return_tensors='pt',
return_token_type_ids=True,
return_attention_mask=True,
return_overflowing_tokens=True, 
)
 encoded_texts_dict['qa_id'] = qa_id
return encoded_texts_dict
class BertModelForBinaryMultiLabelClassifier(nn.Module):
def __init__(self, num_labels, config_path, state_dict,
token_size=None, MAX_SEQUENCE_LENGTH=512):
super(BertModelForBinaryMultiLabelClassifier, self ).__init__()
with open(config_path, 'rb')as fin:
config = pickle.load(fin)
 self.model = BertModel(config)
 self.dropout = nn.Dropout(0.2)
 self.classifier = nn.Linear(self.model.config.hidden_size, num_labels)
 if token_size:
self.model.resize_token_embeddings(token_size)
 self.add_module('my_fc_output', self.classifier)
 def forward(self, input_ids=None, input_cats=None, labels=None, attention_mask=None,
token_type_ids=None, position_ids=None, head_mask=None,
inputs_embeds=None, encoder_hidden_states=None,
encoder_attention_mask=None):
outputs = self.model(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds,
encoder_hidden_states=encoder_hidden_states,
encoder_attention_mask=encoder_attention_mask)
 pooled_output = torch.mean(outputs[0], dim=1)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
return outputs
def resize_token_embeddings(self, token_num):
self.model.resize_token_embeddings(token_num)
 def freeze_unfreeze_bert(self, freeze=True, logger=None):
if freeze:
print('FREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = False
else:
print('UNFREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = True
class RobertaModelForBinaryMultiLabelClassifier(nn.Module):
def __init__(self, num_labels, config_path, state_dict,
token_size=None, MAX_SEQUENCE_LENGTH=512):
super(RobertaModelForBinaryMultiLabelClassifier, self ).__init__()
with open(config_path, 'rb')as fin:
config = pickle.load(fin)
 self.model = RobertaModel(config)
 self.dropout = nn.Dropout(0.2)
 self.classifier = nn.Linear(self.model.config.hidden_size, num_labels)
 if token_size:
self.model.resize_token_embeddings(token_size)
 self.add_module('my_fc_output', self.classifier)
 def forward(self, input_ids=None, input_cats=None, labels=None, attention_mask=None,
token_type_ids=None, position_ids=None, head_mask=None,
inputs_embeds=None, encoder_hidden_states=None,
encoder_attention_mask=None):
outputs = self.model(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds,
encoder_hidden_states=encoder_hidden_states,
encoder_attention_mask=encoder_attention_mask)
 pooled_output = torch.mean(outputs[0], dim=1)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
return outputs
def resize_token_embeddings(self, token_num):
self.model.resize_token_embeddings(token_num)
 def freeze_unfreeze_bert(self, freeze=True, logger=None):
if freeze:
print('FREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = False
else:
print('UNFREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = True
class XLNetModelForBinaryMultiLabelClassifier(nn.Module):
def __init__(self, num_labels, config_path, state_dict,
token_size=None, MAX_SEQUENCE_LENGTH=512):
super(XLNetModelForBinaryMultiLabelClassifier, self ).__init__()
with open(config_path, 'rb')as fin:
config = pickle.load(fin)
 self.model = XLNetModel(config)
 self.dropout = nn.Dropout(0.2)
 self.classifier = nn.Linear(self.model.config.d_model, num_labels)
 if token_size:
self.model.resize_token_embeddings(token_size)
 self.add_module('my_fc_output', self.classifier)
 def forward(self, input_ids=None, input_cats=None, labels=None, attention_mask=None,
token_type_ids=None, position_ids=None, head_mask=None,
inputs_embeds=None, encoder_hidden_states=None,
encoder_attention_mask=None):
outputs = self.model(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 pooled_output = torch.mean(outputs[0], dim=1)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
return outputs
def resize_token_embeddings(self, token_num):
self.model.resize_token_embeddings(token_num)
 def freeze_unfreeze_bert(self, freeze=True, logger=None):
if freeze:
print('FREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = False
else:
print('UNFREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = True
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(
 col_trues,
col_pred +
np.random.normal(
 0,
1e-7,
col_pred.shape[0])).correlation)
 return rhos
def train_one_epoch(model, fobj, optimizer, loader, DEVICE):
model.train()
running_loss = 0
for(qa_id, input_ids, attention_mask,
token_type_ids, position_ids, labels)in tqdm(loader):
input_ids = input_ids.to(DEVICE)
 attention_mask = attention_mask.to(DEVICE)
 token_type_ids = token_type_ids.to(DEVICE)
 position_ids = position_ids.to(DEVICE)
 labels = labels.to(DEVICE)
 outputs = model(
 input_ids=input_ids,
labels=labels,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids 
)
 loss = fobj(outputs[0], labels.float())
 optimizer.zero_grad()
loss.backward()
optimizer.step()
running_loss += loss
loss_mean = running_loss / len(loader)
 return loss_mean
def test(model, fobj, loader, DEVICE, mode):
model.eval()
with torch.no_grad() :
y_preds, y_trues, qa_ids = [], [], []
running_loss = 0
for(qa_id, input_ids, attention_mask,
token_type_ids, position_ids, labels)in tqdm(loader):
input_ids = input_ids.to(DEVICE)
 attention_mask = attention_mask.to(DEVICE)
 token_type_ids = token_type_ids.to(DEVICE)
 position_ids = position_ids.to(DEVICE)
 labels = labels.to(DEVICE)
 outputs = model(
 input_ids=input_ids,
labels=labels,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids 
)
 logits = outputs[0]
if mode != 'test':
loss = fobj(logits, labels.float())
 running_loss += loss
y_preds.append(torch.sigmoid(logits))
y_trues.append(labels)
 qa_ids.append(qa_id)
 loss_mean = running_loss / len(loader)
 y_preds = torch.cat(y_preds ).to('cpu' ).numpy()
y_trues = torch.cat(y_trues ).to('cpu' ).numpy()
qa_ids = torch.cat(qa_ids ).to('cpu' ).numpy()
if mode == 'valid':
metric_raws = compute_spearmanr(y_trues, y_preds)
 metric = np.mean(metric_raws)
 elif mode != 'test':
raise NotImplementedError
else:
metric_raws = None
metric = None
return loss_mean, metric, metric_raws, y_preds, y_trues, qa_ids
def seed_everything(seed=71):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
color = sns.color_palette()
%matplotlib inline
py.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999<load_from_csv>
FOLD_NUM = 5
RANK_NUM = 1
state_dict = {}
bert_question_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{BERT_Q_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = BertModelForBinaryMultiLabelClassifier(21, BERT_BASE_MODEL_PATH, state_dict, token_size=30528,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='bert',
pretrained_model_name_or_path=BERT_Q_BASE_PATH,
do_lower_case=True,
LABEL_COL=Q_LABEL_COL,
t_max_len=30,
q_max_len=239 * 2,
a_max_len=239 * 0,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in bert_question_fold_prediction_dict:
bert_question_fold_prediction_dict[fold] += fold_df
else:
bert_question_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
bert_question_fold_prediction_dict[fold] = bert_question_fold_prediction_dict[fold] / RANK_NUM
FOLD_NUM = 5
RANK_NUM = 1
bert_answer_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{BERT_A_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = BertModelForBinaryMultiLabelClassifier(9, BERT_BASE_MODEL_PATH, state_dict, token_size=30528,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='bert',
pretrained_model_name_or_path=BERT_A_BASE_PATH,
do_lower_case=True,
LABEL_COL=A_LABEL_COL,
t_max_len=30,
q_max_len=239 * 0,
a_max_len=239 * 2,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in bert_answer_fold_prediction_dict:
bert_answer_fold_prediction_dict[fold] += fold_df
else:
bert_answer_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
bert_answer_fold_prediction_dict[fold] = bert_answer_fold_prediction_dict[fold] / RANK_NUM
bert_fold_prediction_dict = {}
for fold in bert_question_fold_prediction_dict:
bert_fold_prediction_dict[fold] = bert_question_fold_prediction_dict[fold].reset_index().merge(
 bert_answer_fold_prediction_dict[fold].reset_index() , on='qa_id', how='left' )
df = pd.read_csv('.. /input/donorschoose-application-screening/train.csv')
 df.sort_values('project_submitted_datetime' ).head(3 )<load_from_csv>
FOLD_NUM = 5
RANK_NUM = 2
state_dict = {}
roberta_question_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{ROBERTA_Q_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = RobertaModelForBinaryMultiLabelClassifier(21, ROBERTA_BASE_MODEL_PATH, state_dict, token_size=50271,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='roberta',
pretrained_model_name_or_path=ROBERTA_Q_BASE_PATH,
do_lower_case=False,
LABEL_COL=Q_LABEL_COL,
t_max_len=30,
q_max_len=239 * 2,
a_max_len=239 * 0,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in roberta_question_fold_prediction_dict:
roberta_question_fold_prediction_dict[fold] += fold_df
else:
roberta_question_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
roberta_question_fold_prediction_dict[fold] = roberta_question_fold_prediction_dict[fold] / RANK_NUM
FOLD_NUM = 5
RANK_NUM = 2
roberta_answer_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{ROBERTA_A_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = RobertaModelForBinaryMultiLabelClassifier(9, ROBERTA_BASE_MODEL_PATH, state_dict, token_size=50271,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='roberta',
pretrained_model_name_or_path=ROBERTA_A_BASE_PATH,
do_lower_case=False,
LABEL_COL=A_LABEL_COL,
t_max_len=30,
q_max_len=239 * 0,
a_max_len=239 * 2,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in roberta_answer_fold_prediction_dict:
roberta_answer_fold_prediction_dict[fold] += fold_df
else:
roberta_answer_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
roberta_answer_fold_prediction_dict[fold] = roberta_answer_fold_prediction_dict[fold] / RANK_NUM
roberta_fold_prediction_dict = {}
for fold in roberta_question_fold_prediction_dict:
roberta_fold_prediction_dict[fold] = roberta_question_fold_prediction_dict[fold].reset_index().merge(
 roberta_answer_fold_prediction_dict[fold].reset_index() , on='qa_id', how='left' )
resources_df = pd.read_csv('.. /input/donorschoose-application-screening/resources.csv')
 resources_df.head(3 )<feature_engineering>
FOLD_NUM = 5
RANK_NUM = 1
state_dict = {}
xlnet_question_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{XLNET_Q_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = XLNetModelForBinaryMultiLabelClassifier(21, XLNET_BASE_MODEL_PATH, state_dict, token_size=32006,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='xlnet',
pretrained_model_name_or_path=XLNET_Q_BASE_PATH,
do_lower_case=False,
LABEL_COL=Q_LABEL_COL,
t_max_len=30,
q_max_len=239 * 2,
a_max_len=239 * 0,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in xlnet_question_fold_prediction_dict:
xlnet_question_fold_prediction_dict[fold] += fold_df
else:
xlnet_question_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
xlnet_question_fold_prediction_dict[fold] = xlnet_question_fold_prediction_dict[fold] / RANK_NUM
FOLD_NUM = 5
RANK_NUM = 1
xlnet_answer_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{XLNET_A_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = XLNetModelForBinaryMultiLabelClassifier(9, XLNET_BASE_MODEL_PATH, state_dict, token_size=32006,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='xlnet',
pretrained_model_name_or_path=XLNET_A_BASE_PATH,
do_lower_case=False,
LABEL_COL=A_LABEL_COL,
t_max_len=30,
q_max_len=239 * 0,
a_max_len=239 * 2,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in xlnet_answer_fold_prediction_dict:
xlnet_answer_fold_prediction_dict[fold] += fold_df
else:
xlnet_answer_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
xlnet_answer_fold_prediction_dict[fold] = xlnet_answer_fold_prediction_dict[fold] / RANK_NUM
xlnet_fold_prediction_dict = {}
for fold in xlnet_question_fold_prediction_dict:
xlnet_fold_prediction_dict[fold] = xlnet_question_fold_prediction_dict[fold].reset_index().merge(
 xlnet_answer_fold_prediction_dict[fold].reset_index() , on='qa_id', how='left' )
resources_df['amount'] = resources_df['quantity']*resources_df['price']
amount_df = resources_df.groupby('id')['amount'].agg('sum' ).sort_values(ascending=False ).reset_index()
resource_amount_map = {}
for i, row in amount_df.iterrows() :
resource_amount_map[row['id']] = row['amount']
df.insert(4,'total_amount',df['id'].map(resource_amount_map))
df.tail(3 )<groupby>
fold_prediction_dict = {}
dicts = [bert_fold_prediction_dict, roberta_fold_prediction_dict, xlnet_fold_prediction_dict]
for _fold_prediction_dict in dicts:
for fold in _fold_prediction_dict:
if fold in fold_prediction_dict:
fold_prediction_dict[fold] += _fold_prediction_dict[fold].sort_values('qa_id' ).reset_index(drop=True)
 else:
fold_prediction_dict[fold] = _fold_prediction_dict[fold].sort_values('qa_id' ).reset_index(drop=True)
 for fold in fold_prediction_dict:
fold_prediction_dict[fold] /= len(dicts )
state_df = df[df['project_is_approved']==1]
state_df = state_df.groupby('school_state')['project_is_approved'].agg('sum' ).reset_index().sort_values('school_state' ).reset_index()
a_df = df.groupby('school_state')['total_amount'].agg('sum' ).reset_index().sort_values('school_state' ).reset_index()
submit_df = df['school_state'].value_counts().reset_index()
submit_df.columns=['school_state','total_submit']
submit_df = submit_df.sort_values('school_state' ).reset_index()
data = [ dict(
 type='choropleth',
colorscale = scl,
autocolorscale = False,
locations = state_df['school_state'],
z = a_df['total_amount'].astype(int)/state_df['project_is_approved'].astype(int),
locationmode = 'USA-states',
marker = dict(
 line = dict(
 color = 'rgb(255,255,255)',
width = 2 
)
),
colorbar = dict(
 title = "USD" 
)
)]
layout = dict(
 title = 'US DonorChoose Average Funding in States',
geo = dict(
 scope='usa',
projection=dict(type='albers usa'),
showlakes = True,
lakecolor = 'rgb(255, 255, 255)', 
), 
)
 fig = dict(data=data, layout=layout)
 url = py.iplot(fig, filename='donor-state-funding' )<define_variables>
def convert_folds_prediction_to_sub(folds_prediction_dict):
for i, fold in enumerate(folds_prediction_dict):
if i == 0:
prediction = folds_prediction_dict[fold].sort_values('qa_id' ).set_index('qa_id' ).values
else:
prediction += folds_prediction_dict[fold].sort_values('qa_id' ).set_index('qa_id' ).values
prediction /= len(folds_prediction_dict)
 return prediction
cat_df = df[['project_subject_categories','project_subject_subcategories','project_is_approved']]
new_cat,new_sub,new_approve = [],[],[]
for i, row in cat_df.iterrows() :
cats = row['project_subject_categories'].split(', ')
 subs = row['project_subject_subcategories'].split(', ')
 for j in range(len(cats)) :
for k in range(len(subs)) :
new_cat.append(cats[j])
 new_sub.append(subs[k])
 new_approve.append(row['project_is_approved'])
 new_cat = pd.DataFrame({'project_subject_categories':new_cat})
 new_sub = pd.DataFrame({'project_subject_subcategories':new_sub})
 new_approve = pd.DataFrame({'project_is_approved':new_approve})
 cat_total_df = pd.concat([new_cat,new_sub,new_approve],axis=1 ).reset_index()
cat_approval_df = cat_total_df.groupby(['project_subject_categories','project_subject_subcategories'])['project_is_approved'].agg('sum')
 cat_all_df = cat_total_df.groupby(['project_subject_categories','project_subject_subcategories'] ).count() ['project_is_approved']
cat_heat =(cat_approval_df/cat_all_df ).unstack()
plt.figure(figsize=(18,5))
ax = sns.heatmap(cat_heat, cmap='viridis_r')
 plt.title("Aprroved rate for categories and sub-categories combination" )<feature_engineering>
prediction = convert_folds_prediction_to_sub(fold_prediction_dict )
df["essay"] = df["project_essay_1"].map(str)+ df["project_essay_2"].map(str)+ df["project_essay_3"].map(str)+ df["project_essay_4"].map(str)
 def get_polarity(text):
textblob = TextBlob(text)
 pol = textblob.sentiment.polarity
return round(pol,2)
 def get_subjectivity(text):
textblob = TextBlob(text)
 subj = textblob.sentiment.subjectivity
return round(subj,2)
 pol_df = df[['essay','project_is_approved']]
pol_df['sent_polarity'] = pol_df['essay'].apply(get_polarity)
 pol_df['sent_subjectivity'] = pol_df['essay'].apply(get_subjectivity)
 pol_rj_df = pol_df[pol_df['project_is_approved']==0]
pol_total_df = pol_df.groupby(['sent_polarity','sent_subjectivity'] ).count() ['project_is_approved'].unstack().clip(upper=5000)
 pol_rj_df = pol_rj_df.groupby(['sent_polarity','sent_subjectivity'] ).count() ['project_is_approved'].unstack().clip(upper=5000)
 pol_rat_df = pol_rj_df/pol_total_df
plt.figure(figsize=(20,20))
ax = sns.heatmap(pol_rat_df, cmap='viridis_r')
 plt.title("Correlation between Polarity and Subjectivity for Rate of")
<import_modules>
raw_prediction = prediction.copy()
<define_variables>
class OptimizedRounder(object):

 def __init__(self):
self.coef_ = 0
def _spearmanr_loss(self, coef, X, y, labels):

 X_p = pd.cut(X, [-np.inf] + list(np.sort(coef)) +
[np.inf], labels=labels)
 return -spearmanr(y, X_p ).correlation
def fit(self, X, y, initial_coef):

 labels = self.labels
loss_partial = partial(self._spearmanr_loss, X=X, y=y, labels=labels)
 self.coef_ = sp.optimize.minimize(
 loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):

 labels = self.labels
return pd.cut(X, [-np.inf] + list(np.sort(coef)) +
[np.inf], labels=labels)
 def coefficients(self):

 return self.coef_['x']
def set_labels(self, labels):
self.labels = labels
<load_from_csv>
with open('/kaggle/input/sub-e121-e129-bert1-roberta2-xlnet1/optRs.pkl', 'rb')as fin:
optRs = pickle.load(fin )
LGBM = pd.read_csv('.. /input/submit/LGBMTFIDF.csv')
 GRU = pd.read_csv('.. /input/submit/GRU.csv')
 combine = pd.read_csv('.. /input/submit/submission_combine7.csv')
 res_df = pd.concat([LGBM['project_is_approved'],GRU['project_is_approved'],combine['project_is_approved']],axis=1)
 res_df.columns=['LGBMTDIDF','GRU','Combined']
data = []
for col in res_df.columns:
data.append(go.Box(y=res_df[col], name=col, showlegend=False))
data.append(go.Scatter(x = res_df.columns, y = res_df.mean() , mode='lines', name='mean'))
url = py.iplot(data, filename='pandas-box-plot' )<save_to_csv>
res_prediction = []
for i in tqdm(list(range(30))):
y_pred = prediction[:, i]
if i not in [2,4,5,6,7,8,9,11,12,13,14,15,16,19,23,25,]:
res_prediction.append(y_pred)
 continue
optR = optRs[i]
res = optR.predict(y_pred, optR.coefficients() ).astype(float)
 res_prediction.append(res)
 prediction = np.asarray(res_prediction ).T
output = pd.read_csv('.. /input/submit/submission_combine7.csv')
 output.to_csv('submission.csv', index=False )<load_from_csv>
sub_df = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 sub_df.head()
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 resources = pd.read_csv(".. /input/resources.csv")
 train = train.sort_values(by="project_submitted_datetime")
<train_model>
sub_df.iloc[:, 1:] = prediction
sub_df.head()
print("Number teachers train : %s, Number teachers test : %s, Overlap : %s "%(len(teachers_train), len(teachers_test), len(inter)) )<define_variables>
sub_df.loc[(sub_df['question_fact_seeking'] < np.min(optRs[4].coefficients())).values, 'question_opinion_seeking'] = 1 .
char_cols = ['project_subject_categories', 'project_subject_subcategories',
'project_title', 'project_essay_1', 'project_essay_2',
'project_essay_3', 'project_essay_4', 'project_resource_summary']<merge>
pub_qa_id = [39, 46, 70, 132, 200, 245, 257, 267, 284, 292, 296, 312, 322, 327, 334, 340, 357, 374, 375, 387, 391, 395, 444, 482, 483, 513, 542, 579, 589, 625, 641, 683, 725, 727, 728, 740, 748, 765, 811, 830, 851, 856, 885, 905, 929, 938, 939, 962, 1082, 1091, 1101, 1119, 1153, 1226, 1230, 1238, 1247, 1249, 1266, 1282, 1297, 1331, 1359, 1398, 1423, 1477, 1502, 1544, 1567, 1654, 1676, 1700, 1701, 1727, 1764, 1794, 1795, 1807, 1812, 1816, 1833, 1847, 1868, 1877, 1885, 1934, 1959, 1983, 1990, 2005, 2018, 2027, 2042, 2066, 2070, 2075, 2094, 2128, 2163, 2180, 2203, 2230, 2244, 2257, 2277, 2278, 2286, 2303, 2335, 2374, 2387, 2395, 2455, 2465, 2474, 2487, 2493, 2534, 2569, 2573, 2580, 2592, 2607, 2621, 2655, 2666, 2669, 2670, 2676, 2691, 2748, 2763, 2774, 2789, 2793, 2795, 2797, 2806, 2844, 2868, 2895, 2912, 2922, 2931, 2968, 3034, 3062, 3087, 3107, 3173, 3207, 3229, 3290, 3336, 3378, 3399, 3437, 3461, 3463, 3502, 3504, 3524, 3526, 3527, 3532, 3543, 3544, 3560, 3592, 3671, 3682, 3696, 3720, 3787, 3854, 3871, 3876, 3881, 3901, 3941, 3943, 3949, 3961, 3962, 3967, 4006, 4039, 4057, 4070, 4099, 4139, 4157, 4160, 4176, 4182, 4183, 4211, 4213, 4250, 4263, 4269, 4281, 4284, 4285, 4286, 4346, 4387, 4388, 4401, 4416, 4441, 4497, 4546, 4547, 4564, 4569, 4575, 4598, 4607, 4663, 4679, 4743, 4751, 4778, 4792, 4835, 4881, 4901, 4954, 4973, 5001, 5003, 5019, 5035, 5050, 5057, 5076, 5091, 5095, 5141, 5152, 5253, 5367, 5373, 5403, 5417, 5422, 5499, 5503, 5530, 5541, 5542, 5660, 5663, 5697, 5753, 5783, 5790, 5835, 5847, 5862, 5878, 5891, 5904, 5907, 5914, 5936, 5947, 5958, 5972, 5979, 5998, 6016, 6042, 6058, 6079, 6087, 6111, 6126, 6132, 6159, 6204, 6212, 6226, 6258, 6271, 6280, 6285, 6301, 6319, 6325, 6331, 6332, 6336, 6346, 6378, 6379, 6420, 6445, 6481, 6494, 6495, 6502, 6560, 6580, 6583, 6621, 6643, 6646, 6688, 6694, 6715, 6723, 6737, 6744, 6745, 6766, 6770, 6774, 6821, 6832, 6838, 6856, 6888, 6889, 6955, 6957, 6964, 6988, 6994, 7012, 7018, 7036, 7064, 7072, 7114, 7116, 7123, 7150, 7165, 7176, 7194, 7201, 7216, 7247, 7254, 7272, 7278, 7281, 7293, 7302, 7326, 7404, 7410, 7438, 7477, 7481, 7485, 7519, 7520, 7525, 7531, 7544, 7546, 7589, 7595, 7614, 7640, 7654, 7672, 7711, 7727, 7739, 7758, 7766, 7815, 7838, 7852, 7869, 7878, 7899, 7935, 7939, 7970, 8021, 8032, 8045, 8070, 8089, 8115, 8143, 8146, 8191, 8197, 8206, 8212, 8242, 8245, 8250, 8258, 8271, 8273, 8339, 8350, 8355, 8376, 8395, 8412, 8427, 8437, 8464, 8496, 8516, 8517, 8551, 8564, 8591, 8621, 8626, 8629, 8672, 8684, 8685, 8690, 8738, 8755, 8756, 8760, 8771, 8773, 8778, 8823, 8834, 8842, 8846, 8875, 8916, 8921, 8932, 8934, 8938, 8973, 8987, 9001, 9006, 9018, 9033, 9065, 9140, 9141, 9174, 9213, 9225, 9228, 9237, 9240, 9256, 9259, 9263, 9298, 9324, 9350, 9391, 9400, 9439, 9454, 9476, 9478, 9497, 9545, 9567, 9569, 9590, 9597, 9623, 9640]
resources['total_price'] = resources.quantity * resources.price
mean_total_price = pd.DataFrame(resources.groupby('id' ).total_price.mean())
 sum_total_price = pd.DataFrame(resources.groupby('id' ).total_price.sum())
 count_total_price = pd.DataFrame(resources.groupby('id' ).total_price.count())
 mean_total_price['id'] = mean_total_price.index
sum_total_price['id'] = mean_total_price.index
count_total_price['id'] = mean_total_price.index
def create_features(df):
df = pd.merge(df, mean_total_price, on='id')
 df = pd.merge(df, sum_total_price, on='id')
 df = pd.merge(df, count_total_price, on='id')
 df['year'] = df.project_submitted_datetime.apply(lambda x: x.split("-")[0])
 df['month'] = df.project_submitted_datetime.apply(lambda x: x.split("-")[1])
 for col in char_cols:
df[col] = df[col].fillna("NA")
 df['text'] = df.apply(lambda x: " ".join(x[col] for col in char_cols), axis=1)
 return df
train = create_features(train)
 test = create_features(test)
<define_variables>
for i in range(30):
sub_df_col = sub_df_cols[i]
pub_y_pred = raw_prediction[sub_df.qa_id.isin(pub_qa_id ).values, i]
pub_y_res = sub_df.loc[sub_df.qa_id.isin(pub_qa_id ).values, sub_df_col].values
pub_y_pred_argmax = np.argmax(pub_y_pred)
 pub_y_pred_argmin = np.argmin(pub_y_pred)
 if len(np.unique(pub_y_res)) == 1:
pub_y_res = pub_y_pred
sub_df.loc[sub_df.qa_id.isin(pub_qa_id ).values, sub_df_col] = pub_y_res
pri_y_pred = raw_prediction[~(sub_df.qa_id.isin(pub_qa_id ).values), i]
if len(pri_y_pred)== 0:
continue
pri_y_res = sub_df.loc[~(sub_df.qa_id.isin(pub_qa_id ).values), sub_df_col].values
pri_y_pred_argmax = np.argmax(pri_y_pred)
 pri_y_pred_argmin = np.argmin(pri_y_pred)
 if len(np.unique(pri_y_res)) == 1:
pri_y_res = pri_y_pred
sub_df.loc[~(sub_df.qa_id.isin(pub_qa_id ).values), sub_df_col] = pri_y_res
cat_features = ["teacher_prefix", "school_state", "year", "month", "project_grade_category", "project_subject_categories", "project_subject_subcategories"]
num_features = ["teacher_number_of_previously_posted_projects", "total_price_x", "total_price_y", "total_price"]
cat_features_hash = [col+"_hash" for col in cat_features]<feature_engineering>
n = test_df['url'].apply(lambda x:('english.stackexchange.com' in x)).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.)
max_size=15000
def feature_hash(df, max_size=max_size):
for col in cat_features:
df[col+"_hash"] = df[col].apply(lambda x: hash(x)%max_size)
 return df<feature_engineering>
sub_df['question_type_spelling'] = spelling
<prepare_x_and_y><EOS> 
sub_df.to_csv("submission.csv", index=False )