Kaggle competition: Leaf Classification
<SOS> metric: MulticlassLoss Kaggle data source: leaf-classification<save_to_csv>
%matplotlib inline
test_predictions = [histories[i].test_predictions for i in range(len(histories)) ]
test_predictions = [np.average(test_predictions[i], axis=0)for i in range(len(test_predictions)) ]
test_predictions = np.mean(test_predictions, axis=0)
 df_sub.iloc[:, 1:] = test_predictions
df_sub.to_csv('submission.csv', index=False )<load_from_csv>
train_df = pd.read_csv('.. /input/train.csv')
 train_df.info()
copy_df = train_df.copy()
test_df = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv' )<define_variables>
species = train_df['species'].unique()
species.sort()
spe_dict = dict(enumerate(species))
inv_spe_dict = {v: k for k,v in spe_dict.items() }
train_df['species_index'] = train_df['species'].map(inv_spe_dict ).astype(int)
 train_df = train_df.drop(['species','id'],axis = 1 )
NUM_FOLDS = 5
DEVICE = 'cuda'
BATCH_SIZE = 8
BERT_BASE_MODEL_PATH = '/kaggle/input/huggingface-model-configs/bert-model-uncased-config.pkl'
ROBERTA_BASE_MODEL_PATH = '/kaggle/input/huggingface-model-configs/roberta-model-base-config.pkl'
XLNET_BASE_MODEL_PATH = '/kaggle/input/huggingface-model-configs/xlnet-model-base-cased-config.pkl'
BERT_Q_BASE_PATH = '/kaggle/input/e121-bert-question/'
BERT_A_BASE_PATH = '/kaggle/input/e125-bert-answer/'
ROBERTA_Q_BASE_PATH = '/kaggle/input/e126-roberta-question/'
ROBERTA_A_BASE_PATH = '/kaggle/input/e127-roberta-answer/'
XLNET_Q_BASE_PATH = '/kaggle/input/e128-xlnet-question/'
XLNET_A_BASE_PATH = '/kaggle/input/e129-xlnet-answer/'<define_variables>
train_data = train_df.values
Q_LABEL_COL = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
]
A_LABEL_COL = [
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]<install_modules>
a = np.random.permutation(train_data)
 b = np.random.permutation(train_data)
 train_data = np.vstack(( a,b))
train_data.shape
!pip install /kaggle/input/sacremoses-master/sacremoses > /dev/null
!pip install --no-deps /kaggle/input/guchio-transformers/*.whl<import_modules>
X,y = train_data[:,:-1],train_data[:,-1]
scaler = StandardScaler().fit(X)
 X = scaler.transform(X )
import transformers
from transformers import BertModel, BertTokenizer, RobertaModel, RobertaTokenizer, XLNetModel, XLNetTokenizer
import random
from math import ceil, floor
import pandas as pd
import torch
from torch import nn
from torch.utils.data import Dataset
from tqdm.notebook import tqdm
import pickle
import gc<train_model>
model = MLPClassifier(hidden_layer_sizes=(150,),activation='logistic',solver='lbfgs',alpha=0.001
,max_iter=200,early_stopping=True,validation_fraction=0.2,
learning_rate='adaptive',tol=1e-8,random_state=1 ).fit(X,y )
OSprint('start training!' )<create_dataframe>
def cv(a,b,model):
cv_X = train_data[a:b,:-1]
cv_X = scaler.transform(cv_X)
 cv_y = train_data[a:b,-1]
cv_p1_y = model.predict(cv_X)
 cv_p2_y = model.predict_log_proba(cv_X)
 print(accuracy_score(cv_y,cv_p1_y))
print(cv_y)
 print(cv_p1_y)
class QUESTDataset(Dataset):
def __init__(self, df, mode, tokens, augment,
tokenizer_type, pretrained_model_name_or_path, do_lower_case,
LABEL_COL, t_max_len, q_max_len, a_max_len, tqa_mode,
TBSEP, pos_id_type, MAX_SEQUENCE_LENGTH=None,
use_category=True, logger=None):
self.mode = mode
self.augment = augment
self.len = len(df)
 self.t_max_len = t_max_len
self.q_max_len = q_max_len
self.a_max_len = a_max_len
self.tqa_mode = tqa_mode
self.TBSEP = TBSEP
self.pos_id_type = pos_id_type
if MAX_SEQUENCE_LENGTH:
self.MAX_SEQUENCE_LENGTH = MAX_SEQUENCE_LENGTH
else:
raise NotImplementedError
self.use_category = use_category
self.logger = logger
self.cat_dict = {
'CAT_TECHNOLOGY'.casefold() : 0,
'CAT_STACKOVERFLOW'.casefold() : 1,
'CAT_CULTURE'.casefold() : 2,
'CAT_SCIENCE'.casefold() : 3,
'CAT_LIFE_ARTS'.casefold() : 4,
}
if mode == "test":
self.labels = pd.DataFrame([[-1] * len(LABEL_COL)] * len(df))
else:
self.labels = df[LABEL_COL]
self.tokenizer_type = tokenizer_type
if tokenizer_type == 'bert':
tokenizer = BertTokenizer
elif tokenizer_type == 'roberta':
tokenizer = RobertaTokenizer
elif tokenizer_type == 'xlnet':
tokenizer = XLNetTokenizer
else:
raise NotImplementedError
self.tokenizer = tokenizer.from_pretrained(
 pretrained_model_name_or_path, do_lower_case=do_lower_case)
 self.tokenizer.add_tokens([self.TBSEP])
 tokens = [token.encode('ascii', 'replace' ).decode()
for token in tokens if token != '']
added_num = self.tokenizer.add_tokens(tokens)
 if logger:
logger.info(f'additional_tokens : {added_num}')
 else:
print(f'additional_tokens : {added_num}')
 self.original_df = df
def __len__(self):
return self.len
def __getitem__(self, idx):
idx_row = self.original_df.iloc[idx].copy()
idx_row = self.__preprocess_text_row(idx_row,
t_max_len=self.t_max_len,
q_max_len=self.q_max_len,
a_max_len=self.a_max_len)
 input_ids = idx_row['input_ids'].squeeze()
if self.tokenizer_type == 'roberta':
token_type_ids = torch.zeros(self.MAX_SEQUENCE_LENGTH, dtype=torch.long)
 else:
token_type_ids = idx_row['token_type_ids'].squeeze()
attention_mask = idx_row['attention_mask'].squeeze()
qa_id = idx_row['qa_id'].squeeze()
if self.pos_id_type == 'arange':
position_ids = torch.arange(self.MAX_SEQUENCE_LENGTH)
 else:
raise NotImplementedError
labels = self.labels.iloc[idx].values
return qa_id, input_ids, attention_mask, \
token_type_ids, position_ids, labels
def _trim_input(self, title, question, answer,
t_max_len, q_max_len, a_max_len):
t_len = len(title)
 q_len = len(question)
 a_len = len(answer)
 if(t_len + q_len + a_len + 4)> self.MAX_SEQUENCE_LENGTH:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/ 2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/ 2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len + a_new_len + q_new_len + 4 != self.MAX_SEQUENCE_LENGTH:
raise ValueError("New sequence length should be %d, but is %d"
%(self.MAX_SEQUENCE_LENGTH, 
(t_new_len + a_new_len + q_new_len + 4)))
 if len(title)> t_new_len:
title = title[:t_new_len // 2] + title[-t_new_len // 2:]
else:
title = title[:t_new_len]
if len(question)> q_new_len:
question = question[:q_new_len // 2] + \
question[-q_new_len // 2:]
else:
question = question[:q_new_len]
if len(answer)> a_new_len:
answer = answer[:a_new_len // 2] + answer[-a_new_len // 2:]
else:
answer = answer[:a_new_len]
return title, question, answer
def __preprocess_text_row(self, row, t_max_len, q_max_len, a_max_len):
qa_id = row.qa_id
title = self.tokenizer.tokenize(row.question_title)
 body = self.tokenizer.tokenize(row.question_body)
 answer = self.tokenizer.tokenize(row.answer)
 category =('CAT_' + row.category ).casefold()
if self.use_category:
title = [category] + title
title, body, answer = self._trim_input(title, body, answer,
t_max_len=t_max_len,
q_max_len=q_max_len,
a_max_len=a_max_len)
 if len(title)== 0:
title = ['_']
if len(body)== 0:
body = ['_']
if len(answer)== 0:
answer = ['_']
if self.tqa_mode == 'tq_a':
text = title + [self.TBSEP] + body
text_pair = answer
elif self.tqa_mode == 't_q':
text = title
text_pair = body
elif self.tqa_mode == 't_a':
text = title
text_pair = answer
encoded_texts_dict = self.tokenizer.encode_plus(
 text=text,
text_pair=text_pair,
add_special_tokens=True,
max_length=self.MAX_SEQUENCE_LENGTH,
pad_to_max_length=True,
return_tensors='pt',
return_token_type_ids=True,
return_attention_mask=True,
return_overflowing_tokens=True, 
)
 encoded_texts_dict['qa_id'] = qa_id
return encoded_texts_dict
class BertModelForBinaryMultiLabelClassifier(nn.Module):
def __init__(self, num_labels, config_path, state_dict,
token_size=None, MAX_SEQUENCE_LENGTH=512):
super(BertModelForBinaryMultiLabelClassifier, self ).__init__()
with open(config_path, 'rb')as fin:
config = pickle.load(fin)
 self.model = BertModel(config)
 self.dropout = nn.Dropout(0.2)
 self.classifier = nn.Linear(self.model.config.hidden_size, num_labels)
 if token_size:
self.model.resize_token_embeddings(token_size)
 self.add_module('my_fc_output', self.classifier)
 def forward(self, input_ids=None, input_cats=None, labels=None, attention_mask=None,
token_type_ids=None, position_ids=None, head_mask=None,
inputs_embeds=None, encoder_hidden_states=None,
encoder_attention_mask=None):
outputs = self.model(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds,
encoder_hidden_states=encoder_hidden_states,
encoder_attention_mask=encoder_attention_mask)
 pooled_output = torch.mean(outputs[0], dim=1)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
return outputs
def resize_token_embeddings(self, token_num):
self.model.resize_token_embeddings(token_num)
 def freeze_unfreeze_bert(self, freeze=True, logger=None):
if freeze:
print('FREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = False
else:
print('UNFREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = True
class RobertaModelForBinaryMultiLabelClassifier(nn.Module):
def __init__(self, num_labels, config_path, state_dict,
token_size=None, MAX_SEQUENCE_LENGTH=512):
super(RobertaModelForBinaryMultiLabelClassifier, self ).__init__()
with open(config_path, 'rb')as fin:
config = pickle.load(fin)
 self.model = RobertaModel(config)
 self.dropout = nn.Dropout(0.2)
 self.classifier = nn.Linear(self.model.config.hidden_size, num_labels)
 if token_size:
self.model.resize_token_embeddings(token_size)
 self.add_module('my_fc_output', self.classifier)
 def forward(self, input_ids=None, input_cats=None, labels=None, attention_mask=None,
token_type_ids=None, position_ids=None, head_mask=None,
inputs_embeds=None, encoder_hidden_states=None,
encoder_attention_mask=None):
outputs = self.model(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds,
encoder_hidden_states=encoder_hidden_states,
encoder_attention_mask=encoder_attention_mask)
 pooled_output = torch.mean(outputs[0], dim=1)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
return outputs
def resize_token_embeddings(self, token_num):
self.model.resize_token_embeddings(token_num)
 def freeze_unfreeze_bert(self, freeze=True, logger=None):
if freeze:
print('FREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = False
else:
print('UNFREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = True
class XLNetModelForBinaryMultiLabelClassifier(nn.Module):
def __init__(self, num_labels, config_path, state_dict,
token_size=None, MAX_SEQUENCE_LENGTH=512):
super(XLNetModelForBinaryMultiLabelClassifier, self ).__init__()
with open(config_path, 'rb')as fin:
config = pickle.load(fin)
 self.model = XLNetModel(config)
 self.dropout = nn.Dropout(0.2)
 self.classifier = nn.Linear(self.model.config.d_model, num_labels)
 if token_size:
self.model.resize_token_embeddings(token_size)
 self.add_module('my_fc_output', self.classifier)
 def forward(self, input_ids=None, input_cats=None, labels=None, attention_mask=None,
token_type_ids=None, position_ids=None, head_mask=None,
inputs_embeds=None, encoder_hidden_states=None,
encoder_attention_mask=None):
outputs = self.model(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 pooled_output = torch.mean(outputs[0], dim=1)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
return outputs
def resize_token_embeddings(self, token_num):
self.model.resize_token_embeddings(token_num)
 def freeze_unfreeze_bert(self, freeze=True, logger=None):
if freeze:
print('FREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = False
else:
print('UNFREEZE bert model !', logger)
 for name, child in self.model.named_children() :
for param in child.parameters() :
param.requires_grad = True
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(
 col_trues,
col_pred +
np.random.normal(
 0,
1e-7,
col_pred.shape[0])).correlation)
 return rhos
def train_one_epoch(model, fobj, optimizer, loader, DEVICE):
model.train()
running_loss = 0
for(qa_id, input_ids, attention_mask,
token_type_ids, position_ids, labels)in tqdm(loader):
input_ids = input_ids.to(DEVICE)
 attention_mask = attention_mask.to(DEVICE)
 token_type_ids = token_type_ids.to(DEVICE)
 position_ids = position_ids.to(DEVICE)
 labels = labels.to(DEVICE)
 outputs = model(
 input_ids=input_ids,
labels=labels,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids 
)
 loss = fobj(outputs[0], labels.float())
 optimizer.zero_grad()
loss.backward()
optimizer.step()
running_loss += loss
loss_mean = running_loss / len(loader)
 return loss_mean
def test(model, fobj, loader, DEVICE, mode):
model.eval()
with torch.no_grad() :
y_preds, y_trues, qa_ids = [], [], []
running_loss = 0
for(qa_id, input_ids, attention_mask,
token_type_ids, position_ids, labels)in tqdm(loader):
input_ids = input_ids.to(DEVICE)
 attention_mask = attention_mask.to(DEVICE)
 token_type_ids = token_type_ids.to(DEVICE)
 position_ids = position_ids.to(DEVICE)
 labels = labels.to(DEVICE)
 outputs = model(
 input_ids=input_ids,
labels=labels,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids 
)
 logits = outputs[0]
if mode != 'test':
loss = fobj(logits, labels.float())
 running_loss += loss
y_preds.append(torch.sigmoid(logits))
y_trues.append(labels)
 qa_ids.append(qa_id)
 loss_mean = running_loss / len(loader)
 y_preds = torch.cat(y_preds ).to('cpu' ).numpy()
y_trues = torch.cat(y_trues ).to('cpu' ).numpy()
qa_ids = torch.cat(qa_ids ).to('cpu' ).numpy()
if mode == 'valid':
metric_raws = compute_spearmanr(y_trues, y_preds)
 metric = np.mean(metric_raws)
 elif mode != 'test':
raise NotImplementedError
else:
metric_raws = None
metric = None
return loss_mean, metric, metric_raws, y_preds, y_trues, qa_ids
def seed_everything(seed=71):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
d = np.random.permutation(train_df.values)
 Xp = d[:,:-1]
Xp = scaler.transform(Xp)
 yp = d[:,-1]
ypp = model.predict(Xp)
 print(accuracy_score(yp,ypp))
FOLD_NUM = 5
RANK_NUM = 1
state_dict = {}
bert_question_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{BERT_Q_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = BertModelForBinaryMultiLabelClassifier(21, BERT_BASE_MODEL_PATH, state_dict, token_size=30528,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='bert',
pretrained_model_name_or_path=BERT_Q_BASE_PATH,
do_lower_case=True,
LABEL_COL=Q_LABEL_COL,
t_max_len=30,
q_max_len=239 * 2,
a_max_len=239 * 0,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in bert_question_fold_prediction_dict:
bert_question_fold_prediction_dict[fold] += fold_df
else:
bert_question_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
bert_question_fold_prediction_dict[fold] = bert_question_fold_prediction_dict[fold] / RANK_NUM
FOLD_NUM = 5
RANK_NUM = 1
bert_answer_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{BERT_A_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = BertModelForBinaryMultiLabelClassifier(9, BERT_BASE_MODEL_PATH, state_dict, token_size=30528,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='bert',
pretrained_model_name_or_path=BERT_A_BASE_PATH,
do_lower_case=True,
LABEL_COL=A_LABEL_COL,
t_max_len=30,
q_max_len=239 * 0,
a_max_len=239 * 2,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in bert_answer_fold_prediction_dict:
bert_answer_fold_prediction_dict[fold] += fold_df
else:
bert_answer_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
bert_answer_fold_prediction_dict[fold] = bert_answer_fold_prediction_dict[fold] / RANK_NUM
bert_fold_prediction_dict = {}
for fold in bert_question_fold_prediction_dict:
bert_fold_prediction_dict[fold] = bert_question_fold_prediction_dict[fold].reset_index().merge(
 bert_answer_fold_prediction_dict[fold].reset_index() , on='qa_id', how='left' )<load_pretrained>
test_df = pd.read_csv('.. /input/test.csv')
 index = test_df.pop('id')
 test_data = test_df.values
test_X = test_data
test_X = scaler.transform(test_X )
FOLD_NUM = 5
RANK_NUM = 2
state_dict = {}
roberta_question_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{ROBERTA_Q_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = RobertaModelForBinaryMultiLabelClassifier(21, ROBERTA_BASE_MODEL_PATH, state_dict, token_size=50271,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='roberta',
pretrained_model_name_or_path=ROBERTA_Q_BASE_PATH,
do_lower_case=False,
LABEL_COL=Q_LABEL_COL,
t_max_len=30,
q_max_len=239 * 2,
a_max_len=239 * 0,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in roberta_question_fold_prediction_dict:
roberta_question_fold_prediction_dict[fold] += fold_df
else:
roberta_question_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
roberta_question_fold_prediction_dict[fold] = roberta_question_fold_prediction_dict[fold] / RANK_NUM
FOLD_NUM = 5
RANK_NUM = 2
roberta_answer_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{ROBERTA_A_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = RobertaModelForBinaryMultiLabelClassifier(9, ROBERTA_BASE_MODEL_PATH, state_dict, token_size=50271,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='roberta',
pretrained_model_name_or_path=ROBERTA_A_BASE_PATH,
do_lower_case=False,
LABEL_COL=A_LABEL_COL,
t_max_len=30,
q_max_len=239 * 0,
a_max_len=239 * 2,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in roberta_answer_fold_prediction_dict:
roberta_answer_fold_prediction_dict[fold] += fold_df
else:
roberta_answer_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
roberta_answer_fold_prediction_dict[fold] = roberta_answer_fold_prediction_dict[fold] / RANK_NUM
roberta_fold_prediction_dict = {}
for fold in roberta_question_fold_prediction_dict:
roberta_fold_prediction_dict[fold] = roberta_question_fold_prediction_dict[fold].reset_index().merge(
 roberta_answer_fold_prediction_dict[fold].reset_index() , on='qa_id', how='left' )<load_pretrained>
predict = model.predict(test_X)
 predict_proba = model.predict_proba(test_X)
 predict_proba
FOLD_NUM = 5
RANK_NUM = 1
state_dict = {}
xlnet_question_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{XLNET_Q_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = XLNetModelForBinaryMultiLabelClassifier(21, XLNET_BASE_MODEL_PATH, state_dict, token_size=32006,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='xlnet',
pretrained_model_name_or_path=XLNET_Q_BASE_PATH,
do_lower_case=False,
LABEL_COL=Q_LABEL_COL,
t_max_len=30,
q_max_len=239 * 2,
a_max_len=239 * 0,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in xlnet_question_fold_prediction_dict:
xlnet_question_fold_prediction_dict[fold] += fold_df
else:
xlnet_question_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
xlnet_question_fold_prediction_dict[fold] = xlnet_question_fold_prediction_dict[fold] / RANK_NUM
FOLD_NUM = 5
RANK_NUM = 1
xlnet_answer_fold_prediction_dict = {}
for fold in range(FOLD_NUM):
for rank in range(RANK_NUM):
OSprint(f'fold -- {fold}')
 STATE_DICT_PATH = f'{XLNET_A_BASE_PATH}/state_dicts/fold_{fold}_rank_{rank}_state_dict.pkl'
with open(STATE_DICT_PATH, 'rb')as fin:
del state_dict
state_dict = pickle.load(fin)
 gc.collect()
model = XLNetModelForBinaryMultiLabelClassifier(9, XLNET_BASE_MODEL_PATH, state_dict, token_size=32006,)
 model.load_state_dict(state_dict)
 model.to(DEVICE)
 test_dataset = QUESTDataset(
 df=test_df,
mode='test',
tokens = [
'CAT_TECHNOLOGY'.casefold() ,
'CAT_STACKOVERFLOW'.casefold() ,
'CAT_CULTURE'.casefold() ,
'CAT_SCIENCE'.casefold() ,
'CAT_LIFE_ARTS'.casefold() ,
],
augment=[],
tokenizer_type='xlnet',
pretrained_model_name_or_path=XLNET_A_BASE_PATH,
do_lower_case=False,
LABEL_COL=A_LABEL_COL,
t_max_len=30,
q_max_len=239 * 0,
a_max_len=239 * 2,
tqa_mode='tq_a',
TBSEP='[TBSEP]',
pos_id_type='arange',
MAX_SEQUENCE_LENGTH=512, 
)
 test_sampler = RandomSampler(data_source=test_dataset)
 test_loader = DataLoader(
 test_dataset,
batch_size=BATCH_SIZE,
sampler=test_sampler,
num_workers=os.cpu_count() ,
worker_init_fn=lambda x: np.random.seed() ,
drop_last=False,
pin_memory=True 
)
 _, _, _, y_preds, _, qa_ids = test(model, None, test_loader, DEVICE, 'test')
 fold_df = pd.DataFrame(y_preds)
 fold_df['qa_id'] = qa_ids
fold_df = fold_df.set_index('qa_id')
 if fold in xlnet_answer_fold_prediction_dict:
xlnet_answer_fold_prediction_dict[fold] += fold_df
else:
xlnet_answer_fold_prediction_dict[fold] = fold_df
del model, test_dataset, test_sampler, test_loader
gc.collect()
xlnet_answer_fold_prediction_dict[fold] = xlnet_answer_fold_prediction_dict[fold] / RANK_NUM
xlnet_fold_prediction_dict = {}
for fold in xlnet_question_fold_prediction_dict:
xlnet_fold_prediction_dict[fold] = xlnet_question_fold_prediction_dict[fold].reset_index().merge(
 xlnet_answer_fold_prediction_dict[fold].reset_index() , on='qa_id', how='left' )<feature_engineering>
species_list = species.tolist()
result = pd.DataFrame(predict_proba,index = index, columns = species_list)
 result
fold_prediction_dict = {}
dicts = [bert_fold_prediction_dict, roberta_fold_prediction_dict, xlnet_fold_prediction_dict]
for _fold_prediction_dict in dicts:
for fold in _fold_prediction_dict:
if fold in fold_prediction_dict:
fold_prediction_dict[fold] += _fold_prediction_dict[fold].sort_values('qa_id' ).reset_index(drop=True)
 else:
fold_prediction_dict[fold] = _fold_prediction_dict[fold].sort_values('qa_id' ).reset_index(drop=True)
 for fold in fold_prediction_dict:
fold_prediction_dict[fold] /= len(dicts )<categorify>
fp = open('submission.csv','w')
 fp.write(result.to_csv() )