Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<import_modules>
np.set_printoptions(suppress=True)
 sys.path.append('./transformers')
 sys.path.append('./sacremoses')
 tf.keras.backend.clear_session()
print(tf.__version__ )
import matplotlib.pyplot as plt
import random<load_pretrained>
MAX_SEQUENCE_LENGTH = 512
train_dataloader = DataLoader(train_dataset, num_workers=multiprocessing.cpu_count() , shuffle=True, batch_size=2**6)
 val_dataloader = DataLoader(val_dataset, num_workers=multiprocessing.cpu_count() , shuffle=False, batch_size=8 )<import_modules>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/robertal/'
tokenizer = RobertaTokenizer(BERT_PATH + "roberta-large-vocab.json", BERT_PATH + "roberta-large-merges.txt")
 df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 output categories:
\t', output_categories)
 input categories:
\t', input_categories)
 output_categories_question =list(df_train.columns[11:32])
 output_categories_answer = list(df_train.columns[32:])
 input_categories_question = list(df_train.columns[[1,2]])
 input_categories_answer = list(df_train.columns[[1,5]])
 print('
output categories_question:
\t', output_categories_question)
 print('
output categories_answer:
\t', output_categories_answer)
 print('
input categories_question:
\t', input_categories_question)
 print('
input categories_answer:
\t', input_categories_answer )
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import pytorch_lightning as pl
from torchvision import models
import pytorch_lightning.callbacks as pl_callbacks<choose_model_class>
def _get_masks(tokens, max_seq_length):

 if len(tokens)> max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1] * len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [1] *(max_seq_length - len(token_ids))
return input_ids
def _trim_input(title, question, max_sequence_length,
t_max_len=100, head=128, tail=281, Q=True, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 t_len = len(t)
 q_len = len(q)
 if(t_len + q_len + 4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
q_head = head
q_tail = 508 - q_head - t_new_len
else:
t_new_len = t_max_len
if(t_new_len + q_len + 4)> max_sequence_length:
q_head = head
q_tail = 508 - q_head - t_new_len
else:
t = t[:t_new_len]
return t, q
t = t[:t_new_len]
q = q[:q_head] + q[-q_tail:]
return t, q
else:
return t, q
def _convert_to_bert_inputs(title, question, tokenizer, max_sequence_length):

 stoken = ["
"] + title + ["
"] + ["
"] + question + ["
"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 return [input_ids, input_masks]
def compute_input_arays(df, columns, tokenizer, max_sequence_length, Q=True):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
if Q:
t, q = instance.question_title, instance.question_body
t, q = _trim_input(t, q, max_sequence_length)
 ids, masks= _convert_to_bert_inputs(t, q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 else:
t, a = instance.question_title, instance.answer
t, a = _trim_input(t, a, max_sequence_length)
 ids, masks = _convert_to_bert_inputs(t, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns])
 def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
class ClassificationModel(pl.LightningModule):
def __init__(self, learning_rate, weights=None):
super(ClassificationModel, self ).__init__()
self.learning_rate = learning_rate
self.loss = nn.CrossEntropyLoss(weight=weights)
 pretrained_resnet = models.resnet18(pretrained=True)
 self.encoder = nn.Sequential(
 nn.Upsample(size=(224, 224), mode='bilinear', align_corners=True),
nn.Conv2d(in_channels=1, out_channels=3, kernel_size=1),
nn.BatchNorm2d(3, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=3, stride=1, padding=1, dilation=1, ceil_mode=False),
pretrained_resnet 
)
 self.decoder = nn.Sequential(
 nn.ReLU(inplace=True),
nn.Linear(in_features=1000, out_features=10)
)
 def configure_optimizers(self):
return optim.Adam(self.parameters() , lr=(self.learning_rate))
def forward(self, x):
x = self.encoder(x)
 x = self.decoder(x)
 return x
def training_step(self, batch, batch_idx):
x, y = batch
y_pred = self(x)
 loss = self.loss(y_pred, y)
 self.log("train_loss", loss, on_epoch=True,
prog_bar=True, logger=False)
 return loss
def validation_step(self, batch, batch_idx):
x, y = batch
y_pred = self(x)
 loss = self.loss(y_pred, y)
 self.log("val_loss", loss, on_epoch=True,
prog_bar=True, logger=False )<train_model>
input_columns_answer = ['question_title', 'answer']
input_columns_question = ['question_title', 'question_body']
find = re.compile(r"^[^.]*")
 df_train['netloc'] = df_train['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 df_test['netloc'] = df_test['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 features = ['netloc', 'category']
merged_answer = pd.concat([df_train[features], df_train[features]])
 merged_question = pd.concat([df_train[features], df_train[features]])
 ohe_answer = OneHotEncoder(handle_unknown='ignore')
 ohe_question = OneHotEncoder(handle_unknown='ignore')
 ohe_answer.fit(merged_answer)
 ohe_question.fit(merged_question)
 for i in range(len(df_test)) :
if df_test.loc[i,'netloc'] not in ohe_answer.categories_[0]:
df_test.loc[i,'netloc'] = 'stackoverflow'
if df_test.loc[i,'category'] not in ohe_answer.categories_[1]:
df_test.loc[i,'category'] = 'TECHNOLOGY'
features_test_answer = ohe_answer.transform(df_test[features] ).toarray()
features_test_question = ohe_question.transform(df_test[features] ).toarray()
module_url = "/kaggle/input/universalsentenceencoderlarge4/"
embed = hub.load(module_url)
 embeddings_test = {}
for text in input_columns_answer:
print(text)
 test_text = df_test[text].str.replace('?', '.' ).str.replace('!', '.' ).tolist()
curr_test_emb = []
batch_size = 16
ind = 0
while ind * batch_size < len(test_text):
curr_test_emb.append(embed(test_text[ind * batch_size:(ind + 1)* batch_size])["outputs"].numpy())
 ind += 1
embeddings_test[text + '_embedding'] = np.vstack(curr_test_emb)
 del embed
K.clear_session()
gc.collect()
l2_dist = lambda x, y: np.power(x - y, 2 ).sum(axis=1)
 cos_dist = lambda x, y:(x*y ).sum(axis=1)
 dist_features_test_answer = np.array([
l2_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
cos_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding'])
 ] ).T
feature_embeddings_answer = np.hstack([item for k, item in embeddings_test.items() ] + [features_test_answer, dist_features_test_answer])
 module_url = "/kaggle/input/universalsentenceencoderlarge4/"
embed = hub.load(module_url)
 embeddings_test = {}
for text in input_columns_question:
print(text)
 test_text = df_test[text].str.replace('?', '.' ).str.replace('!', '.' ).tolist()
curr_test_emb = []
batch_size = 16
ind = 0
while ind * batch_size < len(test_text):
curr_test_emb.append(embed(test_text[ind * batch_size:(ind + 1)* batch_size])["outputs"].numpy())
 ind += 1
embeddings_test[text + '_embedding'] = np.vstack(curr_test_emb)
 del embed
K.clear_session()
gc.collect()
l2_dist = lambda x, y: np.power(x - y, 2 ).sum(axis=1)
 cos_dist = lambda x, y:(x*y ).sum(axis=1)
 dist_features_test_question = np.array([
l2_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding']),
cos_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding'])
 ] ).T
feature_embeddings_question = np.hstack([item for k, item in embeddings_test.items() ] + [features_test_question, dist_features_test_question] )
trainer.fit(model, train_dataloader, val_dataloader )<compute_train_metric>
class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None, Q=True):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
self.Q = Q
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
if epoch >= 5:
if self.Q:
self.model.save_weights(f'checkpoints_question/roberta-base-question-{fold}-{epoch}.h5py')
 else:
self.model.save_weights(f'checkpoints_answer/roberta-base-answer-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model_question() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 feature_embeddings = tf.keras.layers.Input(
(feature_embeddings_question.shape[1],), dtype=tf.float32, name='feature_embeddings')
 bert_model = TFRobertaModel.from_pretrained(BERT_PATH, output_hidden_states=True)
 hidden_output = bert_model([input_word_ids, input_masks])[2]
last_cat_sequence = tf.concat(
(hidden_output[-1], hidden_output[-2], hidden_output[-3]),
2, 
)
 x = tf.keras.layers.GlobalAveragePooling1D()(last_cat_sequence)
 x = tf.concat(( x, feature_embeddings), 1)
 out = tf.keras.layers.Dense(21, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, feature_embeddings], outputs=out)
 return model
def bert_model_answer() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 feature_embeddings = tf.keras.layers.Input(
(feature_embeddings_answer.shape[1],), dtype=tf.float32, name='feature_embeddings')
 bert_model = TFRobertaModel.from_pretrained(BERT_PATH, output_hidden_states=True)
 hidden_output = bert_model([input_word_ids, input_masks])[2]
last_cat_sequence = tf.concat(
(hidden_output[-1], hidden_output[-2], hidden_output[-3]),
2, 
)
 x = tf.keras.layers.GlobalAveragePooling1D()(last_cat_sequence)
 x = tf.concat(( x, feature_embeddings), 1)
 out = tf.keras.layers.Dense(9, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, feature_embeddings], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold,Q=True):
if Q:
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=5,
Q=True)
 else:
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=5,
Q=False)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
train_accuracy = pl.metrics.Accuracy(compute_on_step=False)
 valid_accuracy = pl.metrics.Accuracy(compute_on_step=False)
 train_predictions, train_real = [], []
valid_predictions, valid_real = [], []
model.eval()
model.cuda()
for x, y in tqdm(train_dataloader):
with torch.no_grad() :
y_pred = model(x.cuda())
 train_accuracy(y_pred.cpu() , y)
 train_predictions +=(list(y_pred.argmax(dim=1 ).cpu()))
train_real += list(y)
 for x, y in tqdm(val_dataloader):
with torch.no_grad() :
y_pred = model(x.cuda())
 valid_accuracy(y_pred.cpu() , y)
 valid_predictions +=(list(y_pred.argmax(dim=1 ).cpu()))
valid_real += list(y)
 total_train_accuracy = train_accuracy.compute()
total_valid_accuracy = valid_accuracy.compute()<train_model>
test_inputs_question = compute_input_arays(df_test, input_categories_question, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs_question.append(feature_embeddings_question )
print("training accuracy", round(total_train_accuracy.item() * 100, 3), "%")
 print("validation accuracy", round(total_valid_accuracy.item() * 100, 3), "%" )<compute_test_metric>
test_inputs_answer = compute_input_arays(df_test, input_categories_answer, tokenizer, MAX_SEQUENCE_LENGTH,Q=False)
 test_inputs_answer.append(feature_embeddings_answer )
valid_matrix = confusion_matrix(valid_real, valid_predictions, labels=range(10))
train_matrix = confusion_matrix(train_real, train_predictions, labels=range(10))<categorify>
gc.collect()
fail_images = []
real_labels, fail_labels = [], []
for x, y in tqdm(val_dataloader):
with torch.no_grad() :
y_pred = model(x.cuda())
 y_pred = y_pred.argmax(dim=1 ).cpu()
mask =(y_pred == y ).numpy()
idxs = np.where(mask == False)[0]
if idxs.shape[0] > 0:
for idx in idxs:
fail_images.append(x[idx])
 real_labels.append(y[idx].item())
 fail_labels.append(y_pred[idx].item())
 fail_images = dataset.unormalize(torch.stack(fail_images, dim=0))<load_pretrained>
test_predictions_question = []
test_predictions_answer = []
for i in range(16):
if i == 0:
model_path = '.. /input/largequestion1/q1/roberta-large-features-question-0-3.h5py'
print(model_path)
 if i == 1:
model_path = '.. /input/largequestion1/q1/roberta-large-features-question-1-4.h5py'
print(model_path)
 if i == 2:
model_path = '.. /input/largequestion1/q1/roberta-large-features-question-2-4.h5py'
print(model_path)
 if i == 3:
model_path = '.. /input/largequestion2/q2/q2/roberta-large-features-question-3-3.h5py'
print(model_path)
 if i == 4:
model_path = '.. /input/largequestion2/q2/q2/roberta-large-features-question-4-4.h5py'
print(model_path)
 if i == 5:
model_path = '.. /input/largequestion2/q2/q2/roberta-large-features-question-5-4.h5py'
print(model_path)
 if i == 6:
model_path = '.. /input/largequestion3/q3/roberta-large-features-question-6-5.h5py'
print(model_path)
 if i == 7:
model_path = '.. /input/largequestion3/q3/roberta-large-features-question-7-3.h5py'
print(model_path)
 if i == 8:
model_path = '.. /input/largeanswer2/11/roberta-large-features-answer-2-3.h5py'
print(model_path)
 if i == 9:
model_path = '.. /input/largeanswer1/22/22/roberta-large-features-answer-0-4.h5py'
print(model_path)
 if i == 10:
model_path = '.. /input/largeanswer1/22/22/roberta-large-features-answer-1-3.h5py'
print(model_path)
 if i == 11:
model_path = '.. /input/largeanswer1/22/22/roberta-large-features-answer-6-3.h5py'
print(model_path)
 if i == 12:
model_path = '.. /input/largeanswer1/33/33/roberta-large-features-answer-7-2.h5py'
print(model_path)
 if i == 13:
model_path = '.. /input/largeanswer1/33/33/roberta-large-features-answer-5-2.h5py'
print(model_path)
 if i == 14:
model_path = '.. /input/largeanswer2/11/roberta-large-features-answer-4-5.h5py'
print(model_path)
 if i == 15:
model_path = '.. /input/largeanswer2/11/roberta-large-features-answer-3-3.h5py'
print(model_path)
 if i<8:
K.clear_session()
model = bert_model_question()
print('载入')
 model.load_weights(model_path)
 print("加载")
 test_predictions_question.append(model.predict(test_inputs_question, batch_size=30))
print(i)
 else:
K.clear_session()
model = bert_model_answer()
print('载入')
 model.load_weights(model_path)
 print('载入')
 test_predictions_answer.append(model.predict(test_inputs_answer, batch_size=30))
print(i)
model.to_torchscript("/kaggle/working/mnist_digit.torch.pt" )<create_dataframe>
final_predictions_question = np.mean(test_predictions_question, axis=0)
 final_predictions_answer = np.mean(test_predictions_answer, axis=0 )
submission = sample_submission.copy()<find_best_params>
gc.collect()
model.eval()
predicted_labels = []
for idx, img in tqdm(enumerate(test_dataset), total=len(test_dataset)) :
with torch.no_grad() :
y_predict = model(img.unsqueeze(0 ).cuda())
 predicted_labels.append(y_predict.argmax(dim=1 ).cpu().item() )<save_to_csv>
BERT_PATH = '.. /input/robertabase/'
tokenizer = RobertaTokenizer(BERT_PATH + "roberta-base-vocab.json", BERT_PATH + "roberta-base-merges.txt" )
submission["Label"] = predicted_labels
submission.to_csv('/kaggle/working/submission.csv',index=False )<load_from_disk>
test_predictions_question = []
test_predictions_answer = []
for i in range(16):
if i == 0:
model_path = '/kaggle/input/basequestion/rfbq11/roberta-base-features-question-0-4.h5py'
print(model_path)
 if i == 1:
model_path = '/kaggle/input/basequestion/rfbq11/roberta-base-features-question-1-4.h5py'
print(model_path)
 if i == 2:
model_path = '/kaggle/input/basequestion/rfbq11/roberta-base-features-question-2-6.h5py'
print(model_path)
 if i == 3:
model_path = '/kaggle/input/basequestion/rfbq11/roberta-base-features-question-3-3.h5py'
print(model_path)
 if i == 4:
model_path = '/kaggle/input/basequestion/rfbq11/roberta-base-features-question-4-4.h5py'
print(model_path)
 if i == 5:
model_path = '/kaggle/input/basequestion/rfbq11/roberta-base-features-question-5-4.h5py'
print(model_path)
 if i ==6:
model_path = '/kaggle/input/basequestion/rfbq11/roberta-base-features-question-6-4.h5py'
print(model_path)
 if i ==7:
model_path = '/kaggle/input/basequestion/rfbq11/roberta-base-features-question-7-3.h5py'
print(model_path)
 if i == 8:
model_path = '/kaggle/input/baseanswer/best11/roberta-base-features-answer-0-4.h5py'
print(model_path)
 if i == 9:
model_path = '/kaggle/input/baseanswer/best11/roberta-base-features-answer-1-2.h5py'
print(model_path)
 if i == 10:
model_path = '/kaggle/input/baseanswer/best11/roberta-base-features-answer-2-4.h5py'
print(model_path)
 if i == 11:
model_path = '/kaggle/input/baseanswer/best11/roberta-base-features-answer-3-4.h5py'
print(model_path)
 if i == 12:
model_path = '/kaggle/input/baseanswer/best11/roberta-base-features-answer-4-3.h5py'
print(model_path)
 if i == 13:
model_path = '/kaggle/input/baseanswer/best11/roberta-base-features-answer-5-3.h5py'
print(model_path)
 if i == 14:
model_path = '/kaggle/input/baseanswer/best11/roberta-base-features-answer-6-4.h5py'
print(model_path)
 if i == 15:
model_path = '/kaggle/input/baseanswer/best11/roberta-base-features-answer-7-3.h5py'
print(model_path)
 if i<8:
K.clear_session()
model = bert_model_question()
print('载入')
 model.load_weights(model_path)
 print("加载")
 test_predictions_question.append(model.predict(test_inputs_question, batch_size=60))
print(i)
 else:
K.clear_session()
model = bert_model_answer()
print('载入')
 model.load_weights(model_path)
 print('载入')
 test_predictions_answer.append(model.predict(test_inputs_answer, batch_size=60))
print(i )
json.dump(stats, open('/kaggle/working/stats.json', 'w'))<import_modules>
final_predictions_question_bert = np.mean(test_predictions_question, axis=0)
 final_predictions_answer_bert = np.mean(test_predictions_answer, axis=0 )
import os
import gc
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input, Dropout
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
final_predictions_q = 0.5*final_predictions_question + 0.5*final_predictions_question_bert
final_predictions_a = 0.5*final_predictions_answer + 0.5*final_predictions_answer_bert
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
df_sub.iloc[:, 1:20] = np.around(final_predictions_q[:,:19] *99)/ 99
df_sub.iloc[:, 21] = np.around(final_predictions_q[:,20] *99)/ 99
df_sub.iloc[:, 20] = np.around(final_predictions_q[:,19] *9000)/ 9000
df_sub.iloc[:, 22:] = np.around(final_predictions_a * 99)/ 99
pd.set_option('display.max_columns', None)
 df_sub.head(40 )
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<define_variables>
submission = df_sub
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 EPOCHS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
n = df_test['url'].apply(lambda x:(( 'ell.stackexchange.com' in x)or('english.stackexchange.com' in x)) ).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.)
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_pretrained>
submission.loc[submission['question_asker_intent_understanding'] >= 0.91, ['question_asker_intent_understanding']] = 1.0
submission.loc[(submission['question_asker_intent_understanding'] >=0.88)&(submission['question_asker_intent_understanding'] <0.91),
['question_asker_intent_understanding']] = 8/9
submission.loc[submission['question_body_critical'] <0.43, ['question_body_critical']] = 1/3
submission.loc[submission['question_body_critical'] >0.84, ['question_body_critical']] = 1.0
submission.loc[submission['question_conversational']<= 0.06 ,['question_conversational']] = 0.0
submission.loc[submission['question_expect_short_answer'] >0.72 ,['question_expect_short_answer']] = 1.0
submission.loc[submission['question_expect_short_answer'] <0.4, ['question_expect_short_answer']] = 0.0
submission.loc[(submission['question_expect_short_answer'] >=0.63)&(submission['question_expect_short_answer'] <0.69),
['question_expect_short_answer']] = 2/3
submission.loc[submission['question_fact_seeking']>0.81 ,['question_fact_seeking']] = 1.0
submission.loc[submission['question_has_commonly_accepted_answer']>0.83 ,['question_has_commonly_accepted_answer']] = 1.0
submission.loc[submission['question_has_commonly_accepted_answer'] <0.3, ['question_has_commonly_accepted_answer']] = 0.0
submission.loc[submission['question_interestingness_others']>0.76, ['question_interestingness_others']] = 1.0
submission.loc[submission['question_multi_intent'] >0.8, ['question_multi_intent']] = 1.0
submission.loc[submission['question_multi_intent'] <0.2, ['question_multi_intent']] = 0.0
submission.loc[submission['question_not_really_a_question'] <0.040, ['question_not_really_a_question']] =0.0
submission.loc[submission['question_not_really_a_question'] >=0.040, ['question_not_really_a_question']] =1/3
submission.loc[submission['question_opinion_seeking']<0.25, ['question_opinion_seeking']] = 0.0
submission.loc[(submission['question_opinion_seeking'] >=0.3)&(submission['question_opinion_seeking'] <0.36),
['question_opinion_seeking']] = 1/3
submission.loc[submission['question_opinion_seeking']>0.7, ['question_opinion_seeking']] = 1.0
submission.loc[submission['question_type_choice'] <0.25, ['question_type_choice']] = 0.0
submission.loc[submission['question_type_choice'] >0.86, ['question_type_choice']] = 1.0
submission.loc[submission['question_type_compare'] <0.2, ['question_type_compare']] =0.0
submission.loc[submission['question_type_consequence'] <0.3, ['question_type_consequence']] = 0.0
submission.loc[submission['question_type_definition']<0.2, ['question_type_definition']] = 0.0
submission.loc[submission['question_type_entity'] <0.188, ['question_type_entity']] =0.0
submission.loc[submission['question_type_instructions'] <0.33, ['question_type_instructions']] =0.0
submission.loc[submission['question_type_instructions'] >0.88, ['question_type_instructions']] = 1.0
submission.loc[submission['question_type_procedure'] <0.21, ['question_type_procedure']] =0.0
submission['question_type_spelling'] = spelling
submission.loc[submission['question_type_reason_explanation'] <0.16, ['question_type_reason_explanation']] =0.0
submission.loc[submission['question_type_reason_explanation'] >0.82, ['question_type_reason_explanation']] = 1.0
submission.loc[submission['question_well_written']>0.9, ['question_well_written']] = 1.0
submission.loc[submission['answer_helpful']>0.95, ['answer_helpful']] = 1.0
submission.loc[(submission['answer_level_of_information'] >=0.63)&(submission['answer_level_of_information'] <0.80),
['answer_level_of_information']] = 2/3
submission.loc[submission['answer_level_of_information']>0.80, ['answer_level_of_information']] = 1
submission.loc[submission['answer_plausible'] >0.96, ['answer_plausible']] = 1.0
submission.loc[submission['answer_relevance'] >0.97, ['answer_relevance']] = 1.0
submission.loc[submission['answer_satisfaction'] >0.92, ['answer_satisfaction']] = 1.0
submission.loc[submission['answer_type_instructions'] < 0.3, ['answer_type_instructions']] = 0
submission.loc[submission['answer_type_instructions'] > 0.85, ['answer_type_instructions']] = 1
submission.loc[submission['answer_type_procedure'] < 0.18, ['answer_type_procedure']] = 0.0
submission.loc[submission['answer_type_reason_explanation'] < 0.18, ['answer_type_reason_explanation']] = 0.0
submission.loc[submission['answer_type_reason_explanation'] >0.88, ['answer_type_reason_explanation']] = 1.0
submission.loc[submission['answer_well_written'] >0.93, ['answer_well_written']] = 1.0
submission.loc[(submission['answer_well_written'] >=0.87)&(submission['answer_well_written']<0.91),
['answer_well_written']]=8/9
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<load_from_csv>
submission.to_csv('submission.csv', index=False )
<train_model><EOS> 
submission.to_csv('submission.csv', index=False )