Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<import_modules>
%matplotlib inline
%reload_ext autoreload
%autoreload 2
class MaskedBinaryCrossentropy(tf.keras.losses.BinaryCrossentropy):
def __init__(self,
from_logits=False,
label_smoothing=0,
reduction=losses_utils.ReductionV2.AUTO,
name='binary_crossentropy'):

 super(MaskedBinaryCrossentropy, self ).__init__(
 name=name,
reduction=reduction,
from_logits=from_logits,
label_smoothing=label_smoothing)
 def call(self, y_true, y_pred):
if use_parcor:
y_true=tf.reshape(y_true,[-1,1])
 y_pred=tf.reshape(y_pred,[-1,1])
 y_pred=tf.squeeze(y_pred,axis =-1)
 y_true=tf.squeeze(y_true,axis =-1)
 mask=tf.math.greater_equal(y_true,0)
 mask=tf.cast(mask,y_true.dtype)
 masked_y_true=mask * y_true
l=super().call(masked_y_true,y_pred)
 l=mask*l
if use_parcor:
l=tf.reshape(l,[-1,2])
 l=l[:,0]+l[:,1]
return l<compute_test_metric>
import numpy as np
import pandas as pd
import os
from sklearn.metrics import accuracy_score, f1_score
from sklearn.metrics import precision_score, recall_score
from sklearn.metrics import roc_auc_score, roc_curve, auc
from sklearn.model_selection import train_test_split
import time
import torch
import transformers
from transformers import BertConfig
from transformers import get_linear_schedule_with_warmup
class SquareLoss(tf.keras.losses.Loss):
def __init__(self,reduction=losses_utils.ReductionV2.AUTO, name="squareLoss",lossWeight=0):
super().__init__(reduction=reduction,name=name)
 self.lossWeight=lossWeight
def call(self, y_true, y_pred):
is_same_masked= y_pred*y_true
loss=tf.math.scalar_mul(self.lossWeight,is_same_masked*is_same_masked)
 return loss<categorify>
sys.path.insert(1, '.. /input/tweet-classification-huggingface-wandb1/' )
def build_model(transformer, max_len=512):
if use_parcor:
pair_input_word_ids = Input(shape=(2,max_len,), dtype=tf.int32, name="input_word_ids")
 pair_attention_masks = Input(shape=(2,max_len,), dtype=tf.int32, name="attention_mask")
 input_word_ids = tf.keras.backend.reshape(pair_input_word_ids,shape=(-1,max_len))
attention_masks= tf.keras.backend.reshape(pair_attention_masks,shape=(-1,max_len))
inputs=[pair_input_word_ids,pair_attention_masks]
else:
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 attention_masks = Input(shape=(max_len,), dtype=tf.int32, name="attention_mask")
 inputs=[input_word_ids,attention_masks]
transformer_output = transformer(( input_word_ids,attention_masks))
sequence_output = transformer_output[0]
transformer_output
if pooling_mode == pooling_mode_cls_token:
transformed_features = sequence_output[:, 0, :]
cls_token=transformed_features
else:
float_mask=tf.cast(tf.expand_dims(attention_masks,axis=-1),tf.float32)
 lengths=tf.reduce_sum(float_mask,axis=-2)
 masked_vals=tf.multiply(sequence_output,float_mask)
 sums=tf.reduce_sum(masked_vals,axis=-2)
 avgs=tf.math.divide(sums,lengths)
 maxes=tf.math.reduce_max(masked_vals,axis=-2)
 transformed_features=tf.concat([avgs,maxes],axis=-1)
 cls_token=transformed_features
for dims in fc_dims:
cls_token=Dense(dims, activation='relu' )(cls_token)
 if use_dann:
lang_token=GradReverse()(transformed_features)
 for dims in fc_dims:
lang_token=Dense(dims, activation='relu' )(lang_token)
 lang=Dense(1, activation="sigmoid",name="lang" )(lang_token)
 if use_parcor:
toxic = Dense(1, activation="sigmoid",name="unpair_toxic" )(cls_token)
 toxic=tf.keras.backend.reshape(toxic,shape=(-1,2,1))
toxic=tf.keras.layers.Layer(name="toxic" )(toxic)
 is_same=tf.keras.layers.Subtract(name="is_same" )([toxic[:,1,:],toxic[:,0,:]])
 else:
toxic = Dense(1, activation="sigmoid",name="toxic" )(cls_token)
 out=[toxic]
if use_parcor:
out.append(is_same)
 celoss=MaskedBinaryCrossentropy(
 from_logits=False, label_smoothing=0, reduction=losses_utils.ReductionV2.AUTO,
name='crossentropy_loss' 
)
 loss={"toxic":celoss}
if use_dann:
domainGap=WeightedBinaryCrossentropy(
 from_logits=False, label_smoothing=0, reduction=losses_utils.ReductionV2.AUTO,
name='domaingap_loss',lossWeight=dann_lambda 
)
 out.append(lang)
 loss["lang"]=domainGap
if use_parcor:
loss["is_same"]=SquareLoss(lossWeight=parcor_lambda)
 model = Model(inputs=inputs, outputs=out)
 model.compile(Adam(lr=5e-6), loss=loss, metrics={"toxic":['accuracy',tf.keras.metrics.AUC(name="auc"),tf.keras.metrics.AUC(name="pr-auc",curve="PR")]})
 return model<categorify>
import config
import dataset
import model
import train
import util
def setTrainableLayers(transformer,embeddingsTrainable=False,poolerTrainable=False,hiddenTrainable=8):
transformer.layers[0].pooler.trainable=poolerTrainable
transformer.layers[0].embeddings.trainable=embeddingsTrainable
trainableHiddens=transformer.layers[0].encoder.layer[-1*hiddenTrainable:]
for layer in transformer.layers[0].encoder.layer:
if layer in trainableHiddens:
layer.trainable=True
else:
layer.trainable=False<load_pretrained>
device = 'cuda' if torch.cuda.is_available() else 'cpu'
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 if load_model is not None:
model.load_weights(load_model)
 model.summary()<define_variables>
warmup_epochs = 0
epochs = 5
model_name = 'bert-base-uncased'
test_size = 0.3
dropout = 0.1
num_classes = 2
linear_in = 768
max_len = 160
train_bs = 16
valid_bs = 8
start_lr = 2e-5
SEED = 42
now = datetime.now()
dt_string = now.strftime("%d_%m_%Y_%H_%M_%S_")
 out_path=dt_string+'zero_shot_pro.h5'
write_root=""
if IN_COLAB:
write_root=root+"outputs/"
out_path=write_root+out_path
print(out_path)
<choose_model_class>
util.set_seed(SEED )
if use_pretraining:
metric_prefix=""
if use_parcor:
metric_prefix="_toxic"
mc = ModelCheckpoint(out_path, monitor="val"+metric_prefix+"_auc", mode='max',verbose=1,save_weights_only=True,save_best_only=True)
 es = EarlyStopping(monitor="val"+metric_prefix+"_auc", mode='max',patience=2,verbose=1,restore_best_weights=True)
 lrScheduler=ReduceLROnPlateau(
 monitor="val"+metric_prefix+"_loss",
factor=0.2,
patience=1,
verbose=1,
mode="min",
min_delta=0,
cooldown=0,
min_lr=1e-10,
)<train_model>
train_df = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv' )
if use_pretraining:
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
verbose=1,
callbacks=[mc,es,lrScheduler]
)<load_from_csv>
import preprocess
import re
import string
%%time
sub = pd.read_csv(root+'/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv',nrows=nrows )<define_variables>
ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train_df = preprocess.fix_erraneous(train_df, ids_with_target_error, 0 )
test_files=list(filter(lambda file:"test" in file and "nen" in file, data_files))
test_files<categorify>
def remove_tabs_newlines(text):
text = re.sub(r'\t', ' ', text)
 text = re.sub(r'
', ' ', text)
 return text
if use_parcor:
test_dataset=parallel_labelled_dataset(tf.data.Dataset.from_tensor_slices(test_files ).interleave(perFileDataset,cycle_length=1)).map(_form_tuple ).batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 else:
test_dataset=tf.data.Dataset.from_tensor_slices(test_files ).interleave(perFileDataset,cycle_length=1 ).batch(BATCH_SIZE ).map(_form_tuple ).prefetch(AUTO)
<save_to_csv>
train_df['text'] = train_df['text'].apply(lambda k : remove_tabs_newlines(k))
test_df['text'] = test_df['text'].apply(lambda k : remove_tabs_newlines(k))
if use_pretraining:
l=model.predict(test_dataset, verbose=1)
 if use_dann:
l=l[0]
if use_parcor:
l=np.reshape(l,(-1,1))
sub['toxic'] = l[:,-1]
if use_finetuning:
write_file=write_root+'submission.unfinetuned.csv'
else:
write_file=write_root+'submission.csv'
sub.to_csv(write_file, index=False )<define_variables>
def remove_url(text):
return re.sub(r"http\S+", "URL", text )
if use_finetuning:
now = datetime.now()
dt_string = now.strftime("%d_%m_%Y_%H_%M_%S_")
 out_path=dt_string+'zero_shot_pro.h5'
write_root=""
if IN_COLAB:
write_root=root+"outputs/"
out_path=write_root+out_path
print(out_path)
<choose_model_class>
def remove_non_ascii(text):
return ''.join([x for x in text if x in string.printable] )
if use_finetuning:
metric_prefix=""
if use_parcor:
metric_prefix="_toxic"
mc = ModelCheckpoint(out_path, monitor="val"+metric_prefix+"_auc", mode='max',verbose=1,save_weights_only=True,save_best_only=True)
 es = EarlyStopping(monitor="val"+metric_prefix+"_auc", mode='max',patience=2,verbose=1,restore_best_weights=True)
 lrScheduler=ReduceLROnPlateau(
 monitor="val"+metric_prefix+"_loss",
factor=0.2,
patience=1,
verbose=1,
mode="min",
min_delta=0,
cooldown=0,
min_lr=1e-10,
)<train_model>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text )
if use_finetuning:
train_history = model.fit(
 finetune_dataset,
validation_data=valid_dataset,
epochs=EPOCHS,
verbose=1,
callbacks=[mc,es,lrScheduler]
)<save_to_csv>
def remove_usrname(text):
text = re.sub(r'@\S{0,}', ' USER ', text)
 text = re.sub(r"\s+", " ", text)
 text = re.sub(r'\b(USER )(\1\b)+', r'\1', text)
 return text
if use_finetuning:
l=model.predict(test_dataset, verbose=1)
 if use_dann:
l=l[0]
if use_parcor:
l=np.reshape(l,(-1,1))
sub['toxic'] = l[:,-1]
write_file=write_root+'submission.csv'
sub.to_csv(write_file, index=False )<set_options>
contractions = {
"ain't": "am not",
"aren't": "are not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hadn't've": "had not have",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'd've": "he would have",
"he'll": "he will",
"he's": "he is",
"how'd": "how did",
"how'll": "how will",
"how's": "how is",
"i'd": "i would",
"i'll": "i will",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'll": "it will",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"must've": "must have",
"mustn't": "must not",
"needn't": "need not",
"oughtn't": "ought not",
"shan't": "shall not",
"sha'n't": "shall not",
"she'd": "she would",
"she'll": "she will",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"that'd": "that would",
"that's": "that is",
"there'd": "there had",
"there's": "there is",
"they'd": "they would",
"they'll": "they will",
"they're": "they are",
"they've": "they have",
"wasn't": "was not",
"we'd": "we would",
"we'll": "we will",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"where'd": "where did",
"where's": "where is",
"who'll": "who will",
"who's": "who is",
"won't": "will not",
"wouldn't": "would not",
"you'd": "you would",
"you'll": "you will",
"you're": "you are",
"thx" : "thanks"
}
def remove_contractions(text):
return contractions[text.lower() ] if text.lower() in contractions.keys() else text
def reset_keras() :
tf.keras.backend.clear_session()
print(gc.collect() )<define_variables>
def remove_punctuations(text):
return text.translate(str.maketrans('', '', string.punctuation))
MAX_LEN = 192
DROPOUT = 0.5000001
BATCH_SIZE = 16
TOTAL_STEPS_STAGE1 = 2000
VALIDATE_EVERY_STAGE1 = 200
TOTAL_STEPS_STAGE2 = 200
VALIDATE_EVERY_STAGE2 = 10
LR_TRANSFORMER = 5e-6
LR_HEAD = 1e-3
PRETRAINED_TOKENIZER= 'jplu/tf-xlm-roberta-large'
PRETRAINED_MODEL = '/kaggle/input/jigsaw-mlm-finetuned-xlm-r-large'
D = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'
D_TRANS = '/kaggle/input/jigsaw-train-multilingual-coments-google-api/'
print(tf.__version__)
 logging.getLogger().setLevel(logging.NOTSET)
 AUTO = tf.data.experimental.AUTOTUNE<choose_model_class>
train_df['text'] = train_df['text'].apply(lambda k: remove_usrname(k))
train_df['text'] = train_df['text'].apply(lambda k: remove_contractions(k))
train_df['text'] = train_df['text'].apply(lambda k: remove_url(k))
train_df['text'] = train_df['text'].apply(lambda k: remove_emoji(k))
train_df['text'] = train_df['text'].apply(lambda k: remove_non_ascii(k))
train_df['text'] = train_df['text'].apply(lambda k: remove_punctuations(k))
def connect_to_TPU() :

 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
global_batch_size = BATCH_SIZE * strategy.num_replicas_in_sync
return tpu, strategy, global_batch_size
tpu, strategy, global_batch_size = connect_to_TPU()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_from_csv>
test_df['text'] = test_df['text'].apply(lambda k: remove_usrname(k))
test_df['text'] = test_df['text'].apply(lambda k: remove_contractions(k))
test_df['text'] = test_df['text'].apply(lambda k: remove_url(k))
test_df['text'] = test_df['text'].apply(lambda k: remove_emoji(k))
test_df['text'] = test_df['text'].apply(lambda k: remove_non_ascii(k))
test_df['text'] = test_df['text'].apply(lambda k: remove_punctuations(k))
def load_jigsaw_trans(langs=['tr','it','es','ru','fr','pt'],
columns=['comment_text', 'toxic']):
train_6langs=[]
for i in range(len(langs)) :
fn = D_TRANS+'jigsaw-toxic-comment-train-google-%s-cleaned.csv'%langs[i]
train_6langs.append(downsample(pd.read_csv(fn)[columns]))
return train_6langs
def downsample(df):

 ds_df= pd.concat([
df.query('toxic==1'),
df.query('toxic==0' ).sample(sum(df.toxic))
])
 return ds_df
train_df = pd.concat(load_jigsaw_trans())
 val_df = pd.read_csv(D+'validation.csv')
 test_df = pd.read_csv(D+'test.csv')
 sub_df = pd.read_csv(D+'sample_submission.csv' )<categorify>
def replace_nums(text):
text = re.sub(r'^\d\S{0,}| \d\S{0,}| \d\S{0,}$', ' NUMBER ', text)
 text = re.sub(r"\s+", " ", text)
 text = re.sub(r'\b(NUMBER )(\1\b)+', r'\1', text)
 text = re.sub(r"[0-9]", " ", text)
 return text
%%time
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
 tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_TOKENIZER)
 X_train = regular_encode(train_df.comment_text.values, tokenizer, maxlen=MAX_LEN)
 X_val = regular_encode(val_df.comment_text.values, tokenizer, maxlen=MAX_LEN)
 X_test = regular_encode(test_df.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train_df.toxic.values.reshape(-1,1)
 y_val = val_df.toxic.values.reshape(-1,1 )<create_dataframe>
train_df['text'] = train_df['text'].apply(lambda k: replace_nums(k))
test_df['text'] = test_df['text'].apply(lambda k: replace_nums(k))
def create_dist_dataset(X, y=None, training=False):
dataset = tf.data.Dataset.from_tensor_slices(X)
 if y is not None:
dataset_y = tf.data.Dataset.from_tensor_slices(y)
 dataset = tf.data.Dataset.zip(( dataset, dataset_y))
if training:
dataset = dataset.shuffle(len(X)).repeat()
dataset = dataset.batch(global_batch_size ).prefetch(AUTO)
 dist_dataset = strategy.experimental_distribute_dataset(dataset)
 return dist_dataset
train_dist_dataset = create_dist_dataset(X_train, y_train, True)
 val_dist_dataset = create_dist_dataset(X_val)
 test_dist_dataset = create_dist_dataset(X_test )<choose_model_class>
train_df['text'] = train_df['text'].apply(lambda k: cleanup(k))
test_df['text'] = test_df['text'].apply(lambda k: cleanup(k))
%%time
def create_model_and_optimizer() :
with strategy.scope() :
transformer_layer = TFRobertaModel.from_pretrained(PRETRAINED_MODEL)
 model = build_model(transformer_layer)
 optimizer_transformer = Adam(learning_rate=LR_TRANSFORMER)
 optimizer_head = Adam(learning_rate=LR_HEAD)
 return model, optimizer_transformer, optimizer_head
def build_model(transformer):
inp = Input(shape=(MAX_LEN,), dtype=tf.int32, name="input_word_ids")
 x = transformer(inp)[0][:, 0, :]
x = Dropout(DROPOUT )(x)
 out = Dense(1, activation='sigmoid', name='custom_head' )(x)
 model = Model(inputs=[inp], outputs=[out])
 return model
model, optimizer_transformer, optimizer_head = create_model_and_optimizer()
model.summary()<compute_train_metric>
train_y = train_df['target']
train_df.drop(['target'], axis=1, inplace=True )
def define_losses_and_metrics() :
with strategy.scope() :
loss_object = tf.keras.losses.BinaryCrossentropy(
 reduction=tf.keras.losses.Reduction.NONE, from_logits=False)
 def compute_loss(labels, predictions):
per_example_loss = loss_object(labels, predictions)
 loss = tf.nn.compute_average_loss(
 per_example_loss, global_batch_size = global_batch_size)
 return loss
train_accuracy_metric = tf.keras.metrics.AUC(name='training_AUC')
 return compute_loss, train_accuracy_metric
def train(train_dist_dataset, val_dist_dataset=None, y_val=None,
total_steps=2000, validate_every=200):
best_weights, history = None, []
step = 0
for tensor in train_dist_dataset:
distributed_train_step(tensor)
 step+=1
if(step % validate_every == 0):
train_metric = train_accuracy_metric.result().numpy()
print("Step %d, train AUC: %.5f" %(step, train_metric))
if val_dist_dataset:
val_metric = roc_auc_score(y_val, predict(val_dist_dataset))
print("Step %d, val AUC: %.5f" %(step,val_metric))
history.append(val_metric)
 if history[-1] == max(history):
best_weights = model.get_weights()
train_accuracy_metric.reset_states()
if step == total_steps:
break
model.set_weights(best_weights)
 @tf.function
def distributed_train_step(data):
strategy.experimental_run_v2(train_step, args=(data,))
def train_step(inputs):
features, labels = inputs
transformer_trainable_variables = [ v for v in model.trainable_variables
if(( 'pooler' not in v.name)and 
('custom' not in v.name)) ]
head_trainable_variables = [ v for v in model.trainable_variables
if 'custom' in v.name]
with tf.GradientTape(persistent=True)as tape:
predictions = model(features, training=True)
 loss = compute_loss(labels, predictions)
 gradients_transformer = tape.gradient(loss, transformer_trainable_variables)
 gradients_head = tape.gradient(loss, head_trainable_variables)
 del tape
optimizer_transformer.apply_gradients(zip(gradients_transformer,
transformer_trainable_variables))
optimizer_head.apply_gradients(zip(gradients_head,
head_trainable_variables))
train_accuracy_metric.update_state(labels, predictions)
 def predict(dataset):
predictions = []
for tensor in dataset:
predictions.append(distributed_prediction_step(tensor))
predictions = np.vstack(list(map(np.vstack,predictions)))
 return predictions
@tf.function
def distributed_prediction_step(data):
predictions = strategy.experimental_run_v2(prediction_step, args=(data,))
return strategy.experimental_local_results(predictions)
 def prediction_step(inputs):
features = inputs
predictions = model(features, training=False)
 return predictions
compute_loss, train_accuracy_metric = define_losses_and_metrics()<train_model>
X_train, X_test, y_train, y_test \
= train_test_split(train_df['text'], train_y, random_state=SEED, test_size=test_size, stratify=train_y.values )
%%time
train(train_dist_dataset, val_dist_dataset, y_val,
TOTAL_STEPS_STAGE1, VALIDATE_EVERY_STAGE1 )<split>
tokenizer = transformers.BertTokenizer.from_pretrained(
 model_name,
do_lower_case=True
)
%%time
optimizer_head.learning_rate.assign(1e-4)
 X_train, X_val, y_train, y_val = train_test_split(X_val, y_val, test_size = 0.1)
 train_dist_dataset = create_dist_dataset(X_train, y_train, training=True)
 val_dist_dataset = create_dist_dataset(X_val, y_val)
 train(train_dist_dataset, val_dist_dataset, y_val,
total_steps = TOTAL_STEPS_STAGE2,
validate_every = VALIDATE_EVERY_STAGE2 )<save_to_csv>
train_dataset = dataset.BertDataset(
 text=X_train.values,
tokenizer= tokenizer,
max_len= max_len,
target=y_train.values
)
%%time
sub_df['toxic'] = predict(test_dist_dataset)[:,0]
sub_df.to_csv('submission.csv', index=False )<set_options>
valid_dataset = dataset.BertDataset(
 text=X_test.values,
tokenizer= tokenizer,
max_len= max_len,
target=y_test.values
)
%matplotlib inline<load_from_csv>
train_dl = torch.utils.data.DataLoader(
 train_dataset,
train_bs,
shuffle=True,
num_workers=4
)
train1 = pd.read_csv(".. /input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv(".. /input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<concatenate>
valid_dl = torch.utils.data.DataLoader(
 valid_dataset,
valid_bs,
shuffle=True,
num_workers=1
)
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']]
] )<concatenate>
bert = model.BertHf(model_name, dp=dropout, num_classes=num_classes, linear_in=linear_in)
 bert = bert.to(device )
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1')
 ] ).sample(100000, random_state = 1)
 train = pd.concat([
train,
valid[["comment_text", "toxic"]]
] )<set_options>
for param in bert.parameters() :
param.requires_grad = True
plt.figure(figsize =(12, 8))
msno.bar(train )<load_pretrained>
no_decay = ['bias', 'LayerNorm.weight', 'LayerNorm.bias']
params = list(bert.named_parameters())
 modified_params = [
{'params': [p for n, p in params if not any(nd in n for nd in no_decay)], 'weight_decay': 0.1},
{'params': [p for n, p in params if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
tokenizer = AutoTokenizer.from_pretrained("jplu/tf-xlm-roberta-large" )<string_transform>
optim = torch.optim.AdamW(modified_params, lr=start_lr, eps=1e-8 )
exp_text = "I am currently participating in Jigsaw competition"
tokenizer.tokenize(exp_text )<define_variables>
total_steps = int(len(train_df)* epochs / train_bs)
 warmup_steps = int(len(train_df)* warmup_epochs / train_bs )
MAX_LEN = 192<string_transform>
sched = get_linear_schedule_with_warmup(optim, num_warmup_steps=warmup_steps, num_training_steps=total_steps )
def preprocess(data, max_seq_length = MAX_LEN, tokenizer = tokenizer):
ids = []
masks = []
segment = []
for i in tqdm(range(len(data))):
tokens = tokenizer.tokenize(data[i])
 if len(tokens)> max_seq_length - 2:
tokens = tokens[ : max_seq_length - 2]
input_ids = tokenizer.convert_tokens_to_ids(["[CLS]"] + tokens + ["[SEP]"])
 input_masks = [1] * len(input_ids)
 padding = max_seq_length - len(input_ids)
 input_ids.extend([0] * padding)
 input_masks.extend([0] * padding)
 segment_ids =[0]* max_seq_length
ids.append(input_ids)
 masks.append(input_masks)
 segment.append(segment_ids)
 return(np.array(ids), np.array(masks), np.array(segment)) 
<split>
criterion = torch.nn.CrossEntropyLoss()
train_ids, train_masks, train_segment = preprocess(train["comment_text"].values )<categorify>
train_stat = util.AvgStats()
valid_stat = util.AvgStats()
test_ids, test_masks, test_segment = preprocess(test["content"].values )<categorify>
best_acc = 0.0
best_model_file = str(model_name)+ '_best.pth.tar'
valid_ids, valid_masks, valid_segment = preprocess(valid["comment_text"].values )<prepare_x_and_y>
print("
Epoch\tTrain Acc\tTrain Loss\tTrain F1\tValid Acc\tValid Loss\tValid F1")
 for i in range(epochs):
start = time.time()
losses, ops, targs = train.train(train_dl, bert, criterion, optim, sched, device)
 duration = time.time() - start
train_acc = accuracy_score(targs, ops)
 train_f1_score = f1_score(targs, ops)
 train_loss = sum(losses)/len(losses)
 train_prec = precision_score(targs, ops)
 train_rec = recall_score(targs, ops)
 train_roc_auc = roc_auc_score(targs, ops)
 train_stat.append(train_loss, train_acc, train_f1_score, train_prec, train_rec, train_roc_auc, duration)
 start = time.time()
lossesv, opsv, targsv = train.test(valid_dl, bert, criterion, device)
 duration = time.time() - start
valid_acc = accuracy_score(targsv, opsv)
 valid_f1_score = f1_score(targsv, opsv)
 valid_loss = sum(lossesv)/len(lossesv)
 valid_prec = precision_score(targsv, opsv)
 valid_rec = recall_score(targsv, opsv)
 valid_roc_auc = roc_auc_score(targsv, opsv)
 valid_stat.append(valid_loss, valid_acc, valid_f1_score, valid_prec, valid_rec, valid_roc_auc, duration)
 if valid_acc > best_acc:
best_acc = valid_acc
util.save_checkpoint(bert, True, best_model_file)
 tfpr, ttpr, _ = roc_curve(targs, ops)
 train_stat.update_best(ttpr, tfpr, train_acc, i)
 vfpr, vtpr, _ = roc_curve(targsv, opsv)
 valid_stat.update_best(vtpr, vfpr, best_acc, i)
 print("
{}\t{:06.8f}\t{:06.8f}\t{:06.8f}\t{:06.8f}\t{:06.8f}\t{:06.8f}".format(i+1, train_acc*100, train_loss,
train_f1_score, valid_acc*100,
valid_loss, valid_f1_score))
y_train = train["toxic"].values
y_valid = valid["toxic"].values<set_options>
print("Summary of best run::")
 print("Best Accuracy: {}".format(valid_stat.best_acc))
print("Roc Auc score: {}".format(valid_stat.roc_aucs[valid_stat.best_epoch]))
print("Loss: {}".format(valid_stat.losses[valid_stat.best_epoch]))
print("Area Under Curve: {}".format(auc(valid_stat.fprs, valid_stat.tprs)) )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
test_dataset = dataset.BertDataset(
 text=test_df.text.values,
tokenizer= tokenizer,
max_len= max_len
)
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<choose_model_class>
test_dl = torch.utils.data.DataLoader(
 test_dataset,
1,
shuffle=False,
num_workers=1 
)
def model(roberta_layer, max_len = MAX_LEN):
input_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 pooled_output, sequence_output = roberta_layer([input_word_ids, input_mask, segment_ids])
 x = pooled_output
x = tf.keras.layers.Flatten()(x)
 x = tf.keras.layers.Dense(128, activation = "relu" )(x)
 x = tf.keras.layers.Dense(1, activation = "sigmoid" )(x)
 model = tf.keras.Model(inputs = [input_word_ids, input_mask, segment_ids], outputs = x)
 return model<load_pretrained>
util.load_checkpoint(bert, best_model_file )
with strategy.scope() :
roberta_layer = TFAutoModel.from_pretrained("jplu/tf-xlm-roberta-large", trainable = True)
 model = model(roberta_layer)
 model.compile(tf.keras.optimizers.Adam(lr=1e-5), loss="binary_crossentropy", metrics=["accuracy"])
 model.summary()<split>
_, opst, _ = train.test(test_dl, bert, criterion, device )
skf = StratifiedKFold(n_splits=5, shuffle = True)
 skf.get_n_splits(train_ids, y_train)
 skf<split>
sub_csv = pd.DataFrame(columns=['id', 'target'] )
i = 1
preds = []
for train_index, test_index in skf.split(train_ids, y_train):
print("
")
 print("
print(f"FOLD No {i}")
 print("
tr_ids = train_ids[train_index]
tr_masks = train_masks[train_index]
tr_segment = train_segment[train_index]
vd_ids = train_ids[test_index]
vd_masks = train_masks[test_index]
vd_segment = train_segment[test_index]
y_tr = y_train[train_index]
y_vd = y_train[test_index]
history = model.fit(
(tr_ids, tr_masks, tr_segment), y_tr,
epochs=2,
batch_size=BATCH_SIZE,
validation_data =(( vd_ids, vd_masks, vd_segment), y_vd),
steps_per_epoch = len(tr_ids)//BATCH_SIZE)
 predictions = model.predict(( test_ids, test_masks, test_segment))
preds.append(predictions)
 i += 1
K.clear_session()<prepare_output>
sub_csv['id'] = test_df['id']
sub_csv['target'] = opst
<save_to_csv><EOS> 
sub_csv.to_csv('submission.csv', index=False )