Kaggle competition: Natural Language Processing with Disaster Tweets
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<prepare_x_and_y>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 samp_sub = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv' )
%%time
x_train = regular_encode(train1.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train1.toxic.values
y_valid = valid.toxic.values<define_variables>
!pip install pytorch-pretrained-bert pytorch-nlp
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 n_gpu = torch.cuda.device_count()
torch.cuda.get_device_name(0 )
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
tweet_lens = [len(i)for i in train['text']]
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS+1
)<train_model>
MAX_LEN = max(tweet_lens )
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS * 3
)<save_to_csv>
labels = train['target'].values
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<load_pretrained>
nb_labels = len(labels )
model_json = model.to_json()
with open("model-xlm.json", "w")as json_file:
json_file.write(model_json)
 model.save_weights("model-xlm.h5")
<import_modules>
sentences = ["[CLS] " + query + " [SEP]" for query in train['text']]
sentences_test = ["[CLS] " + query + " [SEP]" for query in test['text']]
print(sentences[0] )
import os<set_options>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
 tokenized_texts = [tokenizer.tokenize(sent)for sent in sentences]
tokenized_texts_test = [tokenizer.tokenize(sent)for sent in sentences_test]
print("Tokenize the first sentence:")
 print(tokenized_texts[0] )
%matplotlib inline<import_modules>
input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt)for txt in tokenized_texts],
maxlen=MAX_LEN, dtype="long", truncating="post", padding="post")
 input_ids_test = pad_sequences([tokenizer.convert_tokens_to_ids(txt)for txt in tokenized_texts_test],
maxlen=MAX_LEN, dtype="long", truncating="post", padding="post" )
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint<import_modules>
input_ids = [tokenizer.convert_tokens_to_ids(x)for x in tokenized_texts]
input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype="long", truncating="post", padding="post")
 input_ids_test = [tokenizer.convert_tokens_to_ids(x)for x in tokenized_texts_test]
input_ids_test = pad_sequences(input_ids_test, maxlen=MAX_LEN, dtype="long", truncating="post", padding="post" )
import transformers
from tokenizers import BertWordPieceTokenizer
from tqdm.notebook import tqdm
from kaggle_datasets import KaggleDatasets<categorify>
attention_masks = []
attention_masks_test = []
for seq in input_ids:
seq_mask = [float(i>0)for i in seq]
attention_masks.append(seq_mask)
 for seq in input_ids_test:
seq_mask = [float(i>0)for i in seq]
attention_masks_test.append(seq_mask)
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<choose_model_class>
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=2018, test_size=0.1)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, input_ids,
random_state=2018, test_size=0.1 )
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
dense_layer = Dense(224, activation='relu' )(cls_token)
 out = Dense(1, activation='sigmoid' )(dense_layer)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model 
<set_options>
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels)
 validation_labels = torch.tensor(validation_labels)
 train_masks = torch.tensor(train_masks)
 validation_masks = torch.tensor(validation_masks)
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
test_masks_sub = torch.tensor(attention_masks_test)
 test_inputs_sub = torch.tensor(input_ids_test)
 train_masks_sub = torch.tensor(attention_masks)
 train_inputs_sub = torch.tensor(input_ids)
 train_labels_sub = torch.tensor(labels )
AUTO = tf.data.experimental.AUTOTUNE 
<define_variables>
batch_size = 32
EPOCHS = 3
BATCH_SIZE = 32 * strategy.num_replicas_in_sync
MAX_LEN = 192<load_pretrained>
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size )
tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')
 tokenizer.save_pretrained('.')
 fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=True)
 fast_tokenizer<define_variables>
model = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=nb_labels)
 model.cuda()
DATA_PATH = "/kaggle/input/jigsaw-multilingual-toxic-comment-classification/"<load_from_csv>
param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.0}
]
train1 = pd.read_csv(os.path.join(DATA_PATH, "jigsaw-toxic-comment-train.csv"))
train2 = pd.read_csv(os.path.join(DATA_PATH, "jigsaw-unintended-bias-train.csv"))
train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv(os.path.join(DATA_PATH, 'validation.csv'))
test = pd.read_csv(os.path.join(DATA_PATH, 'test.csv'))
sub = pd.read_csv(os.path.join(DATA_PATH, 'sample_submission.csv'))<concatenate>
optimizer = BertAdam(optimizer_grouped_parameters,
lr=2e-5,
warmup=.1 )
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=150000, random_state=3982)
 ])
<prepare_x_and_y>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )
x_train = fast_encode(train.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<define_variables>
train_loss_set = []
epochs = 4
for _ in trange(epochs, desc="Epoch"):
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(train_dataloader):
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
optimizer.zero_grad()
loss = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)
 train_loss_set.append(loss.item())
 loss.backward()
optimizer.step()
tr_loss += loss.item()
nb_tr_examples += b_input_ids.size(0)
 nb_tr_steps += 1
print("Train loss: {}".format(tr_loss/nb_tr_steps))
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
 logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print("Validation Accuracy: {}".format(eval_accuracy/nb_eval_steps))
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<train_model>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
train_data_sub = TensorDataset(train_inputs_sub, train_masks_sub, train_labels_sub)
 train_dataloader_sub = DataLoader(train_data_sub, batch_size=batch_size)
 test_data_sub = TensorDataset(test_inputs_sub, test_masks_sub)
 test_dataloader_sub = DataLoader(test_data_sub, batch_size=batch_size)
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS*2
)<save_to_csv>
train_loss_set = []
epochs = 10
for _ in trange(epochs, desc="Epoch"):
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(train_dataloader_sub):
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
optimizer.zero_grad()
loss = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)
 train_loss_set.append(loss.item())
 loss.backward()
optimizer.step()
tr_loss += loss.item()
nb_tr_examples += b_input_ids.size(0)
 nb_tr_steps += 1
print("Train loss: {}".format(tr_loss/nb_tr_steps))
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<load_from_csv>
model.eval()
targs = []
for batch in test_dataloader_sub:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
 logits = logits.detach().cpu().numpy()
targs.append(logits)
submission1 = pd.read_csv(os.path.join(EXTERNAL_DATA_PATH, 'submission-1.csv'))
submission2 = pd.read_csv(os.path.join(EXTERNAL_DATA_PATH, 'submission-2.csv'))
submission3 = pd.read_csv(os.path.join(EXTERNAL_DATA_PATH, 'submission-3.csv'))<save_to_csv>
pred_flat = [np.argmax(i, axis=1 ).flatten().tolist() for i in targs]
submission1['toxic'] = submission1['toxic']*0.05 + submission2['toxic']*0.15 + submission3['toxic']*0.8
submission1.to_csv('submission-1.csv', index=False )<import_modules>
topc_list1 = reduce(lambda x,y: x+y, pred_flat )
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tokenizers import Tokenizer, models<categorify>
sub=pd.DataFrame({'id':samp_sub['id'].values.tolist() ,'target':topc_list1} )
<set_options><EOS> 
sub.to_csv('submission.csv',index=False )