Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
import pandas as pd
import torch
import transformers
from torch.utils.data import Dataset, DataLoader
import torch.nn as nn
from transformers import DistilBertModel, DistilBertTokenizer
from sklearn import metrics
from tqdm import tqdm
import numpy as np
import re
sub_70=['/kaggle/input/009474-fork009473-xlmrobertawithconv/submission(98 ).csv',
'/kaggle/input/009474-fork009473-xlmrobertawithconv1/submission(99 ).csv',
'/kaggle/input/009474-009474-fork009473/submission(97 ).csv',
'/kaggle/input/009471-ensembleversion96-submission-27/submission - 2020-06-21T175246.715.csv',
'/kaggle/input/009475/submission(90 ).csv',
'/kaggle/input/009476/submission(95 ).csv',
'/kaggle/input/009471-ensembleversion96-submission-27/submission - 2020-06-21T175246.715.csv',
'/kaggle/input/009472-howlingwithwolf-tfidfsubmission-27/submission - 2020-06-21T173345.640.csv',
'/kaggle/input/009474-009474-fork009473/submission(97 ).csv',
'/kaggle/input/009475/submission(90 ).csv',
'/kaggle/input/009471-submission-27-submission-72/submission - 2020-06-21T175653.332.csv',
'/kaggle/input/009474-fork009473-xlmrobertawithconv/submission(98 ).csv',
'/kaggle/input/009474-fork009473-xlmrobertawithconv1/submission(99 ).csv',
'/kaggle/input/009473-version6-robertaaugmentationssl09417/submission(100 ).csv',
'/kaggle/input/009472-howlingwithwolf-tfidfsubmission-27/submission - 2020-06-21T173345.640.csv',
'/kaggle/input/9467-xlmrlargexlmrobertasubmission-47/submission - 2020-06-21T181840.475.csv',
'/kaggle/input/009472-tputrainingsuper-ensembleversion96/submission - 2020-06-21T173040.684.csv']
sub_60=['/kaggle/input/009469-finetunedxlmrlargefastxlmroberta/submission - 2020-06-21T180227.893.csv',
'/kaggle/input/009469-finetunedlarge-superfastxlmroberta/submission - 2020-06-21T180023.388.csv',
'/kaggle/input/9462-v/submission - 2020-06-21T210004.249.csv',
'/kaggle/input/009469-finetunedxlmrlargefastxlmroberta/submission - 2020-06-21T180227.893.csv',
'/kaggle/input/9462-v/submission - 2020-06-21T210004.249.csv']
sub_un=['/kaggle/input/9423-v/submission - 2020-06-21T215335.038.csv',
'/kaggle/input/009406/submission(21 ).csv',
'/kaggle/input/009383/submission(17 ).csv',
'/kaggle/input/009259/submission(16 ).csv',
'/kaggle/input/009248/submission(15 ).csv',
'/kaggle/input/9422-v/submission - 2020-06-21T220028.429.csv',
'/kaggle/input/009383/submission(17 ).csv',
'/kaggle/input/009259/submission(16 ).csv',
'/kaggle/input/009423/submission(48 ).csv',
'/kaggle/input/009354/submission(25 ).csv',
'/kaggle/input/9459-vv/submission - 2020-06-21T214937.374.csv',
'/kaggle/input/009406/submission(21 ).csv',
'/kaggle/input/9423-v/submission - 2020-06-21T215335.038.csv',
'/kaggle/input/9398-v/submission - 2020-06-21T205031.461.csv',
'/kaggle/input/9459-vv/submission - 2020-06-21T214937.374.csv',
'/kaggle/input/009248/submission(15 ).csv']<load_from_csv>
device = 'cuda' if cuda.is_available() else 'cpu'
sub1 = pd.read_csv('/kaggle/input/009476/submission(95 ).csv')
 sub2 = pd.read_csv('/kaggle/input/tfidf-v-44/submission(94 ).csv')
 sub3 = pd.read_csv('/kaggle/input/009475-vv/submission - 2020-06-22T112115.366.csv' )<feature_engineering>
df_train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")[["text","target"]]
sub1['toxic'] = sub1['toxic']*0.9 + sub2['toxic']*0.1 
<save_to_csv>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_numbers(text):
text = ''.join([i for i in text if not i.isdigit() ])
 return text
def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_username(text):
url = re.compile(r'@[A-Za-z0-9_]+')
 return url.sub(r'',text )
sub1.to_csv('submission.csv', index=False )<import_modules>
def pre_process_text(text):
text = remove_URL(text)
 text = remove_numbers(text)
 text = remove_html(text)
 text = remove_username(text)
 return " ".join(text.split() )
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import math
import os
import re
import time
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.metrics import AUC
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
MAX_LEN = 160
BATCH_SIZE = 16
EPOCHS = 1
LEARNING_RATE = 1e-05
BERT_PATH = '/kaggle/input/distillbert-huggingface-model/'
MODEL_PATH = "distilbert-base-uncased-pytorch_model.bin"
tokenizer = DistilBertTokenizer.from_pretrained(
 BERT_PATH,
do_lower_case=True
)
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<categorify>
class tweet_Dataset(Dataset):
def __init__(self, dataframe, tokenizer, max_len):
self.data = dataframe
self.tokenizer = tokenizer
self.max_len = max_len
def __getitem__(self, index):
tweet = str(self.data.text[index])
 tweet = pre_process_text(tweet)
 inputs = self.tokenizer.encode_plus(
 tweet,
None,
add_special_tokens=True,
max_length=self.max_len,
pad_to_max_length=True,
return_token_type_ids=True 
)
 ids = inputs['input_ids']
mask = inputs['attention_mask']
return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'targets': torch.tensor(self.data.target[index], dtype=torch.float)
 }
def __len__(self):
return len(self.data )
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
train_params = {'batch_size': BATCH_SIZE,
'shuffle': True,
'num_workers': 0
}
valid_params = {'batch_size': BATCH_SIZE,
'shuffle': True,
'num_workers': 0
}
train_dl = DataLoader(training_set, **train_params)
 valid_dl = DataLoader(validation_set, **valid_params )
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-3), loss='binary_crossentropy', metrics=['accuracy'])
 return model<set_options>
class DistillBERTClass(torch.nn.Module):
def __init__(self):
super().__init__()
self.distill_bert = transformers.DistilBertModel.from_pretrained(BERT_PATH)
 self.drop = torch.nn.Dropout(0.3)
 self.out = torch.nn.Linear(768, 1)
 def forward(self, ids, mask):
distilbert_output = self.distill_bert(ids, mask)
 hidden_state = distilbert_output[0]
pooled_output = hidden_state[:, 0]
output_1 = self.drop(pooled_output)
 output = self.out(output_1)
 return output
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
model = DistillBERTClass()
model.to(device )
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 20
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 200
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
def loss_fn(outputs, targets):
return nn.BCEWithLogitsLoss()(outputs, targets)
 optimizer = torch.optim.Adam(params = model.parameters() , lr=LEARNING_RATE )
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
def eval_fn(data_loader, model):
model.eval()
fin_targets = []
fin_outputs = []
with torch.no_grad() :
for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)) :
ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(ids=ids, mask=mask)
 fin_targets.extend(targets.cpu().detach().numpy().tolist())
 fin_outputs.extend(torch.sigmoid(outputs ).cpu().detach().numpy().tolist())
 fin_outputs = np.array(fin_outputs)>= 0.5
f1 = metrics.f1_score(fin_targets, fin_outputs)
 return f1
train = pd.read_csv('/kaggle/input/toxic-comment-classification/train.csv')
 valid = pd.read_csv('/kaggle/input/toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/toxic-comment-classification/test.csv' )<prepare_x_and_y>
def fit(num_epochs, model, loss_fn, opt, train_dl, valid_dl):
for epoch in range(num_epochs):
model.train()
for _,data in enumerate(train_dl, 0):
ids = data['ids'].to(device, dtype = torch.long)
 mask = data['mask'].to(device, dtype = torch.long)
 targets = data['targets'].to(device, dtype = torch.float)
 outputs = model(ids, mask ).squeeze()
loss = loss_fn(outputs, targets)
 loss.backward()
opt.step()
opt.zero_grad()
valid_acc = eval_fn(valid_dl, model)
 print('Epoch [{}/{}], Train Loss: {:.4f} and Validation f1 {:.4f}'.format(epoch+1, num_epochs, loss.item() ,valid_acc))
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.comment_text.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<define_variables>
fit(7, model, loss_fn, optimizer, train_dl,valid_dl )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
def sentence_prediction(sentence):
max_len = MAX_LEN
tweet = str(sentence)
 tweet = " ".join(tweet.split())
 inputs = tokenizer.encode_plus(
 tweet,
None,
add_special_tokens=True,
max_length=max_len,
pad_to_max_length=True, 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
ids = torch.tensor(ids, dtype=torch.long ).unsqueeze(0)
 mask = torch.tensor(mask, dtype=torch.long ).unsqueeze(0)
 ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 outputs = model(ids=ids, mask=mask)
 outputs = torch.sigmoid(outputs ).cpu().detach().numpy()
return outputs[0][0] > 0.5
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<choose_model_class>
sentence_prediction("We are experiencing slight tremors in London right now" )
filepath = "roberta_model.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, save_weights_only=True, mode='max')
 early = EarlyStopping(monitor='val_accuracy', mode='max', patience=5)
 reduce_lr = ReduceLROnPlateau(monitor='val_accuracy', factor=0.1, patience=2, min_lr=1e-5 )<train_model>
sentence_prediction("Lol this movie is an absolute disaster!" )
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
callbacks=[early, checkpoint, reduce_lr]
)<predict_on_test>
df_test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")[["id","text"]]
y_true = test['toxic'].values
y_pred = model.predict(test_dataset ).ravel()<compute_test_metric>
df_test['target'] = df_test['text'].apply(sentence_prediction )
roc_auc_score(y_true, y_pred )<set_options>
df_test['target'] = df_test['target'].astype(int )
<define_variables><EOS> 
df_test.to_csv("submission.csv", columns = ['id','target'], index=False )