Kaggle competition: Natural Language Processing with Disaster Tweets
train['text_n']=n_corpus
train.drop('text',axis=1 )<string_transform>
import transformers
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
from sklearn.model_selection import train_test_split
n_corpus=[]
for text in tqdm(test['text']):
text = re.sub(r'https?://\S+|www\.\S+', '', text)
 text = re.sub(r'<.*?>', '', text)
 text = re.sub(r'[^a-zA-Z0-9]+', ' ', text)
 text = re.sub(r'[0-9]', '', text)
 text = text.lower()
text = nltk.word_tokenize(text)
 ps = PorterStemmer()
text = [ps.stem(word)for word in text if not word in set(stopwords.words('english')) ]
text = ' '.join(text)
 n_corpus.append(text )<drop_column>
train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
test['text_n']=n_corpus
test.drop('text',axis=1 )<load_from_url>
X = train_df.loc[:,'text']
y = train_df.loc[:,'target']
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
max_len = 0
for text in X:
max_len = max(max_len, len(text))
max_len
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization<categorify>
class Dataset(torch.utils.data.Dataset):
def __init__(self,df,y=None,max_len=164):
self.df = df
self.y = y
self.max_len= max_len
self.tokenizer = transformers.RobertaTokenizer.from_pretrained('roberta-base')
 def __getitem__(self,index):
row = self.df.iloc[index]
ids,masks = self.get_input_data(row)
 data = {}
data['ids'] = ids
data['masks'] = masks
if self.y is not None:
data['out'] = torch.tensor(self.y.iloc[index],dtype=torch.float32)
 return data
def __len__(self):
return len(self.df)
 def get_input_data(self,row):
row = self.tokenizer.encode(row,add_special_tokens=True,add_prefix_space=True)
 padded = row + [0]*(max_len - len(row))
padded = torch.tensor(padded,dtype=torch.int64)
 mask = torch.where(padded != 0 , torch.tensor(1),torch.tensor(0))
return padded,mask
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
train_x,val_x,train_y,val_y = train_test_split(X,y,test_size=0.2,stratify=y)
 train_loader = torch.utils.data.DataLoader(Dataset(train_x,train_y),batch_size=16,shuffle=True,num_workers=2)
 val_loader = torch.utils.data.DataLoader(Dataset(val_x,val_y),batch_size=16,shuffle=False,num_workers=2 )
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model<choose_model_class>
class Model(nn.Module):
def __init__(self):
super(Model,self ).__init__()
self.distilBert = transformers.RobertaModel.from_pretrained('roberta-base')
 self.l0 = nn.Linear(768,512)
 self.l1 = nn.Linear(512,256)
 self.l2 = nn.Linear(256,1)
 self.d0 = nn.Dropout(0.5)
 self.d1 = nn.Dropout(0.5)
 self.d2 = nn.Dropout(0.5)
 nn.init.normal_(self.l0.weight,std=0.2)
 nn.init.normal_(self.l1.weight,std=0.2)
 nn.init.normal_(self.l2.weight,std=0.2)
 def forward(self,ids,masks):
hid = self.distilBert(ids,attention_mask=masks)
 hid = hid[0][:,0]
x = self.d0(hid)
 x = self.l0(x)
 x = F.leaky_relu(x)
 x = self.d1(x)
 x = self.l1(x)
 x = F.leaky_relu(x)
 x = self.d2(x)
 x = self.l2(x)
 return x
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<feature_engineering>
model = Model().to('cuda')
 criterion = nn.BCEWithLogitsLoss(reduction='mean')
 optimizer = torch.optim.AdamW(model.parameters() ,lr=3e-5 )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
def accuracy_score(outputs,labels):
outputs = torch.round(torch.sigmoid(outputs))
correct =(outputs == labels ).sum().float()
return correct/labels.size(0 )
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values<train_model>
from tqdm import tqdm
train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
batch_size=16 
)
 model.save('model.h5' )<save_to_csv>
epochs = 4
for epoch in range(epochs):
epoch_loss = 0.
 model.train()
for data in tqdm(train_loader):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
labels = data['out'].cuda()
labels = labels.unsqueeze(1)
 optimizer.zero_grad()
outputs = model(ids,masks)
 loss = criterion(outputs,labels)
 loss.backward()
optimizer.step()
epoch_loss += loss.item()
print(f"Epoch{epoch+1} loss : {epoch_loss/len(train_loader)}")
 print(f"Epoch{epoch+1} accuracy : ",accuracy_score(outputs,labels ).item())
 val_loss = 0.
 model.eval()
for data in tqdm(val_loader):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
labels = data['out'].cuda()
labels = labels.unsqueeze(1)
 outputs = model(ids,masks)
 loss = criterion(outputs,labels)
 val_loss += loss.item()
print(f"Epoch{epoch+1} val loss : {val_loss/len(val_loader)}")
 print(f"Epoch{epoch+1} val accuracy : ",accuracy_score(outputs,labels ).item())
test_pred = model.predict(test_input)
 submission=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False)
<install_modules>
test_loader = torch.utils.data.DataLoader(Dataset(test_df['text'],y=None),batch_size=16,shuffle=False,num_workers=2 )
!pip install bert-for-tf2
!pip install sentencepiece<import_modules>
preds = []
for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
model.eval()
outputs = model(ids,masks)
 preds += outputs.cpu().detach().numpy().tolist()
try:
%tensorflow_version 2.x
except Exception:
pass 
<compute_test_metric>
pred = np.round(1/(1 + np.exp(-np.array(preds))))
def recall_m(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision_m(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
def f1_m(y_true, y_pred):
precision = precision_m(y_true, y_pred)
 recall = recall_m(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<load_from_csv>
pred = np.array(pred,dtype=np.uint8 )
train= pd.read_csv('.. /input/extensive-pre-processing-for-bert/processed train.csv')
 train.head(5 )<filter>
sub = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv' )
train.loc[4,'processed_text']<load_from_csv>
sub['target'] = pred
test=pd.read_csv('.. /input/extensive-pre-processing-for-bert/processed test.csv')
 test = test.set_index(test['id'])
 test.head(5 )<load_from_csv>
sub.to_csv('submission.csv',index=False )