Kaggle competition: Natural Language Processing with Disaster Tweets
out_df = pd.DataFrame(np.c_[np.arange(1, len(test_set)+1)[:,None], test_pred.numpy() ],
columns=['ImageId', 'Label'])
 out_df.head()<save_to_csv>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
out_df.to_csv('submission.csv', index=False )<import_modules>
pd.set_option("display.max_colwidth", 200)
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 %matplotlib inline
import torch
import torchvision
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
import copy
import time<set_options>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv' )
torch.backends.cudnn.enabled<set_options>
train['target'].value_counts()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device )<define_variables>
def remove_pattern(input_txt, pattern):
r = re.findall(pattern, input_txt)
 for i in r:
input_txt = re.sub(i, '', input_txt)
 return input_txt
train['tweet'] = np.vectorize(remove_pattern )(train['text'], "
test['tweet'] = np.vectorize(remove_pattern )(test['text'], "
train.head()
Num_CNN = 12
n_epochs = 25
batch_size_train = 64
batch_size_test = 1000
learning_rate = 0.01
momentum = 0.5
log_interval = 100
random_seed = 121
torch.manual_seed(random_seed)
 kaggle_input_data = "/kaggle/input/digit-recognizer/train.csv"
kaggle_input_validation = "/kaggle/input/digit-recognizer/test.csv"
kaggle_model_path = "/kaggle/working/model_id_%d.pth"
kaggle_optimizer_path = "/kaggle/working/optimizer_id_%d.pth"
prediction_path = "/kaggle/working/predictions_essemble.csv"<prepare_x_and_y>
train['tweet'] = train['tweet'].str.replace("[^a-zA-Z
test['tweet'] = test['tweet'].str.replace("[^a-zA-Z
train.head()
class Dataset(torch.utils.data.Dataset):
def __init__(self, dataframe):
self.labels = dataframe["label"].to_numpy()
self.dataframe = dataframe.loc[:,dataframe.columns != "label"]
def __len__(self):
return self.dataframe.shape[0]
def __getitem__(self, index):
X = torch.from_numpy(self.dataframe.iloc[index].values.reshape(1,28,28)).float()
y = self.labels[index]
return X,y<load_from_csv>
train['tweet'] = train['tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))
test['tweet'] = test['tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))
train.head()
df_input = pd.read_csv(kaggle_input_data)
 df_validation = pd.read_csv(kaggle_input_validation)
 df_train, df_test = train_test_split(df_input, test_size = 0.01)
 training_set = Dataset(df_train)
 training_generator = torch.utils.data.DataLoader(training_set, batch_size = batch_size_train,
shuffle = True)
 test_set = Dataset(df_test)
 test_generator = torch.utils.data.DataLoader(test_set, batch_size = batch_size_test,
shuffle = True)
 kaggle_validation_set = torch.from_numpy(df_validation.to_numpy() ).view(
 df_validation.shape[0],-1,28,28 ).float()<import_modules>
train['tweet'] = train['tweet'].str.lower()
test['tweet'] = test['tweet'].str.lower()
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler<define_search_model>
set(stopwords.words('english'))
stops = set(stopwords.words('english'))
train['tokenized_sents'] = train.apply(lambda row: nltk.word_tokenize(row['tweet']), axis=1)
 test['tokenized_sents'] = test.apply(lambda row: nltk.word_tokenize(row['tweet']), axis=1)
 def remove_stops(row):
my_list = row['tokenized_sents']
meaningful_words = [w for w in my_list if not w in stops]
return(meaningful_words)
 train['clean_tweet'] = train.apply(remove_stops, axis=1)
 test['clean_tweet'] = test.apply(remove_stops, axis=1)
 train.drop(["tweet","tokenized_sents"], axis = 1, inplace = True)
 test.drop(["tweet","tokenized_sents"], axis = 1, inplace = True )
class Net(nn.Module):
def __init__(self):
super(Net, self ).__init__()
self.conv1 = nn.Conv2d(1, 32, kernel_size=3)
 self.batchnorm1 = nn.BatchNorm2d(32)
 self.conv2 = nn.Conv2d(32, 32, kernel_size=3)
 self.batchnorm2 = nn.BatchNorm2d(32)
 self.conv3 = nn.Conv2d(32, 32, kernel_size=2, stride = 2)
 self.batchnorm3 = nn.BatchNorm2d(32)
 self.conv4 = nn.Conv2d(32, 64, kernel_size=5)
 self.batchnorm4 = nn.BatchNorm2d(64)
 self.conv5 = nn.Conv2d(64, 64, kernel_size=2, stride = 2)
 self.batchnorm5 = nn.BatchNorm2d(64)
 self.conv5_drop = nn.Dropout2d()
self.fc1 = nn.Linear(1024, 128)
 self.fc2 = nn.Linear(128, 10)
 def forward(self, x):
x = self.batchnorm1(F.relu(self.conv1(x)))
 x = self.batchnorm2(F.relu(self.conv2(x)))
 x = self.batchnorm3(F.relu(self.conv3(x)))
 x = self.batchnorm4(F.relu(self.conv4(x)))
 x = self.batchnorm5(F.relu(self.conv5(x)))
 x = self.conv5_drop(x)
 x = x.view(-1, 1024)
 x = F.relu(self.fc1(x))
x = F.log_softmax(self.fc2(x), dim=1)
 return x<define_variables>
def rejoin_words(row):
my_list = row['clean_tweet']
joined_words =(" ".join(my_list))
return joined_words
train['clean_tweet'] = train.apply(rejoin_words, axis=1)
 test['clean_tweet'] = test.apply(rejoin_words, axis=1)
 train.head()
train_losses = []
train_counter = []
test_losses = []
test_counter = [i*len(training_generator.dataset)for i in range(n_epochs + 1)]
log_interval = len(training_generator)
 network_list = []
prediction_tensor = torch.zeros(kaggle_validation_set.shape[0],10 )<train_model>
train.drop(["keyword","location","text"], axis = 1, inplace = True)
 test.drop(["keyword","location","text"], axis = 1, inplace = True)
 train.head()
def train(epoch, network, scheduler, network_id, device):
start_time = time.time()
network.train()
for batch_idx,(data, target)in enumerate(training_generator):
data = data.to(device)
 target = target.to(device)
 optimizer.zero_grad()
output = network(data)
 loss = F.nll_loss(output, target)
 loss.backward()
optimizer.step()
print('CNN : {},\t Train Epoch: {} \tLoss: {:.6f},\t runtime: {:.2f}'.format(
 network_id,
epoch, loss.item() , time.time() -start_time))
train_losses.append(loss.item())
 train_counter.append(epoch)
 torch.save(network.state_dict() , kaggle_model_path%network_id)
 torch.save(optimizer.state_dict() ,kaggle_optimizer_path%network_id)
 scheduler.step()
def test(network):
network.eval()
test_loss = 0
correct = 0
with torch.no_grad() :
for data, target in test_generator:
data = data.to(device)
 target = target.to(device)
 output = network(data)
 test_loss += F.nll_loss(output, target, reduction="sum" ).item()
pred = output.data.max(1, keepdim=True)[1]
correct += pred.eq(target.data.view_as(pred)).sum()
test_loss /= len(test_generator.dataset)
 test_losses.append(test_loss)
 print('
Test set: Avg.loss: {:.4f}, Accuracy: {}/{}({:.5f}%)
 '.format(
 test_loss, correct, len(test_generator.dataset),
100.* correct / len(test_generator.dataset)))
 def validation(network, validation_tensor, prediction_tensor):
with torch.no_grad() :
output = network(validation_tensor)
 prediction_tensor = prediction_tensor + output
return prediction_tensor
def initialise_network(device):
network = Net().to(device)
 optimizer = optim.SGD(network.parameters() , lr=learning_rate, momentum = momentum)
 exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.95)
 return network, optimizer, exp_lr_scheduler<train_model>
tokenizer_QA = BertTokenizer.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')
validation_tensor = kaggle_validation_set.to(device)
 prediction_tensor = prediction_tensor.to(device)
 for id_net in range(1, Num_CNN+1):
network, optimizer, scheduler = initialise_network(device)
 for epoch in range(1, n_epochs+1):
train(epoch, network, scheduler, id_net, device)
 test(network)
 prediction_tensor = validation(network, validation_tensor, prediction_tensor)
 network_list.append(copy.deepcopy(network))<create_dataframe>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer_QA.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer_QA.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
output_df = prediction_tensor.max(1)[1].to("cpu")
 output_df = pd.DataFrame(output_df.numpy() , columns = ["Label"])
 output_df.index.name = "ImageId"
output_df.index = output_df.index + 1<save_to_csv>
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
output_df.to_csv(prediction_path )<import_modules>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
train_input = bert_encode(train.clean_tweet.values, tokenizer_QA, max_len=160)
 test_input = bert_encode(test.clean_tweet.values, tokenizer_QA, max_len=160)
 train_labels = train.target.values
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<categorify>
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
callbacks=[checkpoint],
batch_size=16
)
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
model.load_weights('model.h5')
 test_pred = model.predict(test_input )
<set_options><EOS> 
submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 submission['target'] = test_pred.round().astype(int)
 submission.to_csv('new_submission.csv', index=False )