Kaggle competition: Natural Language Processing with Disaster Tweets
%matplotlib inline 
<define_variables>
print("TF version: ", tf.__version__)
 print("Hub version: ", hub.__version__ )
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )<set_options>
pd.set_option('display.max_colwidth', None)
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu<load_pretrained>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 ids = test.id
print('Total length of the dataset: ', len(train)+len(test))
print('shape of training set: ', train.shape)
 print('shape of testing set: ', test.shape )
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )<load_pretrained>
df_concat = pd.concat([train, test], axis = 0 ).reset_index(drop = True)
 nulls = pd.DataFrame(np.c_[df_concat.isnull().sum() ,(df_concat.isnull().sum() / len(df_concat)) *100],
columns = ['
index = df_concat.columns)
 nulls
frames_per_video = 150
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )<define_variables>
for df in [train, test, df_concat]:
df.keyword.fillna('no_keyword', inplace = True)
 df.location.fillna('no_location', inplace = True )
input_size =224<normalization>
df_concat.groupby(['location'] ).count().text.sort_values(ascending = False )
mean = [0.43216, 0.394666, 0.37645]
std = [0.22803, 0.22145, 0.216989]
normalize_transform = Normalize(mean,std )<choose_model_class>
for df in [train, test, df_concat]:
df.drop(columns = ['location', 'keyword', 'id'], inplace = True )
class MyResNeXt(models.resnet.ResNet):
def __init__(self, training=True):
super(MyResNeXt, self ).__init__(block=models.resnet.Bottleneck,
layers=[3, 4, 6, 3],
groups=32,
width_per_group=4)
 self.fc = nn.Linear(2048, 1 )<load_pretrained>
nlp = spacy.load("en")
 sp = spacy.load('en_core_web_sm')
 nltk.download('stopwords')
 nltk.download('punkt')
 spacy_st = nlp.Defaults.stop_words
nltk_st = stopwords.words('english')
 def clean(tweet, http = True, punc = True, lem = True, stop_w = True):
if http is True:
tweet = re.sub("https?:\/\/t.co\/[A-Za-z0-9]*", '', tweet)
 if stop_w == 'nltk':
tweet = [word for word in word_tokenize(tweet)if not word.lower() in nltk_st]
tweet = ' '.join(tweet)
 elif stop_w == 'spacy':
tweet = [word for word in word_tokenize(tweet)if not word.lower() in spacy_st]
tweet = ' '.join(tweet)
 if lem == True:
lemmatized = [word.lemma_ for word in sp(tweet)]
tweet = ' '.join(lemmatized)
 if punc is True:
tweet = tweet.translate(str.maketrans('', '', string.punctuation))
tweet = re.sub("\s+", ' ', tweet)
 return tweet
checkpoint = torch.load("/kaggle/input/deepfakes-inference-demo/resnext.pth", map_location=gpu)
 model = MyResNeXt().to(gpu)
 model.load_state_dict(checkpoint)
 _ = model.eval()
del checkpoint<predict_on_test>
df_concat['cleaned_text'] = df_concat.text.apply(lambda x: clean(x, lem = False, stop_w = 'nltk', http = True, punc = True))
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.5<predict_on_test>
cleaned_train = df_concat[:train.shape[0]]
cleaned_test = df_concat[train.shape[0]:]
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )<define_variables>
%%HTML
<a id = "Word_Embeddings"></a>
<center>
<iframe width="700" height="315" src="https://www.youtube.com/embed/t5wdTK-QtLA" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" style="position: relative;top: 0;left: 0;" allowfullscreen ng-show="showvideo"></iframe>
</center>
speed_test = False<predict_on_test>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
FullTokenizer = tokenization.FullTokenizer
if speed_test:
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )<predict_on_test>
ans = input("Which Bert should I use?
a.Base uncased
b.Large uncased
c.Basic cased
d.Large cased
")
 if ans is 'a':
BERT_MODEL_HUB = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'
disc = 'Base_uncased'
elif ans is 'b':
BERT_MODEL_HUB = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/2'
disc = 'Large_uncased'
elif ans is 'c':
BERT_MODEL_HUB = 'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/2'
disc = 'Base_cased'
elif ans is 'd':
BERT_MODEL_HUB = 'https://tfhub.dev/tensorflow/bert_en_cased_L-24_H-1024_A-16/2'
disc = 'Large_cased'
bert_layer = hub.KerasLayer(BERT_MODEL_HUB, trainable=True)
 print('Bert layer is ready to use!')
 if ans =='a' or ans =='b':
to_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
vocabulary_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
tokenizer = FullTokenizer(vocabulary_file, to_lower_case)
 elif ans =='c' or ans =='d':
vocabulary_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
tokenizer = FullTokenizer(vocabulary_file, do_lower_case=False)
 print('Bert Tokenizer is ready!!!')
predictions = predict_on_video_set(test_videos, num_workers=4 )<save_to_csv>
sentence = 'Terrorist will crush the Tower'
print('Tokenized version of {} is :
{} '.format(sentence, tokenizer.tokenize(sentence)) )
submission_df = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df.to_csv("submission.csv", index=False )<install_modules>
def tokenize_tweets(text_):
return tokenizer.convert_tokens_to_ids(['[CLS]'] + tokenizer.tokenize(text_)+ ['[SEP]'])
 df_concat['tokenized_tweets'] = df_concat.cleaned_text.apply(lambda x: tokenize_tweets(x))
cleaned_train.head(2 )
!pip install.. /input/kaggle-efficientnet-repo/efficientnet-1.0.0-py3-none-any.whl<import_modules>
max_len = len(max(df_concat.tokenized_tweets, key = len))
print('The maximum length of each sequence besed on tokenized tweets is:', max_len)
 df_concat['padded_tweets'] = df_concat.tokenized_tweets.apply(lambda x: x + [0] *(max_len - len(x)))
 df_concat.head(2 )
import pandas as pd
import tensorflow as tf
import cv2
import glob
from tqdm.notebook import tqdm
import numpy as np
import os
from keras.layers import *
from keras import Model
import matplotlib.pyplot as plt
import time
from keras.applications.xception import Xception
import efficientnet.keras as efn<import_modules>
class TweetClassifier:
def __init__(self, tokenizer, bert_layer, max_len, lr = 0.0001,
epochs = 15, batch_size = 32,
activation = 'sigmoid', optimizer = 'SGD',
beta_1=0.9, beta_2=0.999, epsilon=1e-07,
metrics = 'accuracy', loss = 'binary_crossentropy'):
self.lr = lr
self.epochs = epochs
self.max_len = max_len
self.batch_size = batch_size
self.tokenizer = tokenizer
self.bert_layer = bert_layer
self.activation = activation
self.optimizer = optimizer
self.beta_1 = beta_1
self.beta_2 = beta_2
self.epsilon =epsilon
self.metrics = metrics
self.loss = loss
def encode(self, texts):
all_tokens = []
masks = []
segments = []
for text in texts:
tokenized = self.tokenizer.convert_tokens_to_ids(['[CLS]'] + self.tokenizer.tokenize(text)+ ['[SEP]'])
 len_zeros = self.max_len - len(tokenized)
 padded = tokenized + [0] * len_zeros
mask = [1] * len(tokenized)+ [0] * len_zeros
segment = [0] * self.max_len
all_tokens.append(padded)
 masks.append(mask)
 segments.append(segment)
 return np.array(all_tokens), np.array(masks), np.array(segments)
 def make_model(self):
input_ids = Input(shape =(self.max_len,), dtype = tf.int32, name = 'input_ids')
 input_mask = Input(shape =(self.max_len,), dtype = tf.int32, name = 'input_mask')
 segment_ids = Input(shape =(self.max_len,), dtype = tf.int32, name = 'segment_ids')
 pooled_output, sequence_output = bert_layer([input_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = tf.keras.layers.Dense(1, activation = self.activation )(clf_output)
 model = Model(inputs = [input_ids, input_mask, segment_ids], outputs = out)
 if self.optimizer is 'SGD':
optimizer = SGD(learning_rate = self.lr)
 elif self.optimizer is 'Adam':
optimizer = Adam(learning_rate = self.lr, beta_1=self.beta_1, beta_2=self.beta_2, epsilon=self.epsilon)
 model.compile(loss = self.loss, optimizer = self.optimizer, metrics = [self.metrics])
 print('Model is compiled with {} optimizer'.format(self.optimizer))
return model
def train(self, x):
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss',
save_best_only=True)
 model = self.make_model()
X = self.encode(x['cleaned_text'])
 Y = x['target']
model.fit(X, Y, shuffle = True, validation_split = 0.2,
batch_size=self.batch_size, epochs = self.epochs,
callbacks=[checkpoint])
 print('Model is fit!')
 def predict(self, x):
X_test_encoded = self.encode(x['cleaned_text'])
 best_model = tf.keras.models.load_model('model.h5',custom_objects={'KerasLayer':hub.KerasLayer})
 y_pred = best_model.predict(X_test_encoded)
 return y_pred
import torch
import torch.nn as nn
import torch.nn.functional as F<import_modules>
classifier = TweetClassifier(tokenizer = tokenizer, bert_layer = bert_layer,
max_len = max_len, lr = 0.0001,
epochs = 3, activation = 'sigmoid',
batch_size = 32,optimizer = 'SGD',
beta_1=0.9, beta_2=0.999, epsilon=1e-07 )
print("PyTorch version:", torch.__version__)
 print("CUDA version:", torch.version.cuda)
 print("cuDNN version:", torch.backends.cudnn.version() )<set_options>
classifier.train(cleaned_train )
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu<define_variables>
!git clone https://github.com/mitramir55/Kaggle_NLP_competition.git
perfection = pd.read_csv('Kaggle_NLP_competition/perfect_submission.csv' )
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )<load_pretrained>
y_pred = np.round(classifier.predict(cleaned_test))
print('The score of prediction: ', sklearn.metrics.f1_score(perfection.target, y_pred, average = 'micro'))
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )<define_variables>
sample_sub = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 ids = sample_sub.id
final_submission = pd.DataFrame(np.c_[ids, y_pred.astype('int')], columns = ['id', 'target'])
 final_submission.to_csv('final_submission.csv', index = False)
 final_submission.head()