Kaggle competition: Natural Language Processing with Disaster Tweets
x_on_train = trn_on_train
val_x_on_train = val_on_train
x_on_val = trn_on_val
val_x_on_val = val_on_val<prepare_x_and_y>
import tensorflow as tf
import operator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import *
import pandas as pd
import numpy as np
from gensim.models import Word2Vec, KeyedVectors
import gensim
import time
import random
import matplotlib.pyplot as plt
import string
import nltk
from tensorflow.keras.preprocessing.sequence import pad_sequences
import multiprocessing
import nltk
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from nltk import word_tokenize, pos_tag
import functools
from gensim.models.fasttext import FastText
from nltk.corpus import wordnet
import re
from nltk.tokenize.treebank import TreebankWordDetokenizer
from wordsegment import load, segment
y_val = val[LABEL].values<compute_test_metric>
disaster_text = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv", encoding='utf-8' )
def pr_train(y_i, y):
p = x_on_train[y==y_i].sum(0)
 return(p+3)/(( y==y_i ).sum() +3)
 def pr_val(y_i, y):
p = x_on_val[y==y_i].sum(0)
 return(p+0.2)/(( y==y_i ).sum() +0.2)
 y_train = train[LABEL].values
y_train_v = train_v[LABEL].values
r_train = np.log(pr_train(1,y_train)/ pr_train(0,y_train))
r_val = np.log(pr_val(1,y_train_v)/ pr_val(0,y_train_v))<find_best_model_class>
disaster_text['keyword'] = disaster_text['keyword'].astype(str)
 disaster_text.loc[disaster_text['keyword'] == 'nan','keyword'] = ''
x_nb_on_train = x_on_train.multiply(r_train)
 C_PARAMETERS = [1.5, 2, 4]
models_train = [LogisticRegression(C=c, dual=True, solver='liblinear')for c in C_PARAMETERS]
preds_val_on_train = []
for model in models_train:
model.fit(x_nb_on_train, y_train)
 p = model.predict_proba(val_x_on_train.multiply(r_train)) [:,1]
print(roc_auc_score(y_val, p))
preds_val_on_train.append(p)
 preds_ensemble_val_on_train = 2**(( np.log2(preds_val_on_train[1])+ np.log2(preds_val_on_train[2])) /2)
 print(roc_auc_score(y_val, preds_ensemble_val_on_train))<find_best_model_class>
disaster_text['location'] = disaster_text['location'].astype(str)
 disaster_text.loc[disaster_text['location'] == 'nan','location'] = ''
x_nb_on_val = x_on_val.multiply(r_val)
 models_train_v = [LogisticRegression(C=c, dual=True, solver='liblinear')for c in C_PARAMETERS]
preds_val_on_val = []
for model in models_train_v:
model.fit(x_nb_on_val, y_train_v)
 p = model.predict_proba(val_x_on_val.multiply(r_val)) [:,1]
print(roc_auc_score(y_val, p))
preds_val_on_val.append(p)
 preds_ensemble_val_on_val = 2**(( np.log2(preds_val_on_val[1])+ np.log2(preds_val_on_val[2])) /2)
 print(roc_auc_score(y_val, preds_ensemble_val_on_val))<predict_on_test>
df_mislabeled = disaster_text.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
df_mislabeled.index.tolist()
preds_val = [
models_train[1].predict_proba(val_x_on_train.multiply(r_train)) [:,1],
models_train[2].predict_proba(val_x_on_train.multiply(r_train)) [:,1],
models_train_v[1].predict_proba(val_x_on_val.multiply(r_val)) [:,1],
models_train_v[2].predict_proba(val_x_on_val.multiply(r_val)) [:,1],
]<compute_test_metric>
disaster_text['target_relabeled'] = disaster_text['target'].copy()
disaster_text.loc[disaster_text['text'] == 'like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit', 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == 'Hellfire is surrounded by desires so be careful and donÛªt let your desires control you!
disaster_text.loc[disaster_text['text'] == 'To fight bioterrorism sir.', 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == '.POTUS
disaster_text.loc[disaster_text['text'] == 'CLEARED:incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring', 'target_relabeled'] = 1
disaster_text.loc[disaster_text['text'] == '
disaster_text.loc[disaster_text['text'] == 'In
disaster_text.loc[disaster_text['text'] == 'Who is bringing the tornadoes and floods.Who is bringing the climate change.God is after America He is plaguing her
disaster_text.loc[disaster_text['text'] == 'RT NotExplained: The only known image of infamous hijacker D.B.Cooper.http://t.co/JlzK2HdeTG', 'target_relabeled'] = 1
disaster_text.loc[disaster_text['text'] == "Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...", 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == "wowo--=== 12000 Nigerian refugees repatriated from Cameroon", 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == "He came to a land which was engulfed in tribal war and turned it into a land of peace i.e.Madinah.
 disaster_text.loc[disaster_text['text'] == "Hellfire! We donÛªt even want to think about it or mention it so letÛªs not do anything that leads to it
disaster_text.loc[disaster_text['text'] == "The Prophet(peace be upon him)said 'Save yourself from Hellfire even if it is by giving half a date in charity.'", 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == "Caution: breathing may be hazardous to your health.", 'target_relabeled'] = 1
disaster_text.loc[disaster_text['text'] == "I Pledge Allegiance To The P.O.P.E.And The Burning Buildings of Epic City.??????", 'target_relabeled'] = 0
disaster_text.loc[disaster_text['text'] == "
disaster_text.loc[disaster_text['text'] == "that horrible sinking feeling when youÛªve been at home on your phone for a while and you realise its been on 3G this whole time", 'target_relabeled'] = 0
preds_val_ens = 2**np.mean([np.log2(p)for p in preds_val], axis = 0)
 print(roc_auc_score(y_val, preds_val_ens))<feature_engineering>
disaster_text['target'] = disaster_text['target_relabeled']
is_val_lang = test.lang.isin(['tr', 'es', 'it'] )<count_values>
def to_lower(x):
x = x.lower()
return(x)
 def tokenization(x):
x = word_tokenize(x)
 return(x)
 def get_pos(word):
tag = nltk.pos_tag([word])[0][1][0].upper()
tag_dict = {"J": wordnet.ADJ,
"N": wordnet.NOUN,
"V": wordnet.VERB,
"R": wordnet.ADV}
return tag_dict.get(tag, wordnet.NOUN)
 def stemming_and_lemmatization(x):
lemmatizer = WordNetLemmatizer()
y = [lemmatizer.lemmatize(w, get_pos(w)) for w in x]
return(y)
 def remove_stopwords(x):
return([word for word in x if word not in stopwords.words('english')])
 def remove_punctuations(x):
x = re.sub(re.compile('(( http|ftp|https):\/\/)?[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+
x = re.sub('[^\w\s]','',x)
 x = re.sub("[^a-zA-Z\s]+", '', x)
 return(x)
 def decontracted(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
def hashtags(phrase):
phrase = " ".join(segment(phrase))
return phrase
def spellCheck(phrase):
spell = Speller(lang='en')
 phrase = "".join([spell(w)+" " for w in phrase.split() ])
 return phrase
def text_preprocessing(x):
x = to_lower(x)
 x = decontracted(x)
 x = remove_punctuations(x)
 return(x )
np.sum(is_val_lang.values )<filter>
p_time1 = time.time()
count = 0
load()
disaster_text['preprocessed_text'] = disaster_text['text'].apply(text_preprocessing)
 p_time2 = time.time()
disaster_text.head(10 )
test_val_lang = test.loc[is_val_lang, COMMENT]
test_non_val_lang = test.loc[~is_val_lang, COMMENT] 
<categorify>
p_time2 - p_time1
test_val_lang_on_train = vec_train.transform(test_val_lang)
 test_nonval_lang_on_train = vec_train.transform(test_non_val_lang)
 test_val_lang_on_val = vec_val.transform(test_val_lang )<train_model>
glove2word2vec(glove_input_file="/kaggle/input/glove-pretrained/glove.6B.300d.txt", word2vec_output_file="gensim_glove_vectors.txt" )
test_val_lang_on_train, test_nonval_lang_on_train, test_val_lang_on_val<predict_on_test>
cores = multiprocessing.cpu_count()
w_time1 = time.time()
word2VecModel = gensim.models.KeyedVectors.load_word2vec_format('gensim_glove_vectors.txt', binary=False)
 w_time2 = time.time()
preds1_val_lang_on_train = models_train[0].predict_proba(test_val_lang_on_train.multiply(r_train)) [:,1]
preds2_val_lang_on_train = models_train[1].predict_proba(test_val_lang_on_train.multiply(r_train)) [:,1]
preds4_val_lang_on_train = models_train[2].predict_proba(test_val_lang_on_train.multiply(r_train)) [:,1]
preds1_nonval_lang_on_train = models_train[0].predict_proba(test_nonval_lang_on_train.multiply(r_train)) [:,1]
preds2_nonval_lang_on_train = models_train[1].predict_proba(test_nonval_lang_on_train.multiply(r_train)) [:,1]
preds4_nonval_lang_on_train = models_train[2].predict_proba(test_nonval_lang_on_train.multiply(r_train)) [:,1]<predict_on_test>
w_time2 - w_time1
preds1_val_lang_on_val = models_train_v[0].predict_proba(test_val_lang_on_val.multiply(r_val)) [:,1]
preds2_val_lang_on_val = models_train_v[1].predict_proba(test_val_lang_on_val.multiply(r_val)) [:,1]
preds4_val_lang_on_val = models_train_v[2].predict_proba(test_val_lang_on_val.multiply(r_val)) [:,1]<define_variables>
def word2token(word,model):
try:
return model.vocab[word].index
except KeyError:
return 0
preds_val = 2**(( np.log2(preds2_val_lang_on_train)+ np.log2(preds4_val_lang_on_train)+np.log2(preds2_val_lang_on_val)+ np.log2(preds4_val_lang_on_val)) / 4)
 preds_nonval = 2**(( np.log2(preds2_nonval_lang_on_train)+ np.log2(preds4_nonval_lang_on_train)) / 2 )<feature_engineering>
VALID_PER = 0.00
total_samples = set_x.shape[0]
n_val = int(VALID_PER * total_samples)
 n_train = total_samples - n_val
random_i = random.sample(range(total_samples), total_samples)
 train_x = set_x[random_i[:n_train]]
train_y = set_y[random_i[:n_train]]
val_x = set_x[random_i[n_train:n_train+n_val]]
val_y = set_y[random_i[n_train:n_train+n_val]]
print("Train Shapes - X: {} - Y: {}".format(train_x.shape, train_y.shape))
print("Val Shapes - X: {} - Y: {}".format(val_x.shape, val_y.shape))
categories, ccount = np.unique(train_y, return_counts=True)
 n_categories = len(categories )
test.loc[is_val_lang, 'toxic'] = preds_val
test.loc[~is_val_lang, 'toxic'] = preds_nonval<save_to_csv>
w2v_weights = word2VecModel.vectors
submission1 = test[['id', 'toxic', 'lang']]
submission1.to_csv('submission1.csv', index=False )<load_from_csv>
vocab_size, embedding_size = w2v_weights.shape
lstm_model = Sequential()
lstm_model.add(Embedding(input_dim=vocab_size,
output_dim=300,
weights=[w2v_weights],
input_length=700,
mask_zero=True,
trainable=True))
lstm_model.add(Bidirectional(LSTM(128, return_sequences = True)))
 lstm_model.add(Bidirectional(LSTM(128)))
 lstm_model.add(Dense(n_categories, activation='softmax'))
lstm_model
lstm_model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'] )
submission2 = pd.read_csv('.. /input/tpu-inference-super-fast-xlmroberta/submission.csv' )<feature_engineering>
history = lstm_model.fit(train_x, train_y, epochs=5, batch_size=256,
verbose=1 )
submission1['toxic'] = submission1['toxic'] * 0.04 + submission2['toxic'] * 0.96<feature_engineering>
test_data = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv", encoding='utf-8' )
submission1.loc[submission1["lang"] == "es", "toxic"] *= 1.06
submission1.loc[submission1["lang"] == "fr", "toxic"] *= 1.04
submission1.loc[submission1["lang"] == "it", "toxic"] *= 0.97
submission1.loc[submission1["lang"] == "pt", "toxic"] *= 0.96
submission1.loc[submission1["lang"] == "tr", "toxic"] *= 0.98<save_to_csv>
test_data['keyword'] = test_data['keyword'].astype(str)
 test_data.loc[test_data['keyword'] == 'nan','keyword'] = ''
submission1[['id', 'toxic']].to_csv('submission.csv', index=False )<import_modules>
test_data['location'] = test_data['location'].astype(str)
 test_data.loc[test_data['location'] == 'nan','location'] = ''
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import os<define_variables>
pt_time1 = time.time()
test_data['preprocessed_text'] = test_data['text'].apply(text_preprocessing)
 pt_time2 = time.time()
test_data.head(10 )
sub_path = ".. /input/jigsaw-multilingual-submission-files"
all_files = os.listdir(sub_path)
 all_files<categorify>
pt_time2 - pt_time1
all_files = [f for f in all_files if '0.9361' not in f and '0.9383' not in f and '0.9422' not in f]
all_files<load_from_csv>
test_data['target'] = 2
outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "jigsaw" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
ncol = concat_sub.shape[1]<feature_engineering>
def make_lstm_format_output(X):
string1 = X
string1 = pd.DataFrame(string1, columns =['preprocessed_text'])
 string1['target'] = "X"
sequence1 = SequenceIterator(string1, DROP_THRESHOLD, MAX_SEQUENCE_LENGTH)
 cat_dict = {k: v for k, v in zip(sequence1.categories, range(len(sequence1.categories)))}
set_x = []
set_y = []
for w, c in sequence1:
set_x.append(w)
 set_y.append(cat_dict[c])
 set_x = pad_sequences(set_x, maxlen=MAX_SEQUENCE_LENGTH, padding='pre', value=0)
 return set_x
concat_sub['jigsaw_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['jigsaw_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['jigsaw_mean'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)
 concat_sub['jigsaw_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )<define_variables>
test = make_lstm_format_output(test_data )
cutoff_lo = 0.3
cutoff_hi = 0.7<save_to_csv>
predicted_values = history.model.predict(test )
concat_sub['toxic'] = concat_sub['jigsaw_mean']
concat_sub[['id', 'toxic']].to_csv('submission1.csv',
index=False, float_format='%.6f' )<save_to_csv>
predicted_values = list(np.argmax(predicted_values, axis = 1))
concat_sub['toxic'] = concat_sub['jigsaw_median']
concat_sub[['id', 'toxic']].to_csv('submission2.csv',
index=False, float_format='%.6f' )<save_to_csv>
predictions = pd.merge(test_data[['id']], pd.DataFrame(predicted_values, columns = ['target']),left_index= True, right_index = True )
<feature_engineering><EOS> 
predictions.to_csv('Submission.csv',index = False )