Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<train_model>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from nltk.corpus import stopwords
from nltk.util import ngrams
from nltk.stem import WordNetLemmatizer
import re
from textblob import TextBlob
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, precision_score, recall_score, confusion_matrix
from sklearn.tree import DecisionTreeClassifier
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
import tensorflow as tf
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from gensim.models import Word2Vec
from gensim.models.keyedvectors import KeyedVectors
import time
from keras.layers import Dense, Input, Flatten, Dropout
from keras.layers import Conv1D, MaxPooling1D, Embedding
from keras.models import Sequential
from keras import losses
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
learn.to_fp16()
lr = 1e-2
moms =(0.8,0.7, 0.8)
 lr *= learn.dls.bs/128
learn.fit_one_cycle(1, lr, moms=moms, wd=0.1 )<train_model>
train_data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 submit_data = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
learn.freeze_to(-2)
 lr/=2
learn.fit_one_cycle(1, slice(lr/(2.6**4), lr), moms=moms, wd=0.1 )<train_model>
train_data[train_data['text'].isna() ]
learn.freeze_to(-3)
 lr /=2
learn.fit_one_cycle(1, slice(lr/(2.6**4), lr), moms=moms, wd=0.1 )<train_model>
train_data.groupby('target' ).count()
learn.unfreeze()
lr /= 5
learn.fit_one_cycle(2, slice(lr/(2.6**4),lr), moms=(0.8,0.7,0.8), wd=0.1 )<train_model>
%matplotlib inline
dl = learn.dls.test_dl(test_df['comment_text'] )<predict_on_test>
data = pd.concat([train_data, submit_data])
 data.shape
preds = learn.get_preds(dl=dl )<load_from_csv>
data['text'] = data['text'].apply(lambda x: re.sub(re.compile(r'https?\S+'), '', x))
data['text'] = data['text'].apply(lambda x: re.sub(re.compile(r'[\//:,.!?@&\-'\`"\_
\
data['text'] = data['text'].apply(lambda x: re.sub(re.compile(r'<.*?>'), '', x))
data['text'] = data['text'].apply(lambda x: re.sub(re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE), '', x))
data['text'] = data['text'].apply(lambda x: re.sub(re.compile(r'\d'), '', x))
data['text'] = data['text'].apply(lambda x: re.sub(re.compile(r'[^\w]'), ' ', x))
data['text'] = data['text'].str.lower()
sub = pd.read_csv(path_w/'sample_submission.csv' )<data_type_conversions>
clean_train = data[0:train_data.shape[0]]
clean_submit = data[train_data.shape[0]:-1]
X_train, X_test, y_train, y_test = train_test_split(clean_train['text'], clean_train['target'],
test_size = 0.2, random_state = 4 )
preds[0][0].cpu().numpy()<prepare_output>
def tfidf(words):
tfidf_vectorizer = TfidfVectorizer()
data_feature = tfidf_vectorizer.fit_transform(words)
 return data_feature, tfidf_vectorizer
X_train_tfidf, tfidf_vectorizer = tfidf(X_train.tolist())
 X_test_tfidf = tfidf_vectorizer.transform(X_test.tolist() )
sub[ys] = preds[0]<save_to_csv>
lr_tfidf = LogisticRegression(class_weight = 'balanced', solver = 'lbfgs', n_jobs = -1)
 lr_tfidf.fit(X_train_tfidf, y_train)
 y_predicted_lr = lr_tfidf.predict(X_test_tfidf )
sub.to_csv('submission.csv', index=False )<save_to_csv>
def score_metrics(y_test, y_predicted):
accuracy = accuracy_score(y_test, y_predicted)
 precision = precision_score(y_test, y_predicted)
 recall = recall_score(y_test, y_predicted)
 print("accuracy = %0.3f, precision = %0.3f, recall = %0.3f" %(accuracy, precision, recall))
sub.to_csv('submission.csv', index=False )<load_from_csv>
score_metrics(y_test, y_predicted_lr )
df=pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 df.head()<count_missing_values>
pipeline = Pipeline([ 
('clf', DecisionTreeClassifier(splitter='random', class_weight='balanced'))
])
 parameters = {
'clf__max_depth':(150,160,165),
'clf__min_samples_split':(18,20,23),
'clf__min_samples_leaf':(5,6,7)
 }
df_tfidf = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=-1, scoring='f1')
 df_tfidf.fit(X_train_tfidf, y_train)
 print(df_tfidf.best_estimator_.get_params() )
def missing_value_of_data(data):
total = data.isnull().sum().sort_values(ascending=False)
 precent=round(total/data.shape[0]*100,2)
 return pd.concat([total,precent],axis=1,keys=['Total','Percent'] )<count_missing_values>
y_predicted_dt = df_tfidf.predict(X_test_tfidf )
missing_value_of_data(df )<correct_missing_values>
score_metrics(y_test, y_predicted_dt )
df.dropna(inplace=True )<count_values>
!pip install gensim -i http://pypi.douban.com/simple --trusted-host pypi.douban.com
def count_values_in_columns(data,feature):
total=data.loc[:,feature].value_counts()
percentage = round(data.loc[:,feature].value_counts(normalize=True)*100,2)
 return pd.concat([total,percentage],axis=1,keys=['Total','Percentage'] )<count_values>
stop_words = stopwords.words('english')
 for word in ['us','no','yet']:
stop_words.append(word)
 data_list = []
text_series = data['text']
for i in range(len(text_series)) :
content = text_series.iloc[i]
cutwords = [word for word in content.split(' ')if word not in stop_words if len(word)!= 0]
data_list.append(cutwords )
count_values_in_columns(df,'sentiment' )<count_duplicates>
for i in range(len(data_list)) :
content = data_list[i]
if len(content)<1:
print(i )
def duplicated_values_data(data):
dup=[]
columns=data.columns
for i in columns:
dup.append(sum(data[i].duplicated()))
return pd.concat([pd.Series(columns),pd.Series(dup)],axis=1,keys=['Columns','Duplicate count'] )<count_duplicates>
word2vec_path='./GoogleNews-vectors-negative300.bin.gz'
word2vec_model = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True )
duplicated_values_data(df )<define_variables>
def get_textVector(data_list, word2vec, textsVectors_list):
for i in range(len(data_list)) :
words_perText = data_list[i]
if len(words_perText)< 1:
words_vector = [np.zeros(300)]
else:
words_vector = [word2vec.wv[k] if k in word2vec_model else np.zeros(300)for k in words_perText]
text_vector = np.array(words_vector ).mean(axis=0)
 textsVectors_list.append(text_vector)
 return textsVectors_list
def find_url(string):
try:
text = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +',string)
 except:
text=[]
return "".join(text )<feature_engineering>
textsVectors_list = []
get_textVector(data_list, word2vec_model, textsVectors_list)
 X = np.array(textsVectors_list )
df['url'] = df['text'].apply(lambda x : find_url(x))<categorify>
pd.isnull(X ).any()
def find_emoji(text):
emo_text=emoji.demojize(text)
 line=re.findall(r'\: (.*?)\:',emo_text)
 return line<feature_engineering>
word2vec_X = X[0:train_data.shape[0]]
y = data['target'][0:train_data.shape[0]]
word2vec_submit = X[train_data.shape[0]:-1]
X_train_word2vec, X_test_word2vec, y_train_word2vec, y_test_word2vec = train_test_split(word2vec_X, y,
test_size = 0.2, random_state = 4 )
sentence="I love ⚽ very much 😁"
find_emoji(sentence )<feature_engineering>
word2vec_lr = LogisticRegression(class_weight = 'balanced', solver = 'lbfgs', n_jobs = -1)
 word2vec_lr.fit(X_train_word2vec, y_train_word2vec)
 y_predicted_word2vec_lr = word2vec_lr.predict(X_test_word2vec )
df['emoji'] = df['text'].apply(lambda x: find_emoji(x))<drop_column>
score_metrics(y_test_word2vec, y_predicted_word2vec_lr )
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text )<drop_column>
compare_list = []
for(i,j)in zip(y_test_word2vec, y_predicted_word2vec_lr):
k = i - j
compare_list.append(k)
 wrong_num = [i for i,j in enumerate(compare_list)if j != 0]
text_series[0:train_data.shape[0]][wrong_num]
sentence="Its all about \U0001F600 face"
print(sentence)
 remove_emoji(sentence )<feature_engineering>
lenlen = []
for i in range(len(data_list)) :
content = data_list[i]
perlen = len(content)
 lenlen.append(perlen)
 print(max(lenlen))
df['text']=df['text'].apply(lambda x: remove_emoji(x))<string_transform>
max_sequence_length = 26
embedding_dim = 300
def stop_word_fn(text):
stop_words=set(stopwords.words('english'))
word_tokens= word_tokenize(text)
 non_stop_word=[w for w in word_tokens if not w in stop_words ]
stop_words = [w for w in word_tokens if w in stop_words]
return stop_words<drop_column>
tokenizer = Tokenizer()
tokenizer.fit_on_texts(data_list)
 sequences = tokenizer.texts_to_sequences(data_list)
 word_index = tokenizer.word_index
cnn_data = pad_sequences(sequences, maxlen = max_sequence_length)
 cnn_label = to_categorical(np.asarray(train_data['target']))
print('len of word_index:', len(word_index))
print('shape of data tensor:', cnn_data.shape)
 print('shape of label tensoe:', cnn_label.shape )
example_sent = "This is a sample sentence, showing off the stop words filtration."
stop_word_fn(example_sent )<feature_engineering>
trainCNN_data = cnn_data[0:train_data.shape[0]]
X_train_cnn, X_test_cnn, y_train_cnn, y_test_cnn = train_test_split(trainCNN_data, cnn_label,
test_size = 0.2, random_state = 4)
 X_cnn, X_val_cnn, y_cnn, y_val_cnn = train_test_split(X_train_cnn, y_train_cnn,
test_size = 0.2, random_state = 4 )
df['stop_words']=df['text'].apply(lambda x : stop_word_fn(x))<set_options>
CNNmodel = Sequential()
CNNmodel.add(Embedding(len(word_index)+1, embedding_dim, input_length = max_sequence_length))
CNNmodel.add(Conv1D(filters=250, kernel_size=3, strides=1, padding='valid', activation = 'relu'))
CNNmodel.add(MaxPooling1D(pool_size=3))
CNNmodel.add(Flatten())
 CNNmodel.add(Dense(embedding_dim, activation='relu'))
CNNmodel.add(Dropout(0.8))
CNNmodel.add(Dense(cnn_label.shape[1], activation='sigmoid'))
CNNmodel.summary()
stop = set(stopwords.words('english'))
warnings.filterwarnings(action="ignore")
 cufflinks.go_offline()
cufflinks.set_config_file(world_readable=True, theme='pearl')
<load_from_csv>
CNNmodel.compile(optimizer='adam', loss=losses.binary_crossentropy, metrics=['accuracy'])
 history = CNNmodel.fit(X_cnn, y_cnn, epochs=3, validation_data=(X_val_cnn, y_val_cnn))
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 train.sample(6 )<correct_missing_values>
test_loss, test_acc = CNNmodel.evaluate(X_test_cnn, y_test_cnn, verbose=2)
 print('test loss:',test_loss)
 print('test acc:',test_acc )
train.dropna()
train.shape<feature_engineering>
embedding_matrix = np.zeros(( len(word_index)+ 1, embedding_dim))
for word, i in word_index.items() :
if word in word2vec_model:
embedding_matrix[i] = np.asarray(word2vec_model.wv[word] )
train['target'] = train['selected_text'].str.lower()<feature_engineering>
embedding_layer = Embedding(len(word_index)+1,
embedding_dim,
weights = [embedding_matrix],
input_length = max_sequence_length,
trainable = False )
train['target_url'] =train['target'].apply(lambda x : find_url(x))<feature_engineering>
model = Sequential()
model.add(embedding_layer)
 model.add(Conv1D(filters=150, kernel_size=3, strides=1, padding='valid', activation = 'relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(Flatten())
 model.add(Dense(embedding_dim, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(cnn_label.shape[1], activation='sigmoid'))
model.summary()
def find_star(text):
try:
line=re.findall(r'[*]{2,5}',text)
 except:
line=[]
return len(line)
 train['star']=train['target'].apply(lambda x:find_star(x))<feature_engineering>
model.compile(optimizer='adam', loss=losses.binary_crossentropy, metrics=['accuracy'])
 history = model.fit(X_cnn, y_cnn, epochs=10, validation_data=(X_val_cnn, y_val_cnn))
def find_only_star(text):
try:
if len(text.split())==1:
line=re.findall(r'[*]{2,5}',text)
 return len(line)
 else:
return 0
except:
return 0
train['only_star']=train['target'].apply(lambda x:find_only_star(x))<feature_engineering>
test_loss, test_acc = model.evaluate(X_test_cnn, y_test_cnn, verbose=2)
 print('test loss:',test_loss)
 print('test acc:',test_acc )
train['target']= np.where(train['only_star']==1,"abusive",train['target'] )<feature_engineering>
tf.__version__
def remove_link(string):
try:
text = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +'," ",string)
 except:
text=''
return " ".join(text.split())
 def remove_punct(text):
try:
line = re.sub(r'[!"\$%&'() *+,\-.\/:;=
except:
line=''
return " ".join(line.split())
 train['target']=train['target'].apply(lambda x:remove_link(x))
train['target']=train['target'].apply(lambda x:remove_punct(x))
train['target_tweet_length']=train['target'].str.split().map(lambda x: len(x))<feature_engineering>
hub.__version__
train['target_average_word_len']=train['target'].str.split().apply(lambda x : [len(i)for i in x] ).map(lambda x: np.mean(x))<feature_engineering>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
full_data=pd.concat([train,test])
 full_data['text']=full_data['text'].str.lower()
full_data.shape
full_data['text']=full_data['text'].apply(lambda x:remove_link(x))
full_data['text']=full_data['text'].apply(lambda x:remove_punct(x))<filter>
import tensorflow as tf
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
full_data.loc[full_data['text']=="",['text']]="nothing"<feature_engineering>
def bert_encode(texts, bert_layer, max_len=128):
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len - 2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 input_ids = tokens + [0]* pad_len
all_tokens.append(input_ids)
 masks = [1]*len(input_sequence)+ [0]* pad_len
all_masks.append(masks)
 segments = [0]* max_len
all_segments.append(segments)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(bert_layer, max_len = 128, lr = 1e-5):
input_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,name="input_word_ids")
 input_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,name="input_mask")
 segment_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,name="segment_ids")
 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 dense_out = Dense(1,activation="relu" )(pooled_output)
 drop_out = tf.keras.layers.Dropout(0.8 )(dense_out)
 out = Dense(1,activation="sigmoid" )(pooled_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 adam = tf.keras.optimizers.Adam(lr)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
 return model
def plot_curve(history):
plt.plot(history.history['accuracy'], label='accuracy')
 plt.plot(history.history['val_accuracy'], label='val_accuracy')
 plt.xlabel('Epoch')
 plt.ylabel('Accuracy')
 plt.ylim([0.5,1])
 plt.legend()
plt.show()
full_data['text_tweet_length']=full_data['text'].str.split().map(lambda x: len(x))
full_data['text_average_word_len']=full_data['text'].str.split().apply(lambda x : [len(i)for i in x] ).map(lambda x: np.mean(x))<prepare_x_and_y>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
def corpus_sentiment_stop(data,feature,sentiment):
corpus=create_corpus(data,feature,sentiment)
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)
 x,y=zip(*top)
 return x,y<define_variables>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 train_input = bert_encode(train.text.values, bert_layer, max_len=128)
 train_labels = np.array(train.target )
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<load_from_csv>
test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 test_input = bert_encode(test.text.values, bert_layer, max_len=128)
 model.load_weights('model.h5')
 test_pred = model.predict(test_input )
<load_from_csv><EOS> 
submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 submission['target'] = np.round(test_pred ).astype('int')
 submission.to_csv('submission.csv', index=False)
 submission.groupby('target' ).count()