Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<choose_model_class>
%matplotlib inline
InteractiveShell.ast_node_interactivity = 'all'
!pip install chart_studio
plotly.offline.init_notebook_mode(connected=True)
 cufflinks.go_offline()
cufflinks.set_config_file(world_readable=True, theme='pearl')
 warnings.filterwarnings('ignore' )
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<data_type_conversions>
data = pd.read_csv('.. /input/nlp-getting-started/train.csv' )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()<choose_model_class>
def create_corpus(target):
corpus = []
for i in data[data['target']==target]['text'].str.split() :
for x in i:
corpus.append(x)
 return corpus
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
<categorify>
lemmatizer = WordNetLemmatizer()
def preprocess_data(data):
text = re.sub(r'https?://\S+|www\.\S+|http?://\S+',' ',data)
 text = re.sub(r"won't", " will not", text)
 text = re.sub(r"won't've", " will not have", text)
 text = re.sub(r"can't", " can not", text)
 text = re.sub(r"don't", " do not", text)
 text = re.sub(r"can't've", " can not have", text)
 text = re.sub(r"ma'am", " madam", text)
 text = re.sub(r"let's", " let us", text)
 text = re.sub(r"ain't", " am not", text)
 text = re.sub(r"shan't", " shall not", text)
 text = re.sub(r"sha
't", " shall not", text)
 text = re.sub(r"o'clock", " of the clock", text)
 text = re.sub(r"y'all", " you all", text)
 text = re.sub(r"n't", " not", text)
 text = re.sub(r"n't've", " not have", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r"'s", " is", text)
 text = re.sub(r"'d", " would", text)
 text = re.sub(r"'d've", " would have", text)
 text = re.sub(r"'ll", " will", text)
 text = re.sub(r"'ll've", " will have", text)
 text = re.sub(r"'t", " not", text)
 text = re.sub(r"'ve", " have", text)
 text = re.sub(r"'m", " am", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r'<.*?>',' ',text)
 text = re.sub(r'[0-9]', '', text)
 text = re.sub("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+",' ',text)
 text = re.sub('[^a-zA-Z]',' ',text)
 text = re.sub(r"\([^() ]*\)", "", text)
 text = re.sub('@\S+', '', text)
 text = re.sub('[%s]' % re.escape (
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values<train_model>
common_words = ['via','like','build','get','would','one','two','feel',
'lol','fuck','take','way','may','first','latest','want',
'make','back','see','know','let','look','come','got',
'still','say','think','great','pleas','amp']
def text_cleaning(data):
return ' '.join(i for i in data.split() if i not in common_words)
 data["Cleaned_text"] = data["Cleaned_text"].apply(text_cleaning )
train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
batch_size=16
)<predict_on_test>
def top_ngrams(data,n,grams):
count_vec = CountVectorizer(ngram_range=(grams,grams)).fit(data)
 bow = count_vec.transform(data)
 add_words = bow.sum(axis=0)
 word_freq = [(word, add_words[0, idx])for word, idx in count_vec.vocabulary_.items() ]
word_freq = sorted(word_freq, key = lambda x: x[1], reverse=True)
 return word_freq[:n]
test_pred = model.predict(test_input )<save_to_csv>
common_uni = top_ngrams(data["Cleaned_text"],10,1)
 common_bi = top_ngrams(data["Cleaned_text"],10,2)
 common_tri = top_ngrams(data["Cleaned_text"],10,3)
 common_uni_df = pd.DataFrame(common_uni,columns=['word','freq'])
 common_bi_df = pd.DataFrame(common_bi,columns=['word','freq'])
 common_tri_df = pd.DataFrame(common_tri,columns=['word','freq'] )
submission=pd.DataFrame()
submission['Id']=test_id
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission_3.csv', index=False)
<set_options>
X_inp_clean = data['Cleaned_text']
X_inp_original = data['text']
y_inp = data['target']
np.random.seed(1)
 nltk.download('stopwords')
 tf.random.set_seed(1)
 pd.set_option('display.max_colwidth', 500)
 warnings.filterwarnings('ignore' )<load_from_csv>
word_tokenizer = Tokenizer()
word_tokenizer.fit_on_texts(X_inp_clean.values)
 vocab_length = len(word_tokenizer.word_index)+ 1
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 print("Train Shape :", train.shape)
 print("Test Shape :", test.shape )<count_missing_values>
def embed(corpus):
return word_tokenizer.texts_to_sequences(corpus)
 longest_train = max(X_inp_clean.values, key=lambda sentence: len(word_tokenize(sentence)))
 length_long_sentence = len(word_tokenize(longest_train))
padded_sentences = pad_sequences(embed(X_inp_clean.values),
length_long_sentence, padding='post' )
train.isnull().sum()<count_missing_values>
embeddings_dictionary = dict()
embedding_dim = 100
glove_file = open('.. /input/glove6b100dtxt/glove.6B.100d.txt')
 for line in glove_file:
records = line.split()
word = records[0]
vector_dimensions = np.asarray(records[1:], dtype='float32')
 embeddings_dictionary [word] = vector_dimensions
glove_file.close()
train.isnull().sum()<count_values>
embedding_matrix = np.zeros(( vocab_length, embedding_dim))
for word, index in word_tokenizer.word_index.items() :
embedding_vector = embeddings_dictionary.get(word)
 if embedding_vector is not None:
embedding_matrix[index] = embedding_vector
train['target'].value_counts(normalize = True )<count_values>
X_train, X_val, y_train, y_val = train_test_split(padded_sentences,
y_inp.values,test_size=0.2,random_state=1 )
train.keyword.value_counts()<count_values>
def CNN(hp):
model = keras.Sequential()
hp_learning_rate = hp.Choice('learning_rate', values=[3e-2, 3e-3, 3e-4, 3e-5])
 model.add(Embedding(vocab_length, 100, weights=[embedding_matrix],
input_length=length_long_sentence,trainable=False))
model.add(Conv1D(filters=hp.Int('conv_1_filter',min_value=21,max_value=200,step=14),
kernel_size=hp.Choice('conv_1_kernel',values=[3,4,5]),
activation='relu'))
model.add(Dropout(0.3))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
 model.add(Dense(units = hp.Int('dense_1',min_value=21,max_value=150,step=14),
activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1,activation='sigmoid'))
model.compile(optimizer=keras.optimizers.Adam(learning_rate=hp_learning_rate),
loss=keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'])
 return model
train.location.value_counts()<filter>
tuner_CNN = kt.Hyperband(CNN,objective='val_accuracy',
max_epochs=15,factor=5,
directory='my_dir',
project_name='DisasterTweets_kt',
overwrite=True )
real_tweets = train[train['target']==1]['text']
real_tweets.values[0:5]<define_variables>
stop_early = EarlyStopping(monitor='val_loss', mode='min',
verbose=1, patience=10)
 tuner_CNN.search(X_train, y_train, epochs=15,
validation_data=(X_val,y_val),callbacks=[stop_early])
 best_hps_CNN=tuner_CNN.get_best_hyperparameters(num_trials=1)[0]
fake_tweets = train[train['target']==0]['text']
fake_tweets.values[0:5]<string_transform>
model_CNN = tuner_CNN.hypermodel.build(best_hps_CNN)
 checkpoint = ModelCheckpoint(
 'model_CNN.h5',
monitor = 'val_loss',
verbose = 1,
save_best_only = True 
)
 history_CNN = model_CNN.fit(X_train, y_train,epochs=50,
validation_data=(X_val,y_val),
callbacks=[checkpoint,stop_early] )
def clean_text(text):
text = text.lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text<feature_engineering>
def MultichannelCNN(hp):
inputs1 = Input(shape=(length_long_sentence,))
embedding1 = Embedding(vocab_length, 100, weights=[embedding_matrix],
input_length=length_long_sentence, trainable=False )(inputs1)
 conv1 = Conv1D(filters=hp.Int('conv_1_filter',min_value=21,max_value=150,step=14),
kernel_size=hp.Choice('conv_1_kernel',values=[3,4,5,6,7,8]),
activation='relu' )(embedding1)
 drop1 = Dropout(0.3 )(conv1)
 pool1 = MaxPooling1D()(drop1)
 flat1 = Flatten()(pool1)
 inputs2 = Input(shape=(length_long_sentence,))
embedding2 = Embedding(vocab_length, 100, weights=[embedding_matrix],
input_length=length_long_sentence,trainable=False )(inputs2)
 conv2 = Conv1D(filters=hp.Int('conv_2_filter',min_value=21,max_value=150,step=14),
kernel_size=hp.Choice('conv_2_kernel',values=[3,4,5,6,7,8]),
activation='relu' )(embedding2)
 drop2 = Dropout(0.3 )(conv2)
 pool2 = MaxPooling1D()(drop2)
 flat2 = Flatten()(pool2)
 merged = concatenate([flat1, flat2])
 dense1 = Dense(units = hp.Int('dense_1',min_value=21,max_value=120,step=14),
activation='relu' )(merged)
 drop4 = Dropout(0.5 )(dense1)
 outputs = Dense(1, activation='sigmoid' )(drop4)
 model = Model(inputs=[inputs1, inputs2], outputs=outputs)
 hp_learning_rate = hp.Choice('learning_rate', values=[3e-2, 3e-3, 3e-4, 3e-5])
 model.compile(optimizer=keras.optimizers.Adam(learning_rate=hp_learning_rate),
loss=keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'])
 return model
train['cleaned_text'] = train['text'].apply(lambda x: clean_text(x))
test['cleaned_text'] = test['text'].apply(lambda x: clean_text(x))
train['cleaned_text'].head()<feature_engineering>
tuner_MCNN = kt.Hyperband(MultichannelCNN,objective='val_accuracy',
max_epochs=15,factor=5,
directory='my_dir',
project_name='DisasterTweetsMCNN_kt',
overwrite=True)
 stop_early = EarlyStopping(monitor='val_loss', mode='min',
verbose=1, patience=10)
 tuner_MCNN.search([X_train,X_train], y_train, epochs=15,
validation_data=([X_val,X_val], y_val),
callbacks=[stop_early])
 best_hps_MCNN=tuner_MCNN.get_best_hyperparameters(num_trials=1)[0]
!pip install nlppreprocess
nlp = NLP()
train['stopwords_cleaned'] = train['cleaned_text'].apply(nlp.process)
 test['stopwords_cleaned'] = test['cleaned_text'].apply(nlp.process )<categorify>
model_MCNN = tuner_MCNN.hypermodel.build(best_hps_MCNN)
 checkpoint = ModelCheckpoint(
 'model_MCNN.h5',
monitor = 'val_loss',
verbose = 1,
save_best_only = True 
)
 history_MCNN = model_MCNN.fit([X_train,X_train], y_train,epochs=50,
validation_data=([X_val,X_val], y_val),
callbacks=[checkpoint,stop_early] )
en_model = spacy.load('en', disable=['parser', 'ner'])
 def lemmatization(texts):
output = []
for i in texts:
s = [token.lemma_ for token in en_model(i)]
output.append(' '.join(s))
return output<categorify>
def BiLSTM(hp):
model = Sequential()
model.add(Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights = [embedding_matrix],
input_length=length_long_sentence,trainable = False))
model.add(Bidirectional(CuDNNLSTM(units = hp.Int('dense_1',
min_value=21,max_value=120,step=14)
 ,return_sequences = True)))
 model.add(GlobalMaxPool1D())
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(units = hp.Int('dense_1',min_value=21,
max_value=120,step=14),
activation = "relu"))
model.add(Dropout(0.3))
model.add(Dense(units = hp.Int('dense_1',min_value=21,
max_value=100,step=14),
activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))
hp_learning_rate = hp.Choice('learning_rate',
values=[3e-2, 3e-3, 3e-4, 3e-5])
 model.compile(optimizer=keras.optimizers.Adam(learning_rate=hp_learning_rate),
loss=keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'])
 return model
train['lemmatized_text'] = lemmatization(train['stopwords_cleaned'])
 test['lemmatized_text'] = lemmatization(test['stopwords_cleaned'] )<split>
tuner_BiLSTM = kt.Hyperband(BiLSTM,objective='val_accuracy',
max_epochs=15,factor=5,
directory='my_dir',
project_name='DisasterTweetsBiLSTM_kt',
overwrite=True)
 stop_early = EarlyStopping(monitor='val_loss', mode='min',
verbose=1, patience=12)
 tuner_BiLSTM.search(X_train, y_train, epochs=15,
validation_data=(X_val, y_val),
callbacks=[stop_early])
 best_hps_BiLSTM=tuner_BiLSTM.get_best_hyperparameters(num_trials=1)[0]
x_train, x_test, y_train, y_test = train_test_split(train['stopwords_cleaned'], train['target'],
test_size = 0.2, random_state = 1 )<choose_model_class>
model_BiLSTM = tuner_BiLSTM.hypermodel.build(best_hps_BiLSTM)
 checkpoint = ModelCheckpoint(
 'model_BiLSTM.h5',
monitor = 'val_loss',
verbose = 1,
save_best_only = True 
)
 history_BiLSTM = model_BiLSTM.fit(X_train, y_train, epochs=50,
validation_data=(X_val, y_val),
callbacks=[checkpoint,stop_early] )
hub_layer = hub.KerasLayer('https://tfhub.dev/google/universal-sentence-encoder/4',
input_shape = [],
output_shape = [512],
dtype = tf.string,
trainable = True)
 model = tf.keras.models.Sequential()
model.add(hub_layer)
 model.add(tf.keras.layers.Dense(128, activation = 'relu'))
model.add(tf.keras.layers.Dense(32, activation = 'relu'))
model.add(tf.keras.layers.Dense(1, activation = 'sigmoid'))
model.summary()<choose_model_class>
onehot_encoder = OneHotEncoder(sparse=False)
 y =(np.asarray(y_inp)).reshape(-1,1)
 Y = onehot_encoder.fit_transform(y)
 X_train, X_val, y_train, y_val = train_test_split(X_inp_clean,Y,
test_size=0.2, random_state=1 )
model.compile(optimizer = 'adam',
loss = 'binary_crossentropy',
metrics = ['accuracy'] )<train_model>
model_checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True )
model.fit(x_train,
y_train,
epochs = 1,
validation_data =(x_test, y_test))<predict_on_test>
tokenizer("Hello, this one sentence!", "And this sentence goes with it." )
pred = model.predict_classes(test['stopwords_cleaned'] )<save_to_csv>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen,
add_special_tokens = True,
truncation=True 
)
 return np.array(enc_di['input_ids'])
 X_train_t = regular_encode(list(X_train), tokenizer, maxlen=512)
 X_val_t = regular_encode(list(X_val), tokenizer, maxlen=512 )
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 submission['target'] = pred
submission.to_csv('submission.csv', index=False )<save_to_csv>
AUTO = tf.data.experimental.AUTOTUNE
batch_size = 16
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( X_train_t, y_train)) 
.repeat() 
.shuffle(1995)
.batch(batch_size)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( X_val_t, y_val)) 
.batch(batch_size)
.cache() 
.prefetch(AUTO)
 )
submission.to_csv("submission.csv", index = False )<set_options>
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32,
name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(2, activation='softmax' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(optimizer=keras.optimizers.Adam(lr=1e-5),
loss='categorical_crossentropy', metrics=['accuracy'])
 return model
np.random.seed(0)
 plt.style.use('ggplot')
 stop=set(stopwords.words('english'))
np.random.seed(1)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
transformer_layer = TFAutoModel.from_pretrained(model_checkpoint)
 model_DistilBert = build_model(transformer_layer )
train= pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test=pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train.head()<count_missing_values>
n_steps = X_train.shape[0] // batch_size
history_DistilBert = model_DistilBert.fit(train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=3 )
train.isnull().sum(axis=0 )<count_missing_values>
test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 test["Cleaned_text"] = test["text"].apply(preprocess_data)
 test["Cleaned_text"] = test["Cleaned_text"].apply(text_cleaning)
 test_sentences = pad_sequences(embed(test.Cleaned_text.values),
length_long_sentence, padding='post' )
test.isnull().sum(axis=0 )<count_values>
predsCNN = model_CNN.predict_classes(test_sentences)
 predictions_test = pd.DataFrame(predsCNN)
 test_id = pd.DataFrame(test["id"])
 submissionCNN = pd.concat([test_id,predictions_test],axis=1)
 submissionCNN.columns = ["id","target"]
submissionCNN.to_csv("submissionCNN.csv",index=False )
keyword_cnt = train.keyword.value_counts()
keyword_cnt<count_values>
predsMCNN = model_MCNN.predict([test_sentences,test_sentences])
 predsMCNN =(predsMCNN[:,0] > 0.5 ).astype(np.int)
 predictions_test = pd.DataFrame(predsMCNN)
 submissionMCNN = pd.concat([test_id,predictions_test],axis=1)
 submissionMCNN.columns = ["id","target"]
submissionMCNN.to_csv("submissionMCNN.csv",index=False )
train_fake = train[train['target'] == 1]
keyword_cnt_fake = train_fake.keyword.value_counts()
keyword_cnt_fake<string_transform>
predsBiLSTM = model_BiLSTM.predict(test_sentences)
 predsBiLSTM =(predsBiLSTM[:,0] > 0.5 ).astype(np.int)
 predictions_test = pd.DataFrame(predsBiLSTM)
 submissionBiLSTM = pd.concat([test_id,predictions_test],axis=1)
 submissionBiLSTM.columns = ["id","target"]
submissionBiLSTM.to_csv("submissionBiLSTM.csv",index=False )
n_corpus=[]
for text in tqdm(train['text']):
text = re.sub(r'https?://\S+|www\.\S+', '', text)
 text = re.sub(r'<.*?>', '', text)
 text = re.sub(r'[^a-zA-Z0-9]+', ' ', text)
 text = re.sub(r'[0-9]', '', text)
 text = text.lower()
text = nltk.word_tokenize(text)
 ps = PorterStemmer()
text = [ps.stem(word)for word in text if not word in set(stopwords.words('english')) ]
text = ' '.join(text)
 n_corpus.append(text )<drop_column>
X_test = regular_encode(list(test.Cleaned_text), tokenizer, maxlen=512)
 test1 =(tf.data.Dataset.from_tensor_slices(X_test ).batch(batch_size))
pred = model_DistilBert.predict(test1,verbose = 0)
 pred = np.argmax(pred,axis=-1)
 pred = pred.astype('int32')
 res=pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv',index_col=None)
 res['target'] = pred
res.to_csv('submissionDistilBert.csv',index=False )