Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<save_to_csv>
import pandas as pd
import numpy as np
import math
import re
from bs4 import BeautifulSoup
import random
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<train_model>
!pip install bert-for-tf2
!pip install sentencepiece
EDA_LB_val =.73395
print("EDA_LB_val: {}".format(EDA_LB_val))
DEEP_LB_val =.79336
print("DEEP_LB_val: {}".format(DEEP_LB_val))
WORD_LB_val =.78723
print("WORD_LB_val: {}".format(WORD_LB_val))<define_variables>
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow.keras import layers
import bert
almost_zero = 1e-10
almost_one = 1-almost_zero<load_from_csv>
data=pd.read_csv('.. /input/nlp-getting-started/train.csv' )
EDA_input_file = ".. /input/fork-of-donorschoose-lr-model-cat-smote/LR_stack_glove100_submission.csv"
DEEP_input_file = ".. /input/fork-of-donorschoose-full-cnn-yes-smote/deep_model_fasttext300_fullSMOTE_submission.csv"
WORD_input_file = ".. /input/donorschoose-word-model/WORD_model_smote_fas_submission.csv"
EDA_df = pd.read_csv(EDA_input_file ).rename(columns={'project_is_approved': 'EDA_project_is_approved'})
 DEEP_df = pd.read_csv(DEEP_input_file ).rename(columns={'project_is_approved': 'DEEP_project_is_approved'})
 WORD_df = pd.read_csv(WORD_input_file ).rename(columns={'project_is_approved': 'WORD_project_is_approved'} )<merge>
data.drop(columns=['id','keyword','location'],inplace=True )
ensemble_df = pd.merge(EDA_df, DEEP_df, on='id')
 ensemble_df = pd.merge(ensemble_df, WORD_df, on='id' )<feature_engineering>
def clean_tweet(tweet):
tweet=BeautifulSoup(tweet,'lxml' ).get_text()
tweet= re.sub(r'@[A-Za-z0-9]+',' ',tweet)
 tweet=re.sub(r'https?://[A-Za-z0-9./]+',' ',tweet)
 tweet=re.sub(r'[^a-zA-Z.!?]',' ',tweet)
 tweet=re.sub(r' +',' ',tweet)
 return tweet
power = 68
EDA_weights = EDA_LB_val ** power
DEEP_weights = DEEP_LB_val ** power
WORD_weights = WORD_LB_val ** power
EDA_numbers = ensemble_df['EDA_project_is_approved'].clip(almost_zero,almost_one ).apply(logit)* EDA_weights
DEEP_numbers = ensemble_df['DEEP_project_is_approved'].clip(almost_zero,almost_one ).apply(logit)* DEEP_weights
WORD_numbers = ensemble_df['WORD_project_is_approved'].clip(almost_zero,almost_one ).apply(logit)* WORD_weights
totalweights = EDA_weights + DEEP_weights + WORD_weights
ensemble_df['project_is_approved'] =(EDA_numbers + DEEP_numbers + WORD_numbers)/ totalweights
ensemble_df['project_is_approved'] = ensemble_df['project_is_approved'].apply(expit )<save_to_csv>
data_clean=data['text'].apply(lambda text:clean_tweet(text))
ensemble_df[['id', 'project_is_approved']].to_csv("ensemble_EDA_plus_DEEPfasttext300_plus_WORDfasttext_submission.csv", index=False )<set_options>
data_labels=data.target.values
stop = set(stopwords.words('english'))
tf.logging.set_verbosity(tf.logging.ERROR)
 pd.options.display.max_rows = 10
pd.options.display.float_format = '{:.1f}'.format<load_from_csv>
fulltokenizer=bert.bert_tokenization.FullTokenizer
bert_layer=hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1' )
train = pd.read_csv(".. /input/train.csv", sep=",", parse_dates=['project_submitted_datetime'])
 resources = pd.read_csv(".. /input/resources.csv", sep=",")
 test = pd.read_csv(".. /input/test.csv", sep=",", parse_dates=['project_submitted_datetime'] )<merge>
vocab_file=bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case=bert_layer.resolved_object.do_lower_case.numpy()
tokenizer=fulltokenizer(vocab_file,do_lower_case )
resources['cost'] = resources['quantity'] * resources['price']
resources_aggregated = resources.groupby('id' ).agg({'description': ['nunique'], 'quantity': ['sum'], 'cost': ['mean', 'sum']})
 resources_aggregated.columns = ['unique_items', 'total_quantity', 'mean_cost', 'total_cost']
resources_aggregated.reset_index(inplace=True)
 train = pd.merge(train, resources_aggregated, how='left', on='id')
 test = pd.merge(test, resources_aggregated, how='left', on='id' )<feature_engineering>
def encode_sentence(sent):
return ['[CLS]']+tokenizer.tokenize(sent)+['[SEP]']
train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] = train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] + ' ' + train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2']
train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2'] = train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_3'] + ' ' + train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_4']
train.drop(['project_essay_3', 'project_essay_4'], axis=1, inplace=True)
 test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] = test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] + ' ' + test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2']
test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2'] = test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_3'] + ' ' + test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_4']
test.drop(['project_essay_3', 'project_essay_4'], axis=1, inplace=True)
 train['project_essay_1'] = train['project_essay_1'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_essay_2'] = train['project_essay_2'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_essay_1'] = test['project_essay_1'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_essay_2'] = test['project_essay_2'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_resource_summary'] = train['project_resource_summary'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_title'] = train['project_title'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_resource_summary'] = test['project_resource_summary'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_title'] = test['project_title'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))<define_variables>
data_inputs=[encode_sentence(sent)for sent in data_clean]
def get_quantile_based_boundaries(feature_values, num_buckets):
boundaries = np.arange(1.0, num_buckets)/ num_buckets
quantiles = feature_values.quantile(boundaries)
 return [quantiles[q] for q in quantiles.keys() ]<feature_engineering>
def get_ids(token):
return tokenizer.convert_tokens_to_ids(token)
 def get_masks(token):
return np.char.not_equal(token,['PAD'] ).astype('int')
 def get_segments(token):
seg_ids=[]
current_id=0
for tok in token:
seg_ids.append(current_id)
 if tok=='[SEP]':
current_id=1-current_id
return seg_ids
bins = [0., 2., 10.,]
test["experience"] = np.digitize(test['teacher_number_of_previously_posted_projects'], bins=bins)
 quantiles = get_quantile_based_boundaries(test['unique_items'], 6)
 test["unique_items_binned"] = np.digitize(test['unique_items'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['total_quantity'], 6)
 test["total_quantity_binned"] = np.digitize(test['total_quantity'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['mean_cost'], 6)
 test["mean_cost_binned"] = np.digitize(test['mean_cost'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['total_cost'], 6)
 test["total_cost_binned"] = np.digitize(test['total_cost'], bins=quantiles )<define_variables>
data_with_len = [[sent, data_labels[i], len(sent)]
for i, sent in enumerate(data_inputs)]
random.shuffle(data_with_len)
 data_with_len.sort(key=lambda x: x[2])
 sorted_all = [([get_ids(sent_lab[0]),
get_masks(sent_lab[0]),
get_segments(sent_lab[0])],
sent_lab[1])
 for sent_lab in data_with_len if sent_lab[2] > 7]
def preprocess_features(df):
selected_features = df[
["project_subject_categories",
"project_subject_subcategories",
"project_title",
"project_resource_summary",
"project_essay_1",
"project_essay_2",
"experience",
"unique_items_binned",
"total_quantity_binned",
"mean_cost_binned",
"total_cost_binned",
"encoded_category",
"encoded_subcategory"]]
return selected_features
def preprocess_targets(df):
output_targets = df["project_is_approved"]
return output_targets<categorify>
all_dataset = tf.data.Dataset.from_generator(lambda: sorted_all,
output_types=(tf.int32, tf.int32))
def target_encode(trn_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):
assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return ft_trn_series, ft_tst_series<categorify>
BATCH_SIZE = 32
all_batched = all_dataset.padded_batch(BATCH_SIZE,
padded_shapes=(( 3, None),()),
padding_values=(0, 0))
train['encoded_category'], test['encoded_category'] = target_encode(train['project_subject_categories'], test['project_subject_categories'], train['project_is_approved'])
 train['encoded_subcategory'], test['encoded_subcategory'] = target_encode(train['project_subject_subcategories'], test['project_subject_subcategories'], train['project_is_approved'] )<randomize_order>
NB_BATCHES = math.ceil(len(sorted_all)/ BATCH_SIZE)
 NB_BATCHES_TEST = NB_BATCHES // 10
all_batched.shuffle(NB_BATCHES)
 test_dataset = all_batched.take(NB_BATCHES_TEST)
 train_dataset = all_batched.skip(NB_BATCHES_TEST )
train = train.reindex(
 np.random.permutation(train.index))<define_variables>
class DCNNBERTEmbedding(tf.keras.Model):
def __init__(self,
nb_filters=50,
FFN_units=512,
nb_classes=2,
dropout_rate=0.1,
name="dcnn"):
super(DCNNBERTEmbedding, self ).__init__(name=name)
 self.bert_layer = hub.KerasLayer(
 "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1",
trainable=False)
 self.bigram = layers.Conv1D(filters=nb_filters,
kernel_size=2,
padding="valid",
activation="relu")
 self.trigram = layers.Conv1D(filters=nb_filters,
kernel_size=3,
padding="valid",
activation="relu")
 self.fourgram = layers.Conv1D(filters=nb_filters,
kernel_size=4,
padding="valid",
activation="relu")
 self.pool = layers.GlobalMaxPool1D()
self.dense_1 = layers.Dense(units=FFN_units, activation="relu")
 self.dropout = layers.Dropout(rate=dropout_rate)
 if nb_classes == 2:
self.last_dense = layers.Dense(units=1,
activation="sigmoid")
 else:
self.last_dense = layers.Dense(units=nb_classes,
activation="softmax")
 def embed_with_bert(self, all_tokens):
_, embs = self.bert_layer([all_tokens[:, 0, :],
all_tokens[:, 1, :],
all_tokens[:, 2, :]])
 return embs
def call(self, inputs, training):
x = self.embed_with_bert(inputs)
 print(x.shape)
 x_1 = self.bigram(x)
 x_1 = self.pool(x_1)
 x_2 = self.trigram(x)
 x_2 = self.pool(x_2)
 x_3 = self.fourgram(x)
 x_3 = self.pool(x_3)
 merged = tf.concat([x_1, x_2, x_3], axis=-1)
 merged = self.dense_1(merged)
 merged = self.dropout(merged, training)
 output = self.last_dense(merged)
 return output
training_examples = preprocess_features(train.head(122080))
training_targets = preprocess_targets(train.head(122080))
validation_examples = preprocess_features(train.tail(60000))
validation_targets = preprocess_targets(train.tail(60000))
test_examples = preprocess_features(test.copy() )<feature_engineering>
NB_FILTERS = 100
FFN_UNITS = 256
NB_CLASSES = 2
DROPOUT_RATE = 0.2
BATCH_SIZE = 32
NB_EPOCHS = 3
vectorizer=TfidfVectorizer(stop_words=stop)
 vectorizer.fit(train['project_subject_categories'])
 train_project_subject_categories = vectorizer.transform(training_examples['project_subject_categories'])
 validation_project_subject_categories = vectorizer.transform(validation_examples['project_subject_categories'])
 test_project_subject_categories = vectorizer.transform(test_examples['project_subject_categories'])
 vectorizer.fit(train['project_subject_subcategories'])
 train_project_subject_subcategories = vectorizer.transform(training_examples['project_subject_subcategories'])
 validation_project_subject_subcategories = vectorizer.transform(validation_examples['project_subject_subcategories'])
 test_project_subject_subcategories = vectorizer.transform(test_examples['project_subject_subcategories'] )<feature_engineering>
Dcnn = DCNNBERTEmbedding(nb_filters=NB_FILTERS,
FFN_units=FFN_UNITS,
nb_classes=NB_CLASSES,
dropout_rate=DROPOUT_RATE )
vectorizer=TfidfVectorizer(stop_words=stop, ngram_range=(1, 2), max_df=0.9, min_df=5, max_features=2000)
 vectorizer.fit(train['project_title'])
 train_project_title = vectorizer.transform(training_examples['project_title'])
 validation_project_title = vectorizer.transform(validation_examples['project_title'])
 test_project_title = vectorizer.transform(test_examples['project_title'])
 vectorizer.fit(train['project_resource_summary'])
 train_project_resource_summary = vectorizer.transform(training_examples['project_resource_summary'])
 validation_project_resource_summary = vectorizer.transform(validation_examples['project_resource_summary'])
 test_project_resource_summary = vectorizer.transform(test_examples['project_resource_summary'] )<feature_engineering>
if NB_CLASSES == 2:
Dcnn.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
 else:
Dcnn.compile(loss="sparse_categorical_crossentropy",
optimizer="adam",
metrics=["sparse_categorical_accuracy"] )
vectorizer=TfidfVectorizer(stop_words=stop, ngram_range=(1, 3), max_df=0.9, min_df=5, max_features=2000)
 vectorizer.fit(train['project_essay_1'])
 train_project_essay_1 = vectorizer.transform(training_examples['project_essay_1'])
 validation_project_essay_1 = vectorizer.transform(validation_examples['project_essay_1'])
 test_project_essay_1 = vectorizer.transform(test_examples['project_essay_1'])
 vectorizer.fit(train['project_essay_2'])
 train_project_essay_2 = vectorizer.transform(training_examples['project_essay_2'])
 validation_project_essay_2 = vectorizer.transform(validation_examples['project_essay_2'])
 test_project_essay_2 = vectorizer.transform(test_examples['project_essay_2'] )<feature_engineering>
class MyCustomCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
ckpt_manager.save()
print("Checkpoint saved at {}.".format(checkpoint_path))
def get_polarity(text):
textblob = TextBlob(text)
 pol = textblob.sentiment.polarity
return round(pol,3)
 def get_subjectivity(text):
textblob = TextBlob(text)
 subj = textblob.sentiment.subjectivity
return round(subj,3 )<feature_engineering>
Dcnn.fit(train_dataset,
epochs=NB_EPOCHS,
validation_data=(test_dataset),
callbacks=[MyCustomCallback() ] )
training_examples['polarity1'] = training_examples['project_essay_1'].apply(get_polarity)
 training_examples['subjectivity1'] = training_examples['project_essay_1'].apply(get_subjectivity)
 training_examples['polarity2'] = training_examples['project_essay_2'].apply(get_polarity)
 training_examples['subjectivity2'] = training_examples['project_essay_2'].apply(get_subjectivity )<feature_engineering>
def get_prediction(sentence):
sentence=clean_tweet(sentence)
 tokens = encode_sentence(sentence)
 input_ids = get_ids(tokens)
 input_mask = get_masks(tokens)
 segment_ids = get_segments(tokens)
 inputs = tf.stack(
 [tf.cast(input_ids, dtype=tf.int32),
tf.cast(input_mask, dtype=tf.int32),
tf.cast(segment_ids, dtype=tf.int32)],
axis=0)
 inputs = tf.expand_dims(inputs, 0)
 output = Dcnn(inputs, training=False)
 sentiment = math.floor(output*2)
 if sentiment == 0:
print("Output of the model: {}
Predicted sentiment: negative".format(
 output))
elif sentiment == 1:
print("Output of the model: {}
Predicted sentiment: positive".format(
 output))
return sentiment
validation_examples['polarity1'] = validation_examples['project_essay_1'].apply(get_polarity)
 validation_examples['subjectivity1'] = validation_examples['project_essay_1'].apply(get_subjectivity)
 validation_examples['polarity2'] = validation_examples['project_essay_2'].apply(get_polarity)
 validation_examples['subjectivity2'] = validation_examples['project_essay_2'].apply(get_subjectivity )<feature_engineering>
test=pd.read_csv('.. /input/nlp-getting-started/test.csv' )
test_examples['polarity1'] = test_examples['project_essay_1'].apply(get_polarity)
 test_examples['subjectivity1'] = test_examples['project_essay_1'].apply(get_subjectivity)
 test_examples['polarity2'] = test_examples['project_essay_2'].apply(get_polarity)
 test_examples['subjectivity2'] = test_examples['project_essay_2'].apply(get_subjectivity )<drop_column>
preds=[]
for i in test['text']:
preds.append(get_prediction(i))
to_drop = ['project_subject_categories', 'project_subject_subcategories', 'project_title', 'project_essay_1', 'project_essay_2', 'project_resource_summary']
for col in to_drop:
training_examples.drop([col], axis=1, inplace=True)
 validation_examples.drop([col], axis=1, inplace=True)
 test_examples.drop([col], axis=1, inplace=True )<compute_test_metric>
sub=pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )
training_features = csr_matrix(hstack([training_examples.values, train_project_subject_categories, train_project_subject_subcategories, train_project_title, train_project_resource_summary, train_project_essay_1, train_project_essay_2]))
validation_features = csr_matrix(hstack([validation_examples.values, validation_project_subject_categories, validation_project_subject_subcategories, validation_project_title, validation_project_resource_summary, validation_project_essay_1, validation_project_essay_2]))
test_features = csr_matrix(hstack([test_examples.values, test_project_subject_categories, test_project_subject_subcategories, test_project_title, test_project_resource_summary, test_project_essay_1, test_project_essay_2]))<train_model>
sub['target']=preds
<init_hyperparams>
sub.to_csv('submission.csv',index=False )