Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<import_modules>
!pip install -U pip
!pip install tweet-preprocessor
!pip install -q tf-nightly
!pip install -q tf-models-nightly
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint
from keras.engine import Layer
from keras.layers import Activation, Add, Bidirectional, Conv1D, Dense, Dropout, Embedding, Flatten
from keras.layers import concatenate, GRU, Input, K, LSTM, MaxPooling1D
from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, SpatialDropout1D
from keras.models import Model
from keras.optimizers import Adam
from keras.preprocessing import text, sequence
from sklearn.metrics import accuracy_score, roc_auc_score, log_loss
from sklearn.model_selection import train_test_split
from keras import initializers, regularizers, constraints, optimizers, layers, callbacks<init_hyperparams>
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
gru_len = 128
Routings = 5
Num_capsule = 10
Dim_capsule = 16
dropout_p = 0.3
rate_drop_dense = 0.3<statistical_test>
tf.random.set_seed(123)
 np.random.seed(123)
 start_time = time.time()
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale<choose_model_class>
data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = K.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )<choose_model_class>
stopwords_en = stopwords.words('english')
 def unicode_to_ascii(s):
return ''.join(c for c in unicodedata.normalize('NFD', s)if unicodedata.category(c)!= 'Mn')
 twitter_p.set_options(twitter_p.OPT.URL)
 def preprocess_sentence(w):
w = twitter_p.clean(w)
 w = unicode_to_ascii(w.lower().strip())
 w = re.sub(r"([@
w = re.sub(r'[" "]+', " ", w)
 w = ' '.join([word for word in w.split(' ')if word not in stopwords_en])
 w = w.rstrip().strip()
return w
def get_model(embedding_matrix, sequence_length, dropout_rate, recurrent_units, dense_size):
input1 = Input(shape=(sequence_length,))
embed_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=False )(input1)
 embed_layer = SpatialDropout1D(rate_drop_dense )(embed_layer)
 x = Bidirectional(
 GRU(gru_len, activation='relu', dropout=dropout_p, recurrent_dropout=dropout_p, return_sequences=True))(
 embed_layer)
 capsule = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=Routings,
share_weights=True )(x)
 capsule = Flatten()(capsule)
 capsule = Dropout(dropout_p )(capsule)
 output = Dense(1, activation='sigmoid' )(capsule)
 model = Model(inputs=input1, outputs=output)
 model.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model<train_model>
data = data.fillna(value='')
 data['text'] = data['text'].apply(func=preprocess_sentence)
 data['keyword'] = data['keyword'].apply(func=preprocess_sentence)
 test = test.fillna(value='')
 test['text'] = test['text'].apply(func=preprocess_sentence)
 test['keyword'] = test['keyword'].apply(func=preprocess_sentence )
def _train_model(model, batch_size, train_x, train_y, val_x, val_y):
num_labels = train_y.shape[1]
patience = 5
best_loss = -1
best_weights = None
best_epoch = 0
current_epoch = 0
while True:
model.fit(train_x, train_y, batch_size=batch_size, epochs=1)
 y_pred = model.predict(val_x, batch_size=batch_size)
 total_loss = 0
for j in range(num_labels):
loss = log_loss(val_y[:, j], y_pred[:, j])
 total_loss += loss
total_loss /= num_labels
print("Epoch {0} loss {1} best_loss {2}".format(current_epoch, total_loss, best_loss))
current_epoch += 1
if total_loss < best_loss or best_loss == -1:
best_loss = total_loss
best_weights = model.get_weights()
best_epoch = current_epoch
else:
if current_epoch - best_epoch == patience:
break
model.set_weights(best_weights)
 return model<train_model>
tokenizer = bert.tokenization.FullTokenizer(vocab_file=os.path.join(gs_folder_bert, 'vocab.txt'), do_lower_case=True)
 print('Vocab Size:', len(tokenizer.vocab))
def train_folds(X, y, X_test, fold_count, batch_size, get_model_func):
print("="*75)
 fold_size = len(X)// fold_count
models = []
result_path = "predictions"
if not os.path.exists(result_path):
os.mkdir(result_path)
 for fold_id in range(0, fold_count):
fold_start = fold_size * fold_id
fold_end = fold_start + fold_size
if fold_id == fold_size - 1:
fold_end = len(X)
 train_x = np.concatenate([X[:fold_start], X[fold_end:]])
 train_y = np.concatenate([y[:fold_start], y[fold_end:]])
 val_x = np.array(X[fold_start:fold_end])
 val_y = np.array(y[fold_start:fold_end])
 model = _train_model(get_model_func() , batch_size, train_x, train_y, val_x, val_y)
 train_predicts_path = os.path.join(result_path, "train_predicts{0}.npy".format(fold_id))
test_predicts_path = os.path.join(result_path, "test_predicts{0}.npy".format(fold_id))
train_predicts = model.predict(X, batch_size=512, verbose=1)
 test_predicts = model.predict(X_test, batch_size=512, verbose=1)
 np.save(train_predicts_path, train_predicts)
 np.save(test_predicts_path, test_predicts)
 return models<define_variables>
words = tokenizer.vocab
w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts )
train_file_path = ".. /input/donorschooseorg-preprocessed-data/train_small.csv"
test_file_path = ".. /input/donorschooseorg-preprocessed-data/test_small.csv"
embedding_path = ".. /input/donorschooseorg-preprocessed-data/embeddings_small.vec"
batch_size = 128
recurrent_units = 16
dropout_rate = 0.3
dense_size = 8
sentences_length = 10
fold_count = 2<define_variables>
def search_word(key):
word = key
if word in tokenizer.vocab:
return word
word = ps.stem(key)
 if word in tokenizer.vocab:
return word
word = lc.stem(key)
 if word in tokenizer.vocab:
return word
word = sb.stem(key)
 if word in tokenizer.vocab:
return word
if len(key)> 1:
word = correction(key)
 if word in tokenizer.vocab:
return word
return word
UNKNOWN_WORD = "_UNK_"
END_WORD = "_END_"
NAN_WORD = "_NAN_"
CLASSES = ["project_is_approved"]<load_from_csv>
def encode_sentence(s, tokenizer):
tokens = tokenizer.tokenize(s)
 tokens = list(map(search_word, tokens))
tokens.append('[SEP]')
 return tokenizer.convert_tokens_to_ids(tokens)
 def bert_encode(glue_dict, tokenizer):
sentence1 = tf.ragged.constant([encode_sentence(s, tokenizer)for s in np.array(glue_dict['keyword'])])
 sentence2 = tf.ragged.constant([encode_sentence(s, tokenizer)for s in np.array(glue_dict['text'])])
 cls = [tokenizer.convert_tokens_to_ids(['[CLS]'])] * sentence1.shape[0]
input_word_ids = tf.concat([cls, sentence1, sentence2], axis=-1)
 input_mask = tf.ones_like(input_word_ids ).to_tensor()
type_cls = tf.ones_like(cls)
 type_s1 = tf.ones_like(sentence1)
 type_s2 = tf.ones_like(sentence2)
 input_type_ids = tf.concat([type_cls, type_s1, type_s2], axis=-1 ).to_tensor()
inputs = {
"input_word_ids": input_word_ids.to_tensor() ,
"input_mask": input_mask,
"input_type_ids": input_type_ids
}
return inputs
print("Loading data...")
 train_data = pd.read_csv(train_file_path)
 test_data = pd.read_csv(test_file_path)
 list_sentences_train = train_data["application_text"].fillna(NAN_WORD ).values
list_sentences_test = test_data["application_text"].fillna(NAN_WORD ).values
y_train = train_data[CLASSES].values<string_transform>
glue_train = bert_encode(train, tokenizer)
 glue_train_labels = train['target']
glue_val = bert_encode(val, tokenizer)
 glue_val_labels = val['target']
glue_test = bert_encode(test, tokenizer )
print("Tokenizing sentences in train set...")
 tokenized_sentences_train, words_dict = tokenize_sentences(list_sentences_train, {})
 print("Tokenizing sentences in test set...")
 tokenized_sentences_test, words_dict = tokenize_sentences(list_sentences_test, words_dict )<load_pretrained>
bert_config_file = os.path.join(gs_folder_bert, 'bert_config.json')
 config_dict = json.loads(tf.io.gfile.GFile(bert_config_file ).read())
 bert_config = bert.configs.BertConfig.from_dict(config_dict)
 print(config_dict )
words_dict[UNKNOWN_WORD] = len(words_dict)
 print("Loading embeddings...")
 embedding_list, embedding_word_dict = read_embedding_list(embedding_path)
 embedding_size = len(embedding_list[0] )<categorify>
bert_classifier, bert_encoder = bert.bert_models.classifier_model(bert_config, num_labels=2 )
print("Preparing data...")
 embedding_list, embedding_word_dict = clear_embedding_list(embedding_list, embedding_word_dict, words_dict)
 embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict)
 embedding_list.append([0.] * embedding_size)
 embedding_word_dict[END_WORD] = len(embedding_word_dict)
 embedding_list.append([-1.] * embedding_size)
 embedding_matrix = np.array(embedding_list)
 id_to_word = dict(( id, word)for word, id in words_dict.items())
 train_list_of_token_ids = convert_tokens_to_ids(
 tokenized_sentences_train,
id_to_word,
embedding_word_dict,
sentences_length)
 test_list_of_token_ids = convert_tokens_to_ids(
 tokenized_sentences_test,
id_to_word,
embedding_word_dict,
sentences_length)
 X_train = np.array(train_list_of_token_ids)
 X_test = np.array(test_list_of_token_ids )<load_pretrained>
checkpoint = tf.train.Checkpoint(model=bert_encoder)
 checkpoint.restore(os.path.join(gs_folder_bert, 'bert_model.ckpt')).assert_consumed()
get_model_func = lambda: get_model(
 embedding_matrix,
sentences_length,
dropout_rate,
recurrent_units,
dense_size )<drop_column>
epochs = 10
batch_size = 32
eval_batch_size = 32
train_data_size = len(glue_train_labels)
 steps_per_epoch = int(train_data_size / batch_size)
 num_train_steps = steps_per_epoch * epochs
warmup_steps = int(epochs * train_data_size * 0.1 / batch_size )
del train_data, test_data, list_sentences_train, list_sentences_test
del tokenized_sentences_train, tokenized_sentences_test, words_dict
del embedding_list, embedding_word_dict
del train_list_of_token_ids, test_list_of_token_ids
gc.collect() ;<train_model>
optimizer = nlp.optimization.create_optimizer(2e-5, num_train_steps=num_train_steps, num_warmup_steps=warmup_steps )
print("Starting to train models...")
 models = train_folds(X_train, y_train, X_test, fold_count, batch_size, get_model_func )<save_to_csv>
checkpoint_filepath = '/kaggle/working/checkpoint/best_val'
LABELS = ["project_is_approved"]
base = ".. /input/donorschooseorg-application-screening-predictions/predictions/predictions/"
predict_list = []
for j in range(10):
predict_list.append(np.load(base + "predictions_001/test_predicts%d.npy"%j))
print("Rank averaging on ", len(predict_list), " files")
 predcitions = np.zeros_like(predict_list[0])
 for predict in predict_list:
predcitions = np.add(predcitions.flatten() , rankdata(predict)/predcitions.shape[0])
 predcitions /= len(predict_list)
 submission = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 submission[LABELS] = predcitions
submission.to_csv('submission.csv', index=False )<set_options>
metrics = [tf.keras.metrics.SparseCategoricalAccuracy('accuracy', dtype=tf.float32)]
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
 bert_classifier.compile(optimizer=optimizer, loss=loss, metrics=metrics )
lem = WordNetLemmatizer()
eng_stopwords = set(stopwords.words("english"))
start_time=time.time()
color = sns.color_palette()
sns.set_style("dark")
 warnings.filterwarnings("ignore")
 %matplotlib inline<load_from_csv>
bert_classifier.fit(glue_train, glue_train_labels, validation_data=(glue_val, glue_val_labels), batch_size=32, epochs=epochs, callbacks=[checkpoint_callback] )
train=pd.read_csv(".. /input/donorschoose-application-screening/train.csv")
 resources=pd.read_csv(".. /input/donorschoose-application-screening/resources.csv")
 test=pd.read_csv(".. /input/donorschoose-application-screening/test.csv")
 sample_sub=pd.read_csv(".. /input/donorschoose-application-screening/sample_submission.csv" )<define_variables>
bert_classifier.load_weights(checkpoint_filepath)
nrow_train=train.shape[0]
nrow_test=test.shape[0]
sum=nrow_train+nrow_test
print("Checking proportion of Test-train split")
 print(" : train : test")
 print("rows :",nrow_train,":",nrow_test)
 print("perc :",round(nrow_train*100/sum)," :",round(nrow_test*100/sum))<count_missing_values>
test_target = pd.read_csv('/kaggle/input/test-twitter/perfect_submission.csv' )
print("Check for Percent of missing values in Train dataset")
 null_check=train.isnull().sum() 
(null_check/len(train)) *100<count_missing_values>
results = bert_classifier.evaluate(glue_test, test_target['target'] )
print("Check for Percent of missing values in RESOURCES file")
 null_check=resources.isnull().sum() 
(null_check/len(resources)) *100<count_unique_values>
predictions = bert_classifier.predict(glue_test )
print("There are",resources.description.isnull().sum() ,"NULL entries in description column of Resources dataset")
 null_ids=resources[resources.description.isnull() ].id
print("Those Null entries are from",len(null_ids.unique()),"projects")
 null_entries_train=train[train.id.isin(null_ids)]
print("There are",len(null_entries_train),"of these projects are in train")
 null_entries_test=test[test.id.isin(null_ids)]
print("There are",len(null_entries_test),"of these projects are in test")
 x=null_entries_train.project_is_approved.value_counts()
print("Approval rate of projects with NULL as the description under Resources:",x[1]/(x[0]+x[1])*100)
 x=null_entries_train[null_entries_train.teacher_number_of_previously_posted_projects==0].project_is_approved.value_counts()
print("Approval rate of projects with NULL as the description under Resources and 0 previous project submissions:",x[1]/(x[0]+x[1])*100 )<filter>
predictions = np.argmax(predictions, axis=1 )
null_entries_train[null_entries_train.project_is_approved==0].project_resource_summary.iloc[0]<filter>
submission = pd.DataFrame({"id":test["id"], "target":predictions})
 submission.to_csv('/kaggle/working/submission.csv', index=False )