Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<set_options>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
del X_train, y_train
gc.collect()<load_from_disk>
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras import layers, models, optimizers
from tensorflow.keras.callbacks import ModelCheckpoint
df_test = pd.read_feather('.. /input/ashrae-feather/test.ft')
 row_ids = df_test["row_id"]
df_test.drop("row_id", axis=1, inplace=True)
 df_test = reduce_mem_usage(df_test )<feature_engineering>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )
<merge>
def bert_encode(texts, tokenizer, max_len):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
df_test = df_test.merge(building,left_on='building_id',right_on='building_id',how='left')
 del building
gc.collect()<drop_column>
train = train.fillna(' ')
 test = test.fillna(' ' )
weather_test = fill_weather_dataset(weather_test)
 weather_test = reduce_mem_usage(weather_test )<merge>
max_len = 60
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 train_input = bert_encode(train['location']+' '+train['keyword']+' '+train['text'], tokenizer, max_len=max_len)
 test_input = bert_encode(test['location']+' '+test['keyword']+' '+test['text'], tokenizer, max_len=max_len)
 train_labels = train.target.values
df_test = df_test.merge(weather_test,how='left',on=['timestamp','site_id'])
 del weather_test
gc.collect()<feature_engineering>
input_word_ids = layers.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = layers.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = layers.Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = layers.Dense(1, activation='sigmoid' )(clf_output)
 model = models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(optimizers.Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 model.summary()
df_test = features_engineering(df_test )<predict_on_test>
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
callbacks=[checkpoint],
batch_size=16
)
%%time
pred = []
for model in tqdm(models):
if pred == []:
pred = np.expm1(model.predict(df_test, num_iteration=model.best_iteration)) / len(models)
 else:
pred += np.expm1(model.predict(df_test, num_iteration=model.best_iteration)) / len(models)
 del model
gc.collect()<load_pretrained>
model.load_weights('model.h5')
 test_pred = model.predict(test_input )
<set_options><EOS> 
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )