Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<choose_model_class>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
reduce_lr = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x )<define_variables>
SEED = 1
batch_size = 64
epochs = 50<train_model>
df_train = pd.read_csv('.. /input/nlp-getting-started/train.csv', dtype={'id': np.int16, 'target': np.int8})
 df_test = pd.read_csv('.. /input/nlp-getting-started/test.csv', dtype={'id': np.int16})
 print('Training Set Shape = {}'.format(df_train.shape))
print('Training Set Memory Usage = {:.2f} MB'.format(df_train.memory_usage().sum() / 1024**2))
print('Test Set Shape = {}'.format(df_test.shape))
print('Test Set Memory Usage = {:.2f} MB'.format(df_test.memory_usage().sum() / 1024**2))
history = model.fit_generator(datagen.flow(X_train, y_train, batch_size = batch_size), epochs = epochs,
validation_data =(X_test, y_test), verbose=1,
steps_per_epoch=X_train.shape[0] // batch_size,
callbacks = [reduce_lr] )<predict_on_test>
missing_cols = ['keyword', 'location']
df_train[missing_cols].isnull().sum().index
preds = np.argmax(model.predict(test), axis=1)
 sub_df = {'ImageId':list(range(1, len(test)+ 1)) ,'Label':preds}
submission = pd.DataFrame(sub_df ).astype('int')
 submission.head()<save_to_csv>
missing_cols = ['keyword', 'location']
df_train[missing_cols].isnull().sum().values
submission.to_csv('submission.csv', index=False )<save_to_csv>
df_train.groupby(['text'] ).nunique().sort_values(by='target', ascending=False )
submission.to_csv('submission.csv', index=False )<load_from_csv>
class ClassificationReport(Callback):
def __init__(self, train_data=() , validation_data=()):
super(Callback, self ).__init__()
self.X_train, self.y_train = train_data
self.train_precision_scores = []
self.train_recall_scores = []
self.train_f1_scores = []
self.X_val, self.y_val = validation_data
self.val_precision_scores = []
self.val_recall_scores = []
self.val_f1_scores = []
def on_epoch_end(self, epoch, logs={}):
train_predictions = np.round(self.model.predict(self.X_train, verbose=0))
train_precision = precision_score(self.y_train, train_predictions, average='macro')
 train_recall = recall_score(self.y_train, train_predictions, average='macro')
 train_f1 = f1_score(self.y_train, train_predictions, average='macro')
 self.train_precision_scores.append(train_precision)
 self.train_recall_scores.append(train_recall)
 self.train_f1_scores.append(train_f1)
 val_predictions = np.round(self.model.predict(self.X_val, verbose=0))
val_precision = precision_score(self.y_val, val_predictions, average='macro')
 val_recall = recall_score(self.y_val, val_predictions, average='macro')
 val_f1 = f1_score(self.y_val, val_predictions, average='macro')
 self.val_precision_scores.append(val_precision)
 self.val_recall_scores.append(val_recall)
 self.val_f1_scores.append(val_f1)
 print('
Epoch: {} - Training Precision: {:.6} - Training Recall: {:.6} - Training F1: {:.6}'.format(epoch + 1, train_precision, train_recall, train_f1))
print('Epoch: {} - Validation Precision: {:.6} - Validation Recall: {:.6} - Validation F1: {:.6}'.format(epoch + 1, val_precision, val_recall, val_f1))
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv" )<count_missing_values>
%%time
bert_layer = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1', trainable=True )
train.isnull().sum().sum()<drop_column>
K = 3
skf = StratifiedKFold(n_splits=K, random_state=SEED, shuffle=True)
 DISASTER = df_train['target'] == 1
print('Whole Training Set Shape = {}'.format(df_train.shape))
print('Whole Training Set Unique keyword Count = {}'.format(df_train['keyword'].nunique()))
print('Whole Training Set Target Rate(Disaster){}/{}(Not Disaster)'.format(df_train[DISASTER]['target'].count() , df_train[~DISASTER]['target'].count()))
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train['text'], df_train['target']), 1):
print(f'trn_idx {trn_idx} val_idx {val_idx}')
 print('
Fold {} Training Set Shape = {} - Validation Set Shape = {}'.format(fold, df_train.loc[trn_idx, 'text'].shape, df_train.loc[val_idx, 'text'].shape))
print('Fold {} Training Set Unique keyword Count = {} - Validation Set Unique keyword Count = {}'.format(fold, df_train.loc[trn_idx, 'keyword'].nunique() , df_train.loc[val_idx, 'keyword'].nunique()))
labels = train["label"]
pureimg_train = train.drop(labels = ["label"], axis = 1)
 del train<categorify>
class DisasterDetector:
def __init__(self, bert_layer, max_seq_length=128, lr=0.0001, epochs=15, batch_size=32):
self.bert_layer = bert_layer
self.max_seq_length = max_seq_length
vocab_file = self.bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = self.bert_layer.resolved_object.do_lower_case.numpy()
self.tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 self.lr = lr
self.epochs = epochs
self.batch_size = batch_size
self.scores = {}
self.model = self.build_model()
self.model.summary()
def encode(self, texts):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = self.tokenizer.tokenize(text)
 text = text[:self.max_seq_length - 2]
input_sequence = ['[CLS]'] + text + ['[SEP]']
pad_len = self.max_seq_length - len(input_sequence)
 tokens = self.tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * self.max_seq_length
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(self):
input_word_ids = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='input_word_ids')
 input_mask = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='input_mask')
 segment_ids = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='segment_ids')
 pooled_output, sequence_output = self.bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 optimizer = SGD(learning_rate=self.lr, momentum=0.8)
 model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
 return model
def train(self, X):
for fold,(trn_idx, val_idx)in enumerate(skf.split(X['text'], X['target'])) :
print('
Fold {}
'.format(fold))
X_trn_encoded = self.encode(X.loc[trn_idx, 'text'].str.lower())
 y_trn = X.loc[trn_idx, 'target']
X_val_encoded = self.encode(X.loc[val_idx, 'text'].str.lower())
 y_val = X.loc[val_idx, 'target']
metrics = ClassificationReport(train_data=(X_trn_encoded, y_trn), validation_data=(X_val_encoded, y_val))
self.model.fit(X_trn_encoded, y_trn, validation_data=(X_val_encoded, y_val), callbacks=[metrics], epochs=self.epochs, batch_size=self.batch_size)
 self.scores[fold] = {
'train': {
'precision': metrics.train_precision_scores,
'recall': metrics.train_recall_scores,
'f1': metrics.train_f1_scores
},
'validation': {
'precision': metrics.val_precision_scores,
'recall': metrics.val_recall_scores,
'f1': metrics.val_f1_scores
}
}
def predict(self, X):
X_test_encoded = self.encode(X['text'].str.lower())
 y_pred = self.model.predict(X_test_encoded)
 return y_pred
norm_train = pureimg_train/255
norm_test = test/255<split>
clf = DisasterDetector(bert_layer, max_seq_length=128, lr=0.0001, epochs=10, batch_size=32)
 clf.train(df_train )
feature_train, feature_validate, target_train, target_validate = train_test_split(norm_train, labels, test_size = 0.1, random_state = 0)
<categorify>
y_pred = clf.predict(df_test)
 model_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 model_submission['target'] = np.round(y_pred ).astype('int')
 model_submission.to_csv('model_submission.csv', index=False)
 model_submission.describe()
Test = torch.from_numpy(norm_test.values.reshape(( -1,1,28,28)))
 featuresTrain = torch.from_numpy(feature_train.values.reshape(( -1,1,28,28)))
 targetsTrain = torch.from_numpy(target_train.values)
 featuresValidation = torch.from_numpy(feature_validate.values.reshape(( -1,1,28,28)))
 targetsValidation = torch.from_numpy(target_validate.values )<define_variables>
output_dir = 'cp.ckpt'
print("Saving model to %s" % output_dir)
 model_to_save = clf.model
model_to_save.save_weights(output_dir)