Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<set_options>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
warnings.filterwarnings('ignore' )<import_modules>
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input, LSTM, Bidirectional
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
import numpy as np
import pandas as pd<load_pretrained>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ['[CLS]'] + text + ['[SEP]']
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
nltk.download('stopwords')
<import_modules>
def build_model(bert_layer, max_len=512):
def inner_build_model() :
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name='input_word_ids')
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name='input_mask')
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name='segment_ids')
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy', 'mse'])
 return model
return inner_build_model
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report,accuracy_score,confusion_matrix<load_from_csv>
%%time
module_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1'
bert_layer = hub.KerasLayer(module_url, trainable=True )
train_lab=pd.read_csv('/kaggle/input/word2vec-nlp-tutorial/labeledTrainData.tsv',delimiter='\t',quoting=3)
 train_unlab=pd.read_csv('/kaggle/input/word2vec-nlp-tutorial/unlabeledTrainData.tsv',error_bad_lines=False,delimiter='\t')
 test=pd.read_csv('/kaggle/input/word2vec-nlp-tutorial/testData.tsv',delimiter='\t',quoting=3)
 submission=pd.read_csv('/kaggle/input/word2vec-nlp-tutorial/sampleSubmission.csv' )<drop_column>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
train_lab=train_lab.drop(columns=['id'] )<string_transform>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv' )
def review_word(raw_review):
review=BeautifulSoup(raw_review ).get_text()
review=re.sub('[^a-zA-Z]',' ',review)
 review=review.lower()
review=review.split()
ps=PorterStemmer()
review=[ps.stem(word)for word in review if not word in set(stopwords.words('english')) ]
review=' '.join(review)
 return review<define_variables>
train['token_len'] = train['text'].apply(lambda x: len(tokenizer.tokenize(x)))
 test['token_len'] = test['text'].apply(lambda x: len(tokenizer.tokenize(x)) )
corpus=[]
for i in range(0,25000):
corpus.append(review_word(train_lab['review'][i]))<choose_model_class>
token_max_len = max(train['token_len'].max() , test['token_len'].max())+ 2
display(token_max_len )
tv=TfidfVectorizer(max_features=5000,analyzer='word' )<prepare_x_and_y>
def get_kfold_sets(train, k=5):
kf = KFold(n_splits=k, shuffle=True)
 for train_texts, train_labels in kf.split(train.text.values, train.target.values):
train_texts, valid_texts, train_labels, valid_labels = train_test_split(train.text.values, train.target.values, test_size=0.2)
 train_input = bert_encode(train_texts, tokenizer, max_len=token_max_len)
 valid_input = bert_encode(valid_texts, tokenizer, max_len=token_max_len)
 yield train_input, train_labels, valid_input, valid_labels
def get_train_sets(train):
train_input = bert_encode(train.text.values, tokenizer, max_len=token_max_len)
 train_labels = train.target.values
return train_input, train_labels
X=tv.fit_transform(corpus ).toarray()
y=train_lab['sentiment']<split>
test_input = bert_encode(test.text.values, tokenizer, max_len=token_max_len )
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=0 )<find_best_model_class>
class F1Callback(Callback):
def __init__(self, model, X_val, y_val):
self.model = model
self.X_val = X_val
self.y_val = y_val
def on_epoch_end(self, epoch, logs):
pred = self.model.predict(self.X_val)
 f1_val = f1_score(self.y_val, np.round(pred))
print('
f1_val = ', f1_val )
classifier_nb=GaussianNB()
classifier_nb.fit(X_train,y_train)
 y_pred_nb=classifier_nb.predict(X_test )<compute_test_metric>
def cross_val_score(train, k=3, epochs=10, batch_size=16):
f1_vals = []
models = []
i = 0
for train_input, train_labels, valid_input, valid_labels in get_kfold_sets(train, k=k):
model = clone_model(model_template)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy', 'mse'])
 train_history = model.fit(
 train_input, train_labels,
validation_data=(valid_input, valid_labels),
epochs=epochs,
batch_size=batch_size,
callbacks=[EarlyStopping(patience=1, monitor='val_mse', mode='min', verbose=True)] 
)
 pred = model.predict(valid_input)
 f1_val = f1_score(valid_labels, np.round(pred))
print(f'f1-val: {f1_val}')
 f1_vals.append(f1_val)
 models.append(model)
 df = pd.DataFrame(train_history.history)
 df['f1-val'] = f1_val
df.to_csv(f'history_{i}.csv')
 i += 1
return np.array(f1_vals ).mean() , models
k = 5
f1_val, models = cross_val_score(train, k=k)
 print(f'f1-mean: {f1_val}' )
report=classification_report(y_pred_nb,y_test)
 print('the classification report is
{}'.format(report))<train_model>
train_input, train_labels = get_train_sets(train )
classifier_rf=RandomForestClassifier(random_state=0)
 classifier_rf.fit(X_train,y_train)
 y_pred_rf=classifier_rf.predict(X_test )<compute_test_metric>
def calc_best_threshold(pred, labels):
f1_vals = []
ts = []
for t in np.arange(0.1, 1, 0.1):
f1_val = f1_score(train_labels, [1 if p >= t else 0 for p in train_pred])
 f1_vals.append(f1_val)
 ts.append(t)
 return ts[np.argmax(f1_vals)]
best_ts = []
for model in models:
train_pred = model.predict(train_input)
 tmp = calc_best_threshold(train_pred, train_labels)
 best_ts.append(tmp)
 print(f'best ts: {best_ts}' )
report=classification_report(y_pred_rf,y_test)
 print('the classification report is
{}'.format(report))<train_model>
test_preds = []
for model in models:
test_pred = model.predict(test_input)
 test_preds.append(test_pred)
 test_preds = np.array(test_preds)
 print(test_preds.shape )
classifier_lr=LogisticRegression(random_state=0)
 classifier_lr.fit(X_train,y_train)
 y_pred_lr=classifier_lr.predict(X_test )<compute_test_metric>
test_size = test_preds.shape[1]
mean_pred = []
for s in range(test_size):
tmp = []
for i in range(k):
tmp.append(1 if test_preds[i][s][0] >= best_ts[i] else 0)
 mean = np.mean(tmp)
 mean_pred.append(mean)
 mean_pred = np.array(mean_pred)
 print(mean_pred.shape)
 print(mean_pred[20:])
 print(mean_pred[:20] )
<train_on_grid><EOS> 
submission['target'] = mean_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )