Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
train = pd.read_csv(".. /input/word2vec-nlp-tutorial/labeledTrainData.tsv", delimiter='\t')
 train.head()<load_from_csv>
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score
train_imdb = pd.read_csv(".. /input/imdb-review-dataset/imdb_master.csv", encoding="latin-1", index_col=0)
 train_imdb.head()<categorify>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
train_imdb = train_imdb[train_imdb.label != 'unsup']
train_imdb["sentiment"] = train_imdb.label.map({"neg": 0, "pos": 1})
 train_imdb.drop(["type", "file", "label"], axis=1, inplace=True )<prepare_x_and_y>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
train_enhanced = pd.concat([train, train_imdb], ignore_index=True)
 X_train = train_enhanced.review
y_train = train_enhanced.sentiment<string_transform>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
stopwords_eng = set(stopwords.words("english"))
def clean_text(text):
text = text.lower()
text = re.sub("[^a-z]", " ", text)
 words = [word for word in text.split() if word not in stopwords_eng]
text = " ".join(words)
 return text
X_train = X_train.map(clean_text)
 X_train.head()<normalization>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )
num_words = 6000
tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(X_train)
 X_train_seq = pd.Series(tokenizer.texts_to_sequences(X_train))
X_train_seq.head()<categorify>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
X_train_pad = pad_sequences(X_train_seq, maxlen=512 )<choose_model_class>
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values
model = Sequential()
model.add(Embedding(input_dim=num_words, output_dim=64))
model.add(LSTM(32, return_sequences=True))
model.add(GlobalMaxPool1D())
 model.add(Dense(16, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
model.summary()<train_model>
train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=1,
batch_size=16 
)
 model.save('model.h5' )
model.compile(loss="binary_crossentropy", optimizer='adam', metrics=['accuracy'])
 batch_size = 32
epochs = 2
validation_split = 0.01
model.fit(x=X_train_pad, y=y_train, batch_size=batch_size, epochs=epochs, validation_split=validation_split )<load_from_csv>
cls_layer_model = Model(model.input, outputs=model.get_layer('tf_op_layer_strided_slice' ).output )
test = pd.read_csv(".. /input/word2vec-nlp-tutorial/testData.tsv", delimiter="\t")
 test.head()<categorify>
X_train = cls_layer_model.predict(train_input)
 X_test = cls_layer_model.predict(test_input)
 y_train = train.target.values
X_test = test.review.map(clean_text)
 X_test.head()<categorify>
model_svm = SVC(C=15, kernel='rbf',gamma=0.001)
 model_svm.fit(X_train,y_train )
X_test_seq = tokenizer.texts_to_sequences(X_test)
 X_test_pad = pad_sequences(X_test_seq, maxlen=512 )<predict_on_test>
test_pred = model_svm.predict(X_test )
pred = model.predict(x=X_test_pad)
 y_pred =(pred >= 0.5)* 1<save_to_csv>
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )
results = pd.DataFrame({"id": test.id, "sentiment": y_pred.flatten() })
 results.to_csv("submission.csv", index=False )<load_from_csv>
sample_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 train = pd.read_csv(".. /input/nlp-getting-started/train.csv" )