Kaggle competition: Natural Language Processing with Disaster Tweets
max_features=100000
maxlen=150
embed_size=300<compute_train_metric>
train_data = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 train_data.head(5 )
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))<categorify>
test_data = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 test_data.head(5 )
tok=text.Tokenizer(num_words=max_features,lower=True)
 tok.fit_on_texts(list(X_train)+list(X_test))
X_train=tok.texts_to_sequences(X_train)
 X_test=tok.texts_to_sequences(X_test)
 x_train=sequence.pad_sequences(X_train,maxlen=maxlen)
 x_test=sequence.pad_sequences(X_test,maxlen=maxlen )<categorify>
!pip install BeautifulSoup4
embeddings_index = {}
with open(EMBEDDING_FILE,encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs<feature_engineering>
stop = set(stopwords.words('english'))
stop.update(list(string.punctuation))
def clean_tweets(text):
re1 = re.compile(r' +')
 x1 = text.lower().replace('
'nbsp;', ' ' ).replace('
', "
" ).replace('quot;', "'" ).replace(
 '<br />', "
" ).replace('\"', '"' ).replace('<unk>', 'u_n' ).replace(' @.@ ', '.' ).replace(
 ' @-@ ', '-' ).replace('\', ' \\ ')
 text = re1.sub(' ', html.unescape(x1))
text = unicodedata.normalize('NFKD', text ).encode('ascii', 'ignore' ).decode('utf-8', 'ignore')
 soup = BeautifulSoup(text, 'html.parser')
 text = soup.get_text()
text = re.sub('\[[^]]*\]', '', text)
 text = re.sub(r'http\S+', '', text)
 text = text.replace("@", "")
 text = text.replace("
text = re.sub(r'[^a-zA-Z ]', '', text)
 final_text = []
for word in text.split() :
if word.strip().lower() not in stop:
final_text.append(word.strip().lower())
 text = " ".join(final_text)
 lemmatizer = WordNetLemmatizer()
text = " ".join([lemmatizer.lemmatize(word)for word in text.split() ])
 text = " ".join([lemmatizer.lemmatize(word, pos = 'v')for word in text.split() ])
 text = re.sub("\d", "num", text)
 return text.lower()
train_data['prep_text'] = train_data['text'].apply(clean_tweets)
 train_data['prep_text'].head(5 )
word_index = tok.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector<choose_model_class>
test_data['text'] = test_data['text'].apply(clean_tweets)
 test_data['text'].head(5 )
sequence_input = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(sequence_input)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 preds = Dense(6, activation="sigmoid" )(x)
 model = Model(sequence_input, preds)
 model.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-3),metrics=['accuracy'] )<import_modules>
vocab_size = 1000
tokenizer = Tokenizer(num_words = vocab_size, oov_token = 'UNK')
 tokenizer.fit_on_texts(list(train_data['prep_text'])+ list(test_data['text']))
from keras.utils.vis_utils import plot_model<split>
X_train_ohe = tokenizer.texts_to_matrix(train_data['prep_text'], mode = 'binary')
 X_test_ohe = tokenizer.texts_to_matrix(test_data['text'], mode = 'binary')
 y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape: {X_train_ohe.shape}")
 print(f"X_test shape: {X_test_ohe.shape}")
 print(f"y_train shape: {y_train.shape}" )
batch_size = 128
epochs = 5
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.9, random_state=233 )<train_model>
X_train_ohe, X_val_ohe, y_train, y_val = train_test_split(X_train_ohe, y_train, random_state = 42, test_size = 0.2)
 print(f"X_train shape: {X_train_ohe.shape}")
 print(f"X_val shape: {X_val_ohe.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"y_val shape: {y_val.shape}" )
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),callbacks = callbacks_list,verbose=1)
 model.load_weights(filepath)
 print('Predicting.... ')
 y_pred = model.predict(x_test,batch_size=1024,verbose=1 )<compute_test_metric>
def setup_model() :
model = Sequential()
model.add(layers.Dense(1, activation='sigmoid', input_shape=(vocab_size,)))
 model.compile(optimizer=optimizers.RMSprop(lr=0.001),
loss=losses.binary_crossentropy,
metrics=[metrics.binary_accuracy])
 return model
model = setup_model()
model.summary()
y_df = np.where(y_pred > 0.5, 1, 0 )<data_type_conversions>
history = model.fit(X_train_ohe, y_train, epochs = 20, batch_size = 512, validation_data =(X_val_ohe, y_val))
y_df = pd.DataFrame(y_df, columns=['toxic','severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'])
 y_df = y_df.astype('int' )<define_variables>
_, accuracy = model.evaluate(X_val_ohe, y_val )
labels = ['toxic','severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']<compute_test_metric>
X_train_wc = tokenizer.texts_to_matrix(train_data['prep_text'], mode = 'count')
 X_test_wc = tokenizer.texts_to_matrix(test_data['text'], mode = 'count')
 y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape: {X_train_wc.shape}")
 print(f"X_test shape: {X_test_wc.shape}")
 print(f"y_train shape: {y_train.shape}")
def get_metri_scores(y_test, y_test_pred):
vals = precision_recall_fscore_support(y_test, y_test_pred, average='macro')
 precision = vals[0]
recall = vals[1]
f1 = vals[2]
acc = accuracy_score(y_test, y_test_pred)
 return precision, recall, f1, acc<feature_engineering>
X_train_wc, X_val_wc, y_train, y_val = train_test_split(X_train_wc, y_train, random_state = 42, test_size = 0.2)
 print(f"X_train shape: {X_train_wc.shape}")
 print(f"X_val shape: {X_val_wc.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"y_val shape: {y_val.shape}" )
results_cv = pd.DataFrame({'labels': labels})
 results_cv['acc'] = 0
results_cv['f1'] = 0
results_cv['precision'] = 0
results_cv['recall'] = 0
for col in labels:
print(col)
 precision, recall, f1, acc = get_metri_scores(test[col], y_df[col])
 results_cv['acc'][results_cv['labels']==col] = acc
results_cv['f1'][results_cv['labels']==col] = f1
results_cv['precision'][results_cv['labels']==col] = precision
results_cv['recall'][results_cv['labels']==col] = recall<choose_model_class>
history = model.fit(X_train_wc, y_train, epochs = 20, batch_size = 512, validation_data =(X_val_wc, y_val))
sequence_input = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(sequence_input)
 x = SpatialDropout1D(0.2 )(x)
 x = LSTM(256, return_sequences=True,dropout=0.1,recurrent_dropout=0.1 )(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = Dense(128, activation='relu' )(max_pool)
 x = Dropout(0.1 )(x)
 preds = Dense(6, activation="sigmoid" )(x)
 model2 = Model(sequence_input, preds)
 model2.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-3),metrics=['accuracy'] )<train_on_grid>
_, accuracy = model.evaluate(X_val_wc, y_val )
filepath="weights_novice_model.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 early = EarlyStopping(monitor="val_acc", mode="max", patience=5)
 ra_val = RocAucEvaluation(validation_data=(X_val, y_val), interval = 1)
 callbacks_list = [ra_val,checkpoint, early]
model2.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),callbacks = callbacks_list,verbose=1)
 model2.load_weights(filepath)
 print('Predicting.... ')
 y_pred = model2.predict(x_test,batch_size=1024,verbose=1 )<create_dataframe>
X_train_freq = tokenizer.texts_to_matrix(train_data['prep_text'], mode = 'freq')
 X_test_freq = tokenizer.texts_to_matrix(test_data['text'], mode = 'freq')
 y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape: {X_train_freq.shape}")
 print(f"X_test shape: {X_test_freq.shape}")
 print(f"y_train shape: {y_train.shape}" )
y_df = np.where(y_pred > 0.5, 1, 0)
 y_df = pd.DataFrame(y_df, columns=['toxic','severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'])
 y_df = y_df.astype('int' )<compute_train_metric>
X_train_freq, X_val_freq, y_train, y_val = train_test_split(X_train_freq, y_train, test_size = 0.2, random_state = 42)
 print(f"X_train shape: {X_train_freq.shape}")
 print(f"X_val shape: {X_val_freq.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"y_val shape: {y_val.shape}" )
def get_metri_scores(y_test, y_test_pred):
vals = precision_recall_fscore_support(y_test, y_test_pred, average='macro')
 precision = vals[0]
recall = vals[1]
f1 = vals[2]
acc = accuracy_score(y_test, y_test_pred)
 return precision, recall, f1, acc
results_cv = pd.DataFrame({'labels': labels})
 results_cv['acc'] = 0
results_cv['f1'] = 0
results_cv['precision'] = 0
results_cv['recall'] = 0
for col in labels:
print(col)
 precision, recall, f1, acc = get_metri_scores(test[col], y_df[col])
 results_cv['acc'][results_cv['labels']==col] = acc
results_cv['f1'][results_cv['labels']==col] = f1
results_cv['precision'][results_cv['labels']==col] = precision
results_cv['recall'][results_cv['labels']==col] = recall<install_modules>
history = model.fit(X_train_freq, y_train, epochs = 20, batch_size = 512, validation_data =(X_val_freq, y_val))
!pip install fastai2 --quiet<import_modules>
vectorizer = TfidfVectorizer(max_features = vocab_size)
 vectorizer.fit(list(train_data['prep_text'])+ list(test_data['text']))
X_train_tfidf = vectorizer.transform(list(train_data['prep_text'])).toarray()
X_test_tfidf = vectorizer.transform(list(test_data['text'])).toarray()
y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape {X_train_tfidf.shape}")
 print(f"X_test shape {X_test_tfidf.shape}")
 print(f"y_train shape {y_train.shape}" )
from fastai2.text.all import *<define_variables>
X_train_tfidf, X_val_tfidf, y_train, y_val = train_test_split(X_train_tfidf, y_train, test_size = 0.2, random_state = 42)
 print(f"X_train shape: {X_train_tfidf.shape}")
 print(f"X_val shape: {X_val_tfidf.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"y_val shape: {y_val.shape}" )
path = Path('.. /input/jigsaw-toxic-comment-classification-challenge' )<load_pretrained>
history = model.fit(X_train_tfidf, y_train, epochs = 20, batch_size = 512, validation_data =(X_val_tfidf, y_val))
with ZipFile(path/'train.csv.zip', 'r')as zip_ref:
zip_ref.extractall('.. /output/kaggle/working')
 with ZipFile(path/'test.csv.zip', 'r')as zip_ref:
zip_ref.extractall('.. /output/kaggle/working')
 with ZipFile(path/'test_labels.csv.zip', 'r')as zip_ref:
zip_ref.extractall('.. /output/kaggle/working')
 with ZipFile(path/'sample_submission.csv.zip', 'r')as zip_ref:
zip_ref.extractall('.. /output/kaggle/working' )<define_variables>
embedding_dict={}
with open('.. /input/glovetwitter27b100dtxt/glove.twitter.27B.100d.txt','r')as f:
for line in f:
values=line.split()
word = values[0]
vectors=np.asarray(values[1:],'float32')
 embedding_dict[word]=vectors
f.close()
path_w = Path('.. /output/kaggle/working' )<load_from_csv>
vocab_size = 10000
tokenizer = Tokenizer(num_words = vocab_size, oov_token = 'UNK')
 tokenizer.fit_on_texts(list(train_data['prep_text'])+ list(test_data['text']))
max_len = 15
X_train_seq = tokenizer.texts_to_sequences(train_data['prep_text'])
 X_test_seq = tokenizer.texts_to_sequences(test_data['text'])
 X_train_seq = pad_sequences(X_train_seq, maxlen = max_len, truncating = 'post', padding = 'post')
 X_test_seq = pad_sequences(X_test_seq, maxlen = max_len, truncating = 'post', padding = 'post')
 y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape: {X_train_seq.shape}")
 print(f"X_test shape: {X_test_seq.shape}")
 print(f"y_train shape: {y_train.shape}" )
df = pd.read_csv(path_w/'train.csv' )<create_dataframe>
X_train_seq, X_val_seq, y_train, y_val = train_test_split(X_train_seq, y_train, test_size = 0.2, random_state = 42)
 print(f"X_train shape: {X_train_seq.shape}")
 print(f"X_val shape: {X_val_seq.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"y_val shape: {y_val.shape}" )
blocks =(TextBlock.from_df(text_cols='comment_text', is_lm=True, res_col_name='text'))<load_from_csv>
num_words = len(tokenizer.word_index)
 print(f"Number of unique words: {num_words}" )
test_df = pd.read_csv(path_w/'test.csv' )<concatenate>
embedding_matrix=np.zeros(( num_words,100))
for word,i in tokenizer.word_index.items() :
if i < num_words:
emb_vec = embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i] = emb_vec
text_df = pd.Series.append(df['comment_text'], test_df['comment_text'] )<create_dataframe>
n_latent_factors = 100
model_glove = Sequential()
model_glove.add(layers.Embedding(num_words, n_latent_factors, weights = [embedding_matrix],
input_length = max_len, trainable=True))
model_glove.add(layers.Flatten())
 model_glove.add(layers.Dropout(0.5))
model_glove.add(layers.Dense(1, activation='sigmoid'))
model_glove.summary()
text_df = pd.DataFrame(text_df )<load_from_csv>
model_glove.compile(optimizer = optimizers.RMSprop(lr=0.001),
loss = losses.binary_crossentropy,
metrics = [metrics.binary_accuracy])
 history = model_glove.fit(X_train_seq,
y_train,
epochs=20,
batch_size=512,
validation_data=(X_val_seq, y_val))
get_x = ColReader('text')
 splitter = RandomSplitter(0.1, seed=42 )<create_dataframe>
max_len = 15
X_train_seq = tokenizer.texts_to_sequences(train_data['prep_text'])
 X_test_seq = tokenizer.texts_to_sequences(test_data['text'])
 X_train_seq = pad_sequences(X_train_seq, maxlen = max_len, truncating = 'post', padding = 'post')
 X_test_seq = pad_sequences(X_test_seq, maxlen = max_len, truncating = 'post', padding = 'post')
 y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape: {X_train_seq.shape}")
 print(f"X_test shape: {X_test_seq.shape}")
 print(f"y_train shape: {y_train.shape}
")
 n_latent_factors = 100
model_glove = Sequential()
model_glove.add(layers.Embedding(num_words, n_latent_factors, weights = [embedding_matrix],
input_length = max_len, trainable=True))
model_glove.add(layers.Flatten())
 model_glove.add(layers.Dropout(0.5))
model_glove.add(layers.Dense(1, activation='sigmoid'))
print(f"{model_glove.summary() }
")
 model_glove.compile(optimizer = optimizers.RMSprop(lr=0.001),
loss = losses.binary_crossentropy,
metrics = [metrics.binary_accuracy])
 history = model_glove.fit(X_train_seq,
y_train,
epochs=20,
batch_size=512 )
lm_dblock = DataBlock(blocks=blocks,
get_x=get_x,
splitter=splitter )<load_pretrained>
vocab_size = 1000
tokenizer = Tokenizer(num_words = vocab_size, oov_token = 'UNK')
 tokenizer.fit_on_texts(list(train_data['text'])+ list(test_data['text']))
X_train_wc = tokenizer.texts_to_matrix(train_data['text'], mode = 'count')
 X_test_wc = tokenizer.texts_to_matrix(test_data['text'], mode = 'count')
 y_train = np.array(train_data['target'] ).astype(int)
 print(f"X_train shape: {X_train_wc.shape}")
 print(f"X_test shape: {X_test_wc.shape}")
 print(f"y_train shape: {y_train.shape}")
 X_train_wc, X_val_wc, y_train, y_val = train_test_split(X_train_wc, y_train, test_size = 0.2, random_state = 42)
 print(f"X_train shape: {X_train_wc.shape}")
 print(f"X_val shape: {X_val_wc.shape}")
 print(f"y_train shape: {y_train.shape}")
 print(f"y_val shape: {y_val.shape}
")
 model = setup_model()
history = model.fit(X_train_wc, y_train, epochs = 20, batch_size = 512, validation_data =(X_val_wc, y_val))
<find_best_params><EOS> 
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 test_pred = model_glove.predict(X_test_seq)
 test_pred_int = test_pred.round().astype('int')
 submission['target'] = test_pred_int
submission.to_csv('submission.csv', index=False )