Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<predict_on_test>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
y_test_pred = model.predict_classes(test_set )<save_to_csv>
df_train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 df_test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
sub = pd.DataFrame()
sub['Id'] = test_id
sub['target'] = y_test_pred
sub.to_csv('submission_1.csv',index=False )<train_model>
def format_keyword(df):
df["keyword"] = df["keyword"].fillna(".")
 df["keyword"] = df.keyword.str.replace("%20"," " )
t = Tokenizer()
t.fit_on_texts(X_train.tolist() )<define_variables>
df_train.loc[df_train.target==0]["keyword"].value_counts()
vocab_size = len(t.word_index)+ 1<load_from_csv>
df_count = df_train.text.str.split().str.len()
max(df_count )
embeddings_index = dict()
f = open('.. /input/glove6b100dtxt/glove.6B.100d.txt', mode='rt', encoding='utf-8')
 for line in f:
values = line.split()
word = values[0]
coefs = asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Loaded %s word vectors.' % len(embeddings_index))<categorify>
def process_text(text):
text=text.replace("
","")
 text = re.sub(r'@\S+','',text)
 text = re.sub(r'
text = re.sub(r'https?://\S+|www\.\S+|http?://\S+','',text)
 text = re.sub('[%s]' % re.escape (
encoded_docs = t.texts_to_sequences(X_train.tolist())
 max_length = 100
padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')
 print(padded_docs )<define_variables>
df_train["text"] = df_train.text.transform(lambda x: process_text(x))
df_test["text"] = df_test.text.transform(lambda x: process_text(x))
mis_spelled = []
embedding_matrix = zeros(( vocab_size, 100))
for word, i in t.word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
else:
mis_spelled.append(word )<train_on_grid>
df_train["appears"]=df_train.groupby("text" ).text.transform("count" )
model = Sequential()
e = Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=100, trainable=False)
 model.add(e)
 model.add(Flatten())
 model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[get_f1])
 model.summary()
model.fit(padded_docs, y_train, epochs=50, verbose=0 )<compute_test_metric>
df_train["target_std"]=df_train.groupby("text" ).target.transform(np.std)
 df_train["target_mean"]=df_train.groupby("text" ).target.transform(np.mean )
loss, accuracy = model.evaluate(padded_docs, y_train, verbose=0 )<categorify>
duplicate_ids = df_train.loc[df_train.target_std>0].sort_values(by=["appears","text"],ascending=False ).index
encoded_docs = t.texts_to_sequences(X_test.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )<predict_on_test>
df_train = df_train.drop(index = duplicate_ids )
y_pred = model.predict_classes(padded_docs )<categorify>
df_train = df_train.drop_duplicates(subset=["text"] )
encoded_docs = t.texts_to_sequences(test.text.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )<predict_on_test>
df_train.reset_index(drop=True,inplace=True)
 df_train
y_test_pred = model.predict_classes(padded_docs )<save_to_csv>
nlp = spacy.load("en_core_web_lg")
 keyword_train = np.array([nlp(text ).vector for text in df_train.keyword])
 keyword_test = np.array([nlp(text ).vector for text in df_test.keyword] )
sub = pd.DataFrame()
sub['Id'] = test_id
sub['target'] = y_test_pred
sub.to_csv('submission_2.csv',index=False )<train_model>
def nlp_vectors(text):
res = []
doc = nlp(text)
 for token in doc:
if not token.is_space:
res.append(token.vector)
 return res
def build_nlp_vectors(df_text):
spacy_vectors =([nlp_vectors(text)for text in df_text])
 max_length = 0;
for vector in spacy_vectors:
max_length = max(max_length, len(vector))
print(f"Maximum Length:{ max_length}")
 for i in range(len(spacy_vectors)) :
while(len(spacy_vectors[i])<max_length):
spacy_vectors[i].append([0]*300)
 spacy_vectors = np.array(spacy_vectors)
 print(f"Shape of spacy vector:{spacy_vectors.shape}")
 return spacy_vectors
<categorify>
nlp_train = build_nlp_vectors(df_train.text )
encoded_docs = t.texts_to_sequences(X_train.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')
 print(padded_docs )<define_variables>
tokenizer = ppb.DistilBertTokenizer.from_pretrained("distilbert-base-uncased")
 bert_model = ppb.DistilBertModel.from_pretrained("distilbert-base-uncased" )
vocab_size<choose_model_class>
def process_data(df_text):
tokens = df_text.apply(lambda text: tokenizer.encode(text,add_special_tokens=True))
max_len = 0;
i = 0;
for token in tokens.values:
max_len = max(max_len,len(token))
print(f"Max Length: {max_len}")
 padded = np.array([i+[0]*(max_len-len(i)) for i in tokens.values])
 attention_mask = np.where(padded !=0, 1,0)
 input_ids = torch.tensor(padded)
 attention_mask = torch.tensor(attention_mask)
 with torch.no_grad() :
last_hidden_states = bert_model(input_ids,attention_mask=attention_mask)
 X = last_hidden_states[0][:,0,:].numpy()
print(X.shape)
 return X
def define_model(vocab_size, max_length):
model = Sequential()
model.add(Embedding(vocab_size, 100, input_length=max_length))
model.add(Conv1D(filters=32, kernel_size=8, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
 model.add(Dense(10, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()
plot_model(model, to_file='model.png', show_shapes=True)
 return model<train_model>
X_train = process_data(df_train.text )
model = define_model(vocab_size, max_length)
 model.fit(padded_docs, y_train, epochs=10, verbose=2 )<compute_test_metric>
y_train = df_train.target
loss, accuracy = model.evaluate(padded_docs, y_train, verbose=0 )<categorify>
X_tr, X_val, nlp_tr, nlp_val, kw_tr, kw_val, y_tr, y_val = train_test_split(X_train,nlp_train, keyword_train, y_train, test_size=0.25, train_size=0.75,shuffle=True )
encoded_docs = t.texts_to_sequences(X_test.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )<predict_on_test>
def build_nn() :
model = tf.keras.Sequential()
model.add(layers.Input(shape=(768,)))
 model.add(layers.Dense(128,activation='tanh'))
model.add(layers.Dropout(0.6))
model.add(layers.Dense(32,activation='tanh'))
model.add(layers.Dropout(0.6))
model.add(layers.Dense(8,activation='tanh'))
model.add(layers.Dense(1,activation='sigmoid'))
model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=tf.keras.optimizers.Adam(1e-4),
metrics=['accuracy'])
 return model
y_pred = model.predict_classes(padded_docs )<categorify>
kfold = KFold(n_splits=4, shuffle=True, random_state=1 )
encoded_docs = t.texts_to_sequences(test.text.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )<predict_on_test>
def eval_f1_score(X_val, y_val, model):
pred_val =(model.predict(X_val)>0.5)
 f1 = f1_score(y_val,pred_val)
 return f1
y_test_pred = model.predict_classes(padded_docs )<save_to_csv>
EPOCHS = 100
BATCH_SIZE = 64
sub = pd.DataFrame()
sub['Id'] = test_id
sub['target'] = y_test_pred
sub.to_csv('submission_cnn.csv',index=False )<save_model>
fold = 0
history_by_fold = []
cv_results = []
for train,val in kfold.split(X_train,y_train):
nn_model = build_nn()
history = nn_model.fit(X_train[train],y_train[train],
validation_data=(X_train[val],y_train[val]),
epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=0)
 scores = nn_model.evaluate(X_train[val],y_train[val],verbose=0)
 print(f"-- Fold {fold} -- ")
 print(f"{nn_model.metrics_names[0]}: {scores[0]}")
 print(f"{nn_model.metrics_names[1]}: {scores[1]}")
 print(f"F1 Score: {eval_f1_score(X_train[val],y_train[val],nn_model)}")
 cv_results.append(scores[1])
 history_by_fold.append(history)
 fold+=1
print(f"{np.mean(cv_results)} +\- {np.std(cv_results)}")
 plot_history(history )
model.save('model.h5' )<choose_model_class>
nn_model = build_nn()
history = nn_model.fit(X_tr,y_tr, validation_data=(X_val,y_val),
epochs=EPOCHS, batch_size=BATCH_SIZE,verbose=0)
 scores= nn_model.evaluate(X_val,y_val,verbose=0)
 print(f"Accuracy: {scores[1]}")
 print(f"F1 Score: {eval_f1_score(X_val,y_val,nn_model)}" )
def define_model(length, vocab_size):
inputs1 = Input(shape=(length,))
embedding1 = Embedding(vocab_size, 100 )(inputs1)
 conv1 = Conv1D(32, 4, activation='relu' )(embedding1)
 drop1 = Dropout(0.5 )(conv1)
 pool1 = MaxPooling1D()(drop1)
 flat1 = Flatten()(pool1)
 inputs2 = Input(shape=(length,))
embedding2 = Embedding(vocab_size, 100 )(inputs2)
 conv2 = Conv1D(32, 6, activation='relu' )(embedding2)
 drop2 = Dropout(0.5 )(conv2)
 pool2 = MaxPooling1D()(drop2)
 flat2 = Flatten()(pool2)
 inputs3 = Input(shape=(length,))
embedding3 = Embedding(vocab_size, 100 )(inputs3)
 conv3 = Conv1D(32, 8, activation='relu' )(embedding3)
 drop3 = Dropout(0.5 )(conv3)
 pool3 = MaxPooling1D()(drop3)
 flat3 = Flatten()(pool3)
 merged = concatenate([flat1, flat2, flat3])
 dense1 = Dense(10, activation='relu' )(merged)
 outputs = Dense(1, activation='sigmoid' )(dense1)
 model = Model(inputs=[inputs1, inputs2, inputs3], outputs=outputs)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()
plot_model(model, show_shapes=True, to_file='multichannel.png')
 return model 
<categorify>
def build_LSTM() :
lstm_model = tf.keras.Sequential()
lstm_model.add(layers.Input(shape=(None,300)))
 lstm_model.add(layers.LSTM(16))
lstm_model.add(layers.Dense(8, activation="tanh"))
lstm_model.add(layers.Dense(8, activation="tanh"))
lstm_model.add(layers.Dense(1,activation="sigmoid"))
lstm_model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=tf.keras.optimizers.Adam(5e-5),
metrics=['accuracy'])
 return lstm_model
encoded_docs = t.texts_to_sequences(X_train.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')
<train_model>
EPOCHS = 30;
BATCH_SIZE = 64;
model = define_model(max_length,vocab_size)
 model.fit([padded_docs,padded_docs,padded_docs], array(y_train), epochs=7, batch_size=16 )<compute_test_metric>
kfold = KFold(n_splits=4, shuffle=True, random_state=1 )
loss, accuracy = model.evaluate([padded_docs,padded_docs,padded_docs], y_train, verbose=0 )<categorify>
fold = 0
history_by_fold = []
cv_results = []
for train, val in kfold.split(nlp_train,y_train):
lstm_model = build_LSTM()
history = lstm_model.fit(nlp_train[train],y_train[train],
validation_data=(nlp_train[val],y_train[val]),
epochs=EPOCHS,batch_size=BATCH_SIZE,verbose=0)
 scores = lstm_model.evaluate(nlp_train[val],y_train[val],verbose=0)
 print(f"-- Fold{fold} --")
 print(f"{lstm_model.metrics_names[0]}: {scores[0]}")
 print(f"{lstm_model.metrics_names[1]}: {scores[1]}")
 print(f"F1 Score: {eval_f1_score(nlp_train[val],y_train[val],lstm_model)}")
 cv_results.append(scores[1])
 history_by_fold.append(history)
 fold+=1
print(f"{np.mean(cv_results)} +\- {np.std(cv_results)}" )
encoded_docs = t.texts_to_sequences(X_test.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )<predict_on_test>
lstm_model = build_LSTM()
history = lstm_model.fit(nlp_tr,y_tr,validation_data=(nlp_val,y_val), epochs=EPOCHS, batch_size=BATCH_SIZE )
_, acc = model.evaluate([padded_docs,padded_docs,padded_docs], array(y_test), verbose=0)
 print('Train Accuracy: %.2f' %(acc*100))<categorify>
valid_predict =(lstm_model.predict(nlp_val)> 0.5)
 f1 = f1_score(y_val, valid_predict)
 print(f" F1 Score: {f1}" )
encoded_docs = t.texts_to_sequences(test.text.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )<predict_on_test>
lr_keywords = LogisticRegression(max_iter=500)
 lr_keywords.fit(kw_tr,y_tr)
 val_pred = lr_keywords.predict(kw_val)
 print(f"Accurcay: {accuracy_score(y_val, val_pred)}")
 print(f"F1 score: {f1_score(y_val,val_pred)}" )
y_test_pred = model.predict([padded_docs,padded_docs,padded_docs] )<save_to_csv>
nn_tr_predict = nn_model.predict(X_tr)
 kw_tr_predict = lr_keywords.predict_proba(kw_tr)[:,1]
lstm_tr_predict = lstm_model.predict(nlp_tr)
 nn_val_predict = nn_model.predict(X_val)
 kw_val_predict = lr_keywords.predict_proba(kw_val)[:,1]
lstm_val_predict = lstm_model.predict(nlp_val)
 kw_tr_predict = kw_tr_predict.reshape(( kw_tr_predict.shape[0],1))
kw_val_predict = kw_val_predict.reshape(( kw_val_predict.shape[0],1))
concat_tr = np.concatenate(( nn_tr_predict, kw_tr_predict, lstm_tr_predict), axis=1)
 concat_val = np.concatenate(( nn_val_predict, kw_val_predict, lstm_val_predict), axis=1)
sub = pd.DataFrame()
sub['Id'] = test_id
sub['target'] = y_test_pred
sub.to_csv('submission_multi-cnn.csv',index=False )<load_from_url>
lr = LogisticRegression()
lr.fit(concat_tr,y_tr)
 val_pred = lr.predict(concat_val)
 print(f"Accurcay: {accuracy_score(y_val, val_pred)}")
 print(f"F1 score: {f1_score(y_val,val_pred)}" )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
X_test = process_data(df_test.text )
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization<load_from_csv>
nlp_test = build_nlp_vectors(df_test.text )
train= pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test=pd.read_csv('.. /input/nlp-getting-started/test.csv' )<categorify>
df_test["nn_predict"]= nn_model.predict(X_test)
 df_test["lstm_predict"]= lstm_model.predict(nlp_test)
 df_test["keyword_predict"] = lr_keywords.predict_proba(keyword_test)[:,1]
features = ["nn_predict","keyword_predict","lstm_predict"]
test_features = df_test[features]
predict = lr.predict(test_features )
<choose_model_class><EOS> 
output = pd.DataFrame({"id":df_test.id, "target":predict})
 output.to_csv("submission.csv",index=False)
 output