Kaggle competition: Natural Language Processing with Disaster Tweets
def create_tokenizer(lines):
tokenizer = Tokenizer()
tokenizer.fit_on_texts(lines)
 return tokenizer<prepare_x_and_y>
df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv", na_filter=False)
 df.head()
X = train.text
y = train.target
test_id = test.id
test.drop(["id","location","keyword"],1,inplace = True )<split>
nlp = spacy.load("en_core_web_sm")
 def preprocess(text):
doc = nlp(text)
 token_semstop = [word for word in doc if not word.is_stop if not word.text == '
text = ' '.join(token.lower_ for token in token_semstop)
 text = re.sub(r'(@\w+|https?:\S+)', '', text)
 text = text.replace(r'&amp;?', r'and')
 text = re.sub(r'(&gt|&lt)', '', text)
 text = text.encode(encoding="ascii", errors="ignore" ).decode()
return text
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 42 )<string_transform>
train_df['text'] = train_df['text'].apply(preprocess)
 train_df.head()
tokenizer = create_tokenizer(X_train)
 X_train_set = tokenizer.texts_to_matrix(X_train, mode = 'freq')
<choose_model_class>
second_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv", na_filter=False)
 second_df.head()
test_df = second_df[['text']].copy()
test_df['text'] = test_df['text'].apply(preprocess)
def define_model(n_words):
model = Sequential()
model.add(Dense(128, input_shape=(n_words,), activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics = [get_f1])
 model.summary()
plot_model(model, to_file='model.png', show_shapes=True)
 return model<train_model>
vectorizer = TfidfVectorizer(use_idf=True, ngram_range=(1,2), preprocessor=preprocess)
 tfidf_data = vectorizer.fit_transform(train_df['text'])
model.fit(X_train_set,y_train,epochs=10,verbose=2 )<predict_on_test>
labels = train_df['target'].values
mnb = MultinomialNB()
mnb.fit(tfidf_data, labels)
 X_train, X_test, y_train, y_test = train_test_split(tfidf_data, labels, random_state=42, test_size=0.2)
 print(classification_report(y_test, mnb.predict(X_test), digits=4))
X_test_set = tokenizer.texts_to_matrix(X_test, mode = 'freq')
 y_pred = model.predict_classes(X_test_set )<compute_test_metric>
submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 tfidf_test = vectorizer.transform(test_df['text'])
 test_df['target'] = mnb.predict(tfidf_test)
 test_df.head()
<prepare_x_and_y><EOS> 
submission['target'] = test_df['target']
submission.to_csv("sample_submission.csv", index=False)
 submission.head()