Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<choose_model_class>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 display(train)
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 display(test )
model=Sequential()
model.add(layers.Conv2D(64,(3,3), padding='same', input_shape=(28, 28, 1)))
 model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.Conv2D(64,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.Conv2D(64,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.MaxPooling2D(2, 2))
model.add(layers.Dropout(0.2))
model.add(layers.Conv2D(128,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.Conv2D(128,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.Conv2D(128,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.MaxPooling2D(2,2))
model.add(layers.Dropout(0.2))
model.add(layers.Conv2D(256,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.Conv2D(256,(3,3), padding='same'))
model.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.MaxPooling2D(2,2))
model.add(layers.Dropout(0.4))
model.add(layers.Flatten())
 model.add(layers.Dense(256))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.BatchNormalization())
 model.add(layers.Dense(256))
model.add(layers.LeakyReLU(alpha=0.1))
model.add(layers.BatchNormalization())
 model.add(layers.Dense(10, activation='softmax')) 
<choose_model_class>
train.drop(
 [
6449, 7034, 3589, 3591, 3597, 3600, 3603,
3604, 3610, 3613, 3614, 119, 106, 115,
2666, 2679, 1356, 7609, 3382, 1335, 2655,
2674, 1343, 4291, 4303, 1345, 48, 3374,
7600, 164, 5292, 2352, 4308, 4306, 4310,
1332, 1156, 7610, 2441, 2449, 2454, 2477,
2452, 2456, 3390, 7611, 6656, 1360, 5771,
4351, 5073, 4601, 5665, 7135, 5720, 5723,
5734, 1623, 7533, 7537, 7026, 4834, 4631,
3461, 6366, 6373, 6377, 6378, 6392, 2828,
2841, 1725, 3795, 1251, 7607
], inplace=True 
)
 duplicates = pd.concat(x for _, x in train.groupby(["text"])if len(x)> 1)
 with pd.option_context("display.max_rows", None, "max_colwidth", 240):
display(duplicates[["id", "target", "text"]] )
optimizer = RMSprop(learning_rate=0.002,rho=0.9)
 model.compile(optimizer=optimizer,loss='sparse_categorical_crossentropy',metrics=['accuracy'] )<train_model>
train.drop(
 [
4290, 4299, 4312, 4221, 4239, 4244, 2830,
2831, 2832, 2833, 4597, 4605, 4618, 4232,
4235, 3240, 3243, 3248, 3251, 3261, 3266,
4285, 4305, 4313, 1214, 1365, 6614, 6616,
1197, 1331, 4379, 4381, 4284, 4286, 4292,
4304, 4309, 4318, 610, 624, 630, 634, 3985,
4013, 4019, 1221, 1349, 6091, 6094,
6103, 6123, 5620, 5641
], inplace=True
)
datagen = ImageDataGenerator(
 rotation_range=11,
zoom_range=0.4,
width_shift_range=0.3,
height_shift_range=0.3, 
)
 datagen.fit(xtrain )<choose_model_class>
def fix_text_issues(x):
x = x.lower()
x = x.replace("&amp;", "and")
 x = x.replace("&lt;", "<")
 x = x.replace("&gt;", ">")
 x = re.sub("(\W|^)hwy\.( \W)", "\\1highway\\2", x)
 x = re.sub("(\W|^)ave.( \W)", "\\1avenue\\2", x)
 x = re.sub("(\W|^)fyi(\W)", "\\1for your information\\2", x)
 x = re.sub("(\W|^)ain't(\W)", "\\1am not\\2", x)
 x = re.sub("(\W|^)can't(\W)", "\\1cannot\\2", x)
 x = re.sub("(\W|^)cant(\W)", "\\1cannot\\2", x)
 x = re.sub("(\W|^)rt(\W)", "\\1retweet\\2", x)
 x = x.replace("g'day", "good day")
 x = x.replace("giv'n", "given")
 x = x.replace("let's", "let us")
 x = x.replace("ma'am", "madam")
 x = x.replace("ne'er", "never")
 x = x.replace("o'clock", "of the clock")
 x = x.replace("o'er", "over")
 x = x.replace("ol'", "old")
 x = x.replace("shan't", "shall not")
 x = x.replace("y'all", "you all")
 x = x.replace("'tis", "it is")
 x = x.replace("'hood", "neighborhood")
 x = x.replace("can ªt", "cannot")
 x = x.replace("åÊ", " ")
 x = x.replace("ÛÏ", " ")
 x = x.replace("?Û", " ")
 x = x.replace("Û_", " ")
 x = x.replace("Û÷", " ")
 x = x.replace("ûò", " ")
 x = x.replace("û", " ")
 x = re.sub("\W'twas", " it was", x)
 x = re.sub("\W'cause", " because", x)
 x = re.sub("(\w)'ve", "\\1 have", x)
 x = re.sub("(\w)n't", "\\1 not", x)
 x = re.sub("(\w)'s", "\\1 is", x)
 x = re.sub("(\w)'d", "\\1 had", x)
 x = re.sub("(\w)'ll", "\\1 will", x)
 x = re.sub("(\w)'re", "\\1 are", x)
 x = re.sub("(\w)'m", "\\1 am", x)
 x = re.sub("http[s]*://t.co/\S+", "", x)
 x = x.replace("...", " ")
 x = strip_multiple_whitespaces(x)
 x = strip_punctuation(x)
 x = stem_text(x)
 return x.strip()
def clean_text(df):
df["text"] = df["text"].apply(fix_text_issues)
 clean_text(train)
 clean_text(test )
learning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(
 monitor='loss',
factor=0.2,
patience=2,
verbose=1,
mode="auto",
min_delta=0.0001,
cooldown=0,
min_lr=0.00001 
)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=300, restore_best_weights=True )<train_model>
pipeline = Pipeline([('tfidf', TfidfVectorizer(decode_error="ignore")) ,('clf', SVC(random_state=2020)) ])
 parameters = {
'tfidf__ngram_range':(( 1,1),(1,2),(2,2)) ,
'tfidf__use_idf':(True, False),
'tfidf__smooth_idf':(True, False),
'tfidf__sublinear_tf':(True, False),
'clf__C':(1.5, 1.7, 1.9),
}
grid = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=3, cv=3)
 grid_result = grid.fit(train["text"], train["target"])
 print("Best score: {:0.5f}".format(grid_result.best_score_))
print("Best parameters: {}".format(grid_result.best_params_))
history=model.fit_generator(datagen.flow(xtrain, ytrain, batch_size=1024),
steps_per_epoch=len(xtrain)//1024,
epochs=50,
validation_data=(np.array(xval),np.array(yval)) ,
validation_steps=50,
callbacks=[learning_rate_reduction, es])
<save_to_csv>
x_train, x_valid, y_train, y_valid = train_test_split(
 train["text"], train["target"], test_size=0.2, random_state=2020 
)
 vectorizer = TfidfVectorizer(
 decode_error="ignore",
ngram_range=(1,2),
smooth_idf=False,
sublinear_tf=True,
use_idf=True 
)
 x_train_tfidf = vectorizer.fit_transform(x_train)
 x_valid_tfidf = vectorizer.transform(x_valid)
 model = SVC(random_state=2020, C=1.7)
 model.fit(x_train_tfidf, y_train)
 train_predictions = model.predict(x_valid_tfidf)
 print(classification_report(y_valid, train_predictions, target_names=["Not Real", "Real"]))
score = model.score(x_valid_tfidf, y_valid)
 print("--> Mean accuracy {:0.5}".format(score))
print("" )
ytest = model.predict_classes(xtest)
 id_col = np.arange(ytest.shape[0])
 submission = pd.DataFrame({'id': id_col, 'label': ytest})
 submission.to_csv('submission.csv', index = False )<define_variables>
vectorizer = TfidfVectorizer(
 decode_error="ignore",
ngram_range=(1,2),
smooth_idf=False,
sublinear_tf=True,
use_idf=True 
)
 train_tfidf = vectorizer.fit_transform(train["text"])
 model = SVC(random_state=2020, C=1.7)
 model.fit(train_tfidf, train["target"] )
<import_modules><EOS> 
clean_text(test)
 test_tfidf = vectorizer.transform(test["text"])
 predictions = model.predict(test_tfidf)
 submission = pd.DataFrame({"id": test["id"], "target": predictions})
 submission.to_csv("submission.csv", index=False )