Kaggle competition: Natural Language Processing with Disaster Tweets
show_random_image(dig_images_2D )<data_type_conversions>
train_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 sub_df = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )
dig_images_2D = dig_images_2D.astype('float')
 dig_images_2D /= 255<split>
train_df.count()
images_train,imames_val,labels_train,labels_val = train_test_split(train_images_2D, train_labels,
random_state=42,test_size=0.15 )<init_hyperparams>
test_df.count()
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(images_train )<load_from_csv>
from nltk.corpus import stopwords
import nltk
import re
import string
from sklearn import feature_extraction, linear_model, model_selection, preprocessing
test_data = pd.read_csv(path + 'test.csv', index_col='id')
 test_data<load_from_csv>
def change_text(text):
text = re.sub(r"n't", " not", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r"'s", " is", text)
 text = re.sub(r"'d", " would", text)
 text = re.sub(r"'ll", " will", text)
 text = re.sub(r"'t", " not", text)
 text = re.sub(r"'ve", " have", text)
 text = re.sub(r"'m", " am", text)
 text = re.sub(r"won't", "will not", text)
 text = re.sub(r"can't", "can not", text)
 return text
train_df['text'] = train_df['text'].apply(lambda x : change_text(x))
test_df['text'] = test_df['text'].apply(lambda x : change_text(x))
train_df['text'].head()
submission = pd.read_csv(path + 'sample_submission.csv', index_col='id')
 submission<data_type_conversions>
def clean_preprocessor(text):
text = text.lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub("\\W"," ",text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text
train_df['text'] = train_df['text'].apply(lambda x : clean_preprocessor(x))
test_df['text'] = test_df['text'].apply(lambda x : clean_preprocessor(x))
train_df['text'].head()
test_images_2D = test_images_2D.astype('float')
 test_images_2D /= 255<define_variables>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train_df['text'] = train_df['text'].apply(lambda x : remove_emoji(x))
test_df['text'] = test_df['text'].apply(lambda x : remove_emoji(x))
input_shape =(dim, dim, 1)
 num_classes = 10<choose_model_class>
from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.linear_model import LogisticRegression
optimizer = 'rmsprop'
loss = 'categorical_crossentropy'
metrics = ['accuracy']<choose_model_class>
stopwords = stopwords.words('english')
 count_vectorizer = CountVectorizer(token_pattern=r'\w{1,}', ngram_range=(1, 2), stop_words = stopwords)
 train_vector = count_vectorizer.fit_transform(train_df['text'])
 test_vector = count_vectorizer.transform(test_df['text'] )
epochs = 100
batch_size = 1024
early_stop = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=3,
verbose=True,
mode='auto',
baseline=None,
restore_best_weights=False)
 callbacks = [early_stop]<choose_model_class>
clf = LogisticRegression(C=0.9,max_iter=1000,penalty='l2')
 scores = model_selection.cross_val_score(clf, train_vector, train_df["target"], cv=7, scoring="f1")
 print(scores )
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),
activation='relu',
input_shape=input_shape))
model.add(Conv2D(64,(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))<choose_model_class>
clf.fit(train_vector, train_df["target"] )
kernel_size =(5, 5)
 model = Sequential()
model.add(Conv2D(32, kernel_size=kernel_size, activation='relu',
input_shape=input_shape))
model.add(Conv2D(64, kernel_size=kernel_size, activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))<choose_model_class>
sub_df["target"] = clf.predict(test_vector)
 sub_df.to_csv("sample_submission.csv", index=False )