Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<count_missing_values>
Embedding,
BatchNormalization,
Dense,
TimeDistributed,
Dropout,
Bidirectional,
Flatten,
GlobalMaxPool1D)
 precision_score,
recall_score,
f1_score,
classification_report,
accuracy_score 
)
print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi1'].value_counts())
 print(train_df['tipovivi1'].value_counts())
<feature_engineering>
dataset = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )
train_df.loc[(train_df['v2a1'].isnull() & train_df['tipovivi1'] == 1), 'v2a1'] = 0
test_df.loc[(test_df['v2a1'].isnull() & test_df['tipovivi1'] == 1), 'v2a1'] = 0<count_missing_values>
train = dataset.text.values
test = test.text.values
sentiments = dataset.target.values
print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi1'].value_counts() )<count_missing_values>
word_tokenizer = Tokenizer()
word_tokenizer.fit_on_texts(train)
 vocab_length = len(word_tokenizer.word_index)+ 1
print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi2'].value_counts())
 print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi3'].value_counts())
 print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi4'].value_counts())
 print(train_df.loc[train_df['v2a1'].isnull() ]['tipovivi5'].value_counts() )<feature_engineering>
def metrics(pred_tag, y_test):
print("F1-score: ", f1_score(pred_tag, y_test))
print("Precision: ", precision_score(pred_tag, y_test))
print("Recall: ", recall_score(pred_tag, y_test))
print("Acuracy: ", accuracy_score(pred_tag, y_test))
print("-"*50)
 print(classification_report(pred_tag, y_test))
def embed(corpus):
return word_tokenizer.texts_to_sequences(corpus )
train_df.loc[(train_df['v2a1'].isnull() & train_df['tipovivi4'] == 1), 'v2a1'] = 0
test_df.loc[(test_df['v2a1'].isnull() & test_df['tipovivi4'] == 1), 'v2a1'] = 0<count_values>
longest_train = max(train, key=lambda sentence: len(word_tokenize(sentence)))
 length_long_sentence = len(word_tokenize(longest_train))
padded_sentences = pad_sequences(embed(train), length_long_sentence, padding='post')
 test_sentences = pad_sequences(
 embed(test),
length_long_sentence,
padding='post'
)
print(train_df.loc[train_df['v2a1'].isnull() ]['Target'].value_counts() )<data_type_conversions>
embeddings_dictionary = dict()
embedding_dim = 200
glove_file = open('.. /input/glove-twitter/glove.twitter.27B.' + str(embedding_dim)+ 'd.txt', encoding="utf8")
 for line in glove_file:
records = line.split()
word = records[0]
vector_dimensions = np.asarray(records[1:], dtype='float32')
 embeddings_dictionary [word] = vector_dimensions
glove_file.close()
train_df['v2a1'].fillna(train_df['v2a1'].mean() , inplace = True)
 test_df['v2a1'].fillna(test_df['v2a1'].mean() , inplace = True )<data_type_conversions>
embedding_matrix = np.zeros(( vocab_length, embedding_dim))
for word, index in word_tokenizer.word_index.items() :
if index >= vocab_length:
continue
embedding_vector = embeddings_dictionary.get(word)
 if embedding_vector is not None:
embedding_matrix[index] = embedding_vector
train_df.fillna(train_df.mean() , inplace = True)
 test_df.fillna(test_df.mean() , inplace = True )<find_best_params>
def BLSTM() :
model = Sequential()
model.add(Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights = [embedding_matrix],
input_length=length_long_sentence,
trainable=False))
model.add(Bidirectional(LSTM(16)))
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
 model.summary()
return model
best_correlation = best_correlations.index
best_correlation<define_variables>
def BLSTM() :
model = Sequential()
model.add(Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights = [embedding_matrix],
input_length=length_long_sentence,
trainable=False))
model.add(Bidirectional(LSTM(length_long_sentence, return_sequences = True, recurrent_dropout=0.2)))
 model.add(GlobalMaxPool1D())
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
 return model
d = {'dependency':'dependency, Dependency rate', 'v18q1':'v18q1, number of tablets household owns', 'epared1':'epared1, if walls are bad', 'qmobilephone':'qmobilephone,
'pisocemento':'pisocemento, =1 if predominant material on the floor is cement',
'eviv1':'eviv1, =1 if floor are bad', 'instlevel8':'instlevel8, =1 undergraduate and higher education', 'rooms':'rooms, number of all rooms in the house', 'r4h1':'r4h1, Males younger than 12 years of age',
'v18q': 'v18q, owns a tablet:', 'edjefe':'edjefe, years of education of male head of household', 'SQBedjefe':'SQBedjefe, years of education of male head of household squared',
'etecho3':'etecho3, =1 if roof are good', 'r4m1':'r4m1, Females younger than 12 years of age', 'SQBovercrowding':'SQBovercrowding, overcrowding squared',
'paredblolad':'paredblolad, =1 if predominant material on the outside wall is block or brick', 'SQBmeaned':'SQBmeaned, square of the mean years of education of adults(>=18)in the household',
'pisomoscer':'pisomoscer, "=1 if predominant material on the floor is mosaic, ceramic, terrazo"', 'overcrowding':'overcrowding,
'eviv3':'eviv3, =1 if floor are good', 'SQBescolari' :'SQBescolari, years of schooling squared',
'escolari':'escolari, years of schooling', 'cielorazo':'cielorazo, =1 if the house has ceiling', 'SQBhogar_nin':'SQBhogar_nin, Number of children 0 to 19 in household, squared',
'r4t1':'r4t1, persons younger than 12 years of age', 'hogar_nin':'hogar_nin, Number of children 0 to 19 in household',
'meaneduc':'meaneduc,average years of education for adults(18+)', 'Target':'Target', 'elimbasu5':'elimbasu5, "=1 if rubbish disposal mainly by throwing in river, creek or sea"'}<count_values>
reduce_lr = ReduceLROnPlateau(
 monitor='val_loss',
factor=0.2,
verbose =1,
patience=5,
min_lr=0.001 
)
 for idx in range(5):
print("*"*20 + '
Modelo: ' + str(idx)+ '
')
 reduce_lr = ReduceLROnPlateau(
 monitor='val_loss',
factor=0.2,
verbose =1,
patience=5,
min_lr=0.001 
)
 checkpoint = ModelCheckpoint(
 'model_' + str(idx)+ '.h5',
monitor='val_loss',
mode='auto',
verbose=1,
save_weights_only = True,
save_best_only=True 
)
 X_train, X_test, y_train, y_test = train_test_split(
 padded_sentences,
sentiments,
test_size=0.5 
)
 model = BLSTM()
model.fit(X_train,
y_train,
batch_size=32,
epochs=15,
validation_data=[X_test, y_test],
callbacks = [reduce_lr, checkpoint],
verbose=1 )
print(len(train_df.loc[(train_df['SQBmeaned']>900)]))
print(train_df['SQBmeaned'].value_counts(sort = True))<filter>
%%time
x_models = []
labels = []
for idx in glob('*.h5'):
model = BLSTM()
model.load_weights(idx)
 x_models.append(model)
 for idx in x_models:
preds = idx.predict_classes(test_sentences)
 labels.append(preds)
 labels = scipy.stats.mode(labels)[0]
labels = np.squeeze(labels )
<drop_column><EOS> 
submission.target = labels
submission.to_csv("submission.csv", index=False)
 submission.target.value_counts().plot.bar() ;