Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
train_data = applyFE(train_data )<prepare_x_and_y>
import numpy as np
import pandas as pd
import tensorflow as tf
import tokenization
from plotnine import *
from tensorflow.keras.layers import Dense, Input, Concatenate, BatchNormalization
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import EarlyStopping
from collections import Counter
from sklearn.model_selection import StratifiedShuffleSplit
x_train = train_data.copy()
y_train = train_target.copy()<train_on_grid>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
param_grid = {
'n_estimators': [270],
'max_depth' : [14]
}
rfc=RandomForestClassifier(class_weight = 'balanced_subsample', n_estimators = 200,max_depth = 16)
 scorer = metrics.make_scorer(metrics.f1_score, average = 'weighted')
 rf_cv = GridSearchCV(estimator=rfc, param_grid=param_grid, scoring=scorer, cv= 3)
 rf_cv.fit(x_train, y_train )<train_model>
input_dir = '/kaggle/input/nlp-getting-started/'
output_dir = '/kaggle/working/'
checkpoint_dir = '/kaggle/input/checkpoint-nlp-disaster/'
word1vec_dir = '/kaggle/input/word2vec-google/GoogleNews-vectors-negative300.bin'
df_model, df_submission = pd.read_csv(input_dir + 'train.csv'), pd.read_csv(input_dir + 'test.csv')
 df_model.head()
print('Best score for training data:', rf_cv.best_score_,"
")
 print('Best n_estimators:',rf_cv.best_estimator_.n_estimators,"
")
 print('Best max_depth:',rf_cv.best_estimator_.max_depth,"
" )<compute_train_metric>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
y_pred_validate = rf_cv.predict(x_train)
 print("Classification report for rf model %s:
%s
"
%(rf_cv, metrics.classification_report(y_train, y_pred_validate)) )<load_from_csv>
def tokenize_df(df_in):
df_out = df_in.copy()
df_out['str_tokens'] = df_out['text'].apply(tokenizer.tokenize)
 apply_cls_sep = lambda X: ['[CLS]'] + X + ['[SEP]']
df_out['str_tokens'] = df_out['str_tokens'].apply(apply_cls_sep)
 df_out['id_tokens'] = df_out['str_tokens'].apply(tokenizer.convert_tokens_to_ids)
 df_out['n_tokens'] = df_out['id_tokens'].apply(len)
 return df_out
df_model = tokenize_df(df_model)
 df_submission = tokenize_df(df_submission)
 df_model.head()
path = '/kaggle/input/costa-rican-household-poverty-prediction/test.csv'
test_data = pd.read_csv(path)
 test_data = applyFE(test_data)
 y_pred = rf_cv.predict(test_data)
 path = '/kaggle/input/costa-rican-household-poverty-prediction/sample_submission.csv'
test = pd.read_csv(path)
 test['Target'] = y_pred
test.to_csv("submission.csv", index= False)
<load_from_csv>
def define_mask(df_in, window_size=140):
df_out = df_in.copy()
df_out['pad_size'] = window_size - df_out['n_tokens']
masked_list, masked_id_token = list() , list()
for i in range(0, df_out.shape[0]):
masked_list.append([0] * df_out['pad_size'][i] + [1] * df_out['n_tokens'][i])
 masked_id_token.append([0] * df_out['pad_size'][i] + df_out['id_tokens'][i])
 df_out['masked_list'], df_out['masked_id_token'] = masked_list, masked_id_token
return df_out
df_model = define_mask(df_model)
 df_submission = define_mask(df_submission)
 df_model.head()
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape<concatenate>
def build_minimal_model(bert_layer, max_len=140):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name='input_word_ids')
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name='input_mask')
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name='segment_ids')
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model
df_all = df.append(test)
 df_all.shape<count_values>
model = build_minimal_model(bert_layer, max_len=140)
 model.summary()
df_all['dependency'].value_counts()<count_values>
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.8, random_state=42)
 for train_index, test_index in sss.split(df_model, df_model['target']):
df_model = df_model.iloc[test_index.tolist() + train_index.tolist() , :]
df_all['edjefa'].value_counts()<count_values>
def extract_inputs(df_in, max_len=140):
return np.array(df_in['masked_id_token'].tolist()),\
np.array(df_in['masked_list'].tolist()),\
np.array([[0] * max_len for i in range(0, df_in.shape[0])])
 train_input = extract_inputs(df_model)
 test_input = extract_inputs(df_submission)
 train_labels = df_model['target']
df_all['edjefe'].value_counts()<data_type_conversions>
flag_fit_models = False
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int )<count_values>
if flag_fit_models:
train_history_minimal_model = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
batch_size=16,
callbacks=[EarlyStopping(restore_best_weights=True, patience=1)])
 else:
train_history_minimal_model = pd.read_csv(checkpoint_dir +
'train_history_minimal_model_no_label.csv' )
df_all['dependency'].value_counts()<data_type_conversions>
if flag_fit_models:
pd.DataFrame.from_dict(train_history_minimal_model.history ).\
to_csv(output_dir + 'train_history_minimal_model_no_label.csv', index=False)
 pd.DataFrame({
'id': df_submission['id'],
'target': [X[0] for X in list(model.predict(test_input)) ]
} ).to_csv(output_dir + 'predictions_model_no_sentence_labels.csv', index=False)
 else:
df_out = pd.read_csv(checkpoint_dir + 'predictions_model_no_sentence_labels.csv' ).loc[:, ['id', 'target']]
df_out['target'] = df_out['target'].apply(lambda X: 1 if X >= 0.5 else 0)
 df_out.to_csv(output_dir + 'predictions_model_no_sentence_labels.csv', index=False )
df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )<count_missing_values>
df_eda = pd.DataFrame(dict(all_texts=df_model['text'].str.lower().str.split() , target=df_model['target'].tolist()))
df_eda.head()
df_all.isnull().sum()<count_missing_values>
df_eda['hashtag_count'] = df_eda['all_texts'].apply(lambda X: sum([1.for Y in X if Y[0] == '
df_eda['at_count'] = df_eda['all_texts'].apply(lambda X: sum([1.for Y in X if Y[0] == '@']))
df_eda['exclamation_count'] = df_eda['all_texts'].apply(lambda X: sum([1.for Y in ''.join(X)if Y == '!']))
df_eda['interrogation_count'] = df_eda['all_texts'].apply(lambda X: sum([1.for Y in ''.join(X)if Y == '?']))
df_eda.head()
df_all[df_all['parentesco1'] == 1]['v2a1'].isnull().sum()<count_values>
df_melt_eda = pd.melt(df_eda, id_vars=['target'], value_vars=['hashtag_count', 'at_count', 'exclamation_count'])
 df_melt_eda['bigger_than_0'] = df_melt_eda['value'].apply(lambda X: 1.if X > 0 else 0)
 df_melt_eda.head()
df_all['v18q'].value_counts()<count_missing_values>
df_bigger_than_0_prob = df_melt_eda.groupby(['target', 'variable'] ).\
agg({'bigger_than_0': ['mean', 'count']})
 df_bigger_than_0_prob = df_bigger_than_0_prob['bigger_than_0'].reset_index()
df_bigger_than_0_prob['target'] = df_bigger_than_0_prob['target'].apply(lambda X: 'Disaster!' if X > 0 else 'Fake...')
 df_bigger_than_0_prob['std'] = np.sqrt(( df_bigger_than_0_prob['mean'] *\ 
(1 - df_bigger_than_0_prob['mean'])) /\
df_bigger_than_0_prob['count'])
 df_bigger_than_0_prob['mean']
df_bigger_than_0_prob
df_all.isnull().sum().sort_values()<categorify>
df_model['n_characters'] = df_model['text'].apply(lambda X: len(''.join(X.split())))
 df_model['n_']
df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True )<define_variables>
def index_to_row(df_in):
df_in['Epoch'] = df_in.index
return df_in
history_extra_features = index_to_row(pd.read_csv(checkpoint_dir + 'train_history_extra_features.csv'))
history_minimal_model_no_label = index_to_row(pd.read_csv(checkpoint_dir + 'train_history_minimal_model_no_label.csv'))
history_minimal_model_with_label = index_to_row(pd.read_csv(checkpoint_dir + 'train_history_minimal_model_with_label.csv'))
history_with_correction = index_to_row(pd.read_csv(checkpoint_dir + 'train_history_with_correction.csv'))
<save_to_csv><EOS> 
history_extra_features['Model'] = 'Extra Features'
history_minimal_model_no_label['Model'] = 'Minimal'
history_minimal_model_with_label['Model'] = 'Label'
history_with_correction['Model'] = 'Auto Correction'
df_histories = pd.concat([
history_extra_features,
history_minimal_model_no_label,
history_minimal_model_with_label,
history_with_correction
], axis=0, ignore_index=True)
 df_histories.head()