Kaggle competition: Natural Language Processing with Disaster Tweets
def treino(df,model,train_submit=False):
if train_submit:
X, y = df.drop("Target",axis=1),df["Target"]
model.fit(X,y)
 return model
if "Target" in df.columns:
X, y = df.drop("Target",axis=1),df["Target"]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3,stratify=y, random_state=42)
 model.fit(X_train,y_train)
 predicted = model.predict(X_test)
 print('Classifcation report:
', classification_report(y_test, predicted))
print('Confusion matrix:
', confusion_matrix(y_test, predicted))
return model
else:
return model.predict(df )<train_model>
seed(42)
 tf.random.set_seed(42)
def treino_sampling(df,model,train_submit=False):
if train_submit:
X, y = df.drop("Target",axis=1),df["Target"]
model.fit(X,y)
 return model
if "Target" in df.columns:
resampling = RandomOverSampler()
pipeline = Pipeline([('Resampling', resampling),('XGBClassifier', model)])
 X, y = df.drop("Target",axis=1),df["Target"]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3,stratify=y, random_state=42)
 pipeline.fit(X_train, y_train)
 predicted = model.predict(X_test)
 print('Classifcation report:
', classification_report(y_test, predicted))
print('Confusion matrix:
', confusion_matrix(y_test, predicted))
return model
else:
return model.predict(df )<find_best_model_class>
train_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 sub = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )
def sampling_predict(df,func):
resampling = func()
model = XGBClassifier()
pipeline = Pipeline([('Resampling', resampling),('XGBClassifier', model)])
 X=df.drop("Target",axis=1)
 y=df.Target
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3,random_state=0)
 pipeline.fit(X_train, y_train)
 predicted = pipeline.predict(X_test)
 print('Classifcation report:
', classification_report(y_test, predicted))
print('Confusion matrix:
', confusion_matrix(y_test, predicted))<choose_model_class>
train_df.isnull().sum()
xgb=XGBClassifier()
modelo_final_sem_sampler = treino(df=dropa_trata(df),model=xgb )<define_variables>
test_df.isnull().sum()
under = ["TomekLinks"]
over =("RandomOverSampler","ADASYN","BorderlineSMOTE","SMOTE","SVMSMOTE")
 df_d1 = df.fillna(-1 )<predict_on_test>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
for model in under:
print("
"+model+"
")
 sampling_predict(dropa_trata(df_d1),eval(model))<predict_on_test>
import tokenization
for model in over:
print("
"+model+"
")
 sampling_predict(dropa_trata(df_d1),eval(model))<define_search_model>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
modelo_final = treino_sampling(dropa_trata(df),model=xgb )<save_to_csv>
Dropout_num = 0
learning_rate = 7e-6
valid = 0.2
epochs_num = 3
batch_size_num = 16
target_corrected = False
target_big_corrected = False
df_test = pd.read_csv("/kaggle/input/costa-rican-household-poverty-prediction/test.csv")
 preds = treino(dropa_trata(df_test),modelo_final_sem_sampler)
 z = pd.Series(preds,name="Target")
 df_entrega = pd.concat([df_test.Id,z], axis=1)
 df_entrega.to_csv("/kaggle/working/submission.csv",index=False )<load_from_csv>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
if Dropout_num == 0:
out = Dense(1, activation='sigmoid' )(clf_output)
 else:
x = Dropout(Dropout_num )(clf_output)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model
pd.options.display.max_columns = None
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<data_type_conversions>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
mapping = {"yes": 1, "no": 0}
for df in [train, test]:
df['dependency'] = df['dependency'].replace(mapping ).astype(np.float64)
 df['edjefa'] = df['edjefa'].replace(mapping ).astype(np.float64)
 df['edjefe'] = df['edjefe'].replace(mapping ).astype(np.float64)
 train[['dependency', 'edjefa', 'edjefe']].describe()<concatenate>
if target_corrected:
ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train_df.loc[train_df['id'].isin(ids_with_target_error),'target'] = 0
train_df[train_df['id'].isin(ids_with_target_error)]
test['Target'] = np.nan
data = train.append(test, ignore_index = True )<count_unique_values>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )<filter>
train_input = bert_encode(train_df.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test_df.text.values, tokenizer, max_len=160)
 train_labels = train_df.target.values
train[train['idhogar'] == not_equal.index[0]][['idhogar', 'parentesco1', 'Target']]<count_unique_values>
train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=5,
batch_size=8 
)
 model.save('model.h5' )
households_leader = train.groupby('idhogar')['parentesco1'].sum()
households_no_head = train.loc[train['idhogar'].isin(households_leader[households_leader == 0].index), :]
print('There are {} households without a head.'.format(households_no_head['idhogar'].nunique()))<groupby>
test_pred = model.predict(test_input )
<feature_engineering><EOS> 
sub['target'] = test_pred.round().astype(int)
 sub.to_csv('submission.csv', index=False)
 sub.head()