Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
!pip install bert-for-tf2
df = pd.concat([traindf, testdf], axis=0, sort=False)
 df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['IsWomanOrBoy'] =(( df.Title == 'Master')|(df.Sex == 'female'))
df['LastName'] = df.Name.str.split(',' ).str[0]
family = df.groupby(df.LastName ).Survived
df['WomanOrBoyCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).count())
 df['WomanOrBoyCount'] = df.mask(df.IsWomanOrBoy, df.WomanOrBoyCount - 1, axis=0)
 df['FamilySurvivedCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).sum())
 df['FamilySurvivedCount'] = df.mask(df.IsWomanOrBoy, df.FamilySurvivedCount - \
df.Survived.fillna(0), axis=0)
 df['WomanOrBoySurvived'] = df.FamilySurvivedCount / df.WomanOrBoyCount.replace(0, np.nan)
 df.WomanOrBoyCount = df.WomanOrBoyCount.replace(np.nan, 0)
 df['Alone'] =(df.WomanOrBoyCount == 0)
 df['Title'] = df['Title'].replace('Ms','Miss')
 df['Title'] = df['Title'].replace('Mlle','Miss')
 df['Title'] = df['Title'].replace('Mme','Mrs')
 df['Embarked'] = df['Embarked'].fillna('S')
 df['Deck'] = df['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'M')
 med_fare = df.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
df['Fare'] = df['Fare'].fillna(med_fare)
 df['Age'] = df.groupby(['Sex', 'Pclass', 'Title'])['Age'].apply(lambda x: x.fillna(x.median()))
df['Family_Size'] = df['SibSp'] + df['Parch'] + 1
cols_to_drop = ['Name','Ticket','Cabin', 'IsWomanOrBoy', 'WomanOrBoyCount', 'FamilySurvivedCount']
df = df.drop(cols_to_drop, axis=1)
 df.WomanOrBoySurvived = df.WomanOrBoySurvived.fillna(0)
 df.Alone = df.Alone.fillna(0)
 target = df.Survived.loc[traindf.index]
df = df.drop(['Survived'], axis=1)
 train, test = df.loc[traindf.index], df.loc[testdf.index]<data_type_conversions>
import numpy as np
import pandas as pd
import re
import tensorflow as tf
from tensorflow_core.python.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow_core.python.keras.models import Model
from tensorflow_core.python.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
from bert.tokenization import bert_tokenization
import math
import seaborn as sns
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
from sklearn.model_selection import StratifiedKFold
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
categorical_columns = []
features = train.columns.values.tolist()
for col in features:
if train[col].dtype in numerics: continue
categorical_columns.append(col)
 for col in categorical_columns:
if col in train.columns:
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))<split>
def clean_text(text):
new_text = []
for each in text.split() :
if each.isalpha() :
new_text.append(each)
 cleaned_text = ' '.join(new_text)
 cleaned_text = re.sub(r'https?:\/\/t.co\/[A-Za-z0-9]+','',cleaned_text)
 return cleaned_text
Xtrain, Xval, Ztrain, Zval = train_test_split(train, target, test_size=0.2, random_state=0)
 train_set = lgbm.Dataset(Xtrain, Ztrain, silent=False)
 valid_set = lgbm.Dataset(Xval, Zval, silent=False )<init_hyperparams>
def bert_encode(texts, tokenizer, max_len =512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ['[CLS]'] + text +['[SEP]']
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0]*pad_len
pad_masks = [1]*len(input_sequence)+ [0]*pad_len
segment_ids = [0]*max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_bert(bert_layer, max_len =512):
adam = Adam(lr=6e-6)
 input_word_ids = Input(shape =(max_len,),dtype ='int32')
 input_mask = Input(shape =(max_len,),dtype ='int32')
 segment_ids = Input(shape =(max_len,),dtype ='int32')
 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:,0,:]
out = Dense(1, activation ='sigmoid' )(clf_output)
 model = Model(inputs =[input_word_ids, input_mask, segment_ids], outputs =out)
 model.compile(optimizer=adam ,loss = 'binary_crossentropy', metrics =['accuracy'])
 return model
bert_layer = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2', trainable=True)
 train_data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_data = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = bert_tokenization.FullTokenizer(vocab_file, do_lower_case)
 train_data['text'] = train_data['text'].apply(lambda x: clean_text(x))
test_data['text'] = test_data['text'].apply(lambda x: clean_text(x))
all_models = []
all_loss = []
skf = StratifiedKFold(n_splits = 5, random_state =42, shuffle=True)
 for fold,(train_idx,val_idx)in enumerate(skf.split(train_data['text'],train_data['target'])) :
print('Fold:'+str(fold))
train_input = bert_encode(train_data.loc[train_idx,'text'], tokenizer, max_len=100)
 train_labels = train_data.loc[train_idx,'target']
valid_input = bert_encode(train_data.loc[val_idx,'text'], tokenizer, max_len=100)
 valid_labels = train_data.loc[val_idx,'target']
model = build_bert(bert_layer, max_len=100)
 model.fit(train_input, train_labels,epochs =3, batch_size = 16)
 valid_loss, valid_acc = model.evaluate(valid_input,valid_labels, batch_size =16)
 all_models.append(model)
 all_loss.append(valid_loss)
params = {
'boosting_type':'gbdt',
'objective': 'binary',
'num_leaves': 31,
'learning_rate': 0.05,
'max_depth': -1,
'subsample': 0.8,
'bagging_fraction' : 1,
'max_bin' : 50 ,
'bagging_freq': 20,
'colsample_bytree': 0.6,
'metric': 'binary',
'min_split_gain': 0.5,
'min_child_weight': 1,
'min_child_samples': 2,
'scale_pos_weight':1,
'zero_as_missing': True,
'seed':0,
}
modelL = lgbm.train(params, train_set = train_set, num_boost_round=2000,
early_stopping_rounds=10, verbose_eval=10, valid_sets=valid_set )<create_dataframe>
test_text = list(test_data['text'])
 test_input = bert_encode(test_text, tokenizer, max_len=100)
 min_loss_index = all_loss.index(min(all_loss))
results = all_models[min_loss_index].predict(test_input)
 submission_data = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 submission_data['target'] = results.round().astype('int')
 submission_data.to_csv('/kaggle/working/submission.csv',index=False, header=True)