Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
!pip -q install pyspellchecker
input_path = '/kaggle/input/porto-seguro-safe-driver-prediction/'
df = pd.read_csv(input_path+'train.csv')
 df_test = pd.read_csv(input_path+'test.csv' )<define_variables>
import os
import numpy as np
import pandas as pd
import random
import re
import string
from spellchecker import SpellChecker
from tqdm.notebook import tqdm
import warnings
from sklearn.metrics import roc_auc_score, f1_score, matthews_corrcoef
from sklearn.model_selection import StratifiedKFold
import tensorflow as tf
import tensorflow_addons as tfa
import tensorflow.keras.backend as K
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.layers import Dropout, Dense, Input
from tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler
from tensorflow.keras.models import Model
from tokenizers import BertWordPieceTokenizer
from transformers import TFBertModel, AdamWeightDecay, BertTokenizerFast
id_test = df_test['id']<define_variables>
warnings.filterwarnings("ignore" )
target = df['target']<count_values>
SEED = 42
MAX_LEN = 192
EPOCHS = 6
NUM_SPLITS = 5
LR = 3e-5
target.value_counts()<drop_column>
os.environ['PYTHONHASHSEED']=str(SEED)
 random.seed(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED )
df.drop(columns=['target'],inplace=True )<load_pretrained>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv" )
df_metadata = pickle.load(open('/kaggle/input/pssdpickledfmetedatapickle/df_metedata_pickle','rb'))<data_type_conversions>
test["target"] = -1
def preprocessing(df):
df.replace(to_replace=-1,value=np.nan,inplace=True)
 for col in df.columns:
if df_metadata.loc[col,'Dropped']:
df.drop(columns=[col],inplace=True)
 continue
df[col].fillna(df_metadata.loc[col,'Missing'],inplace=True)
 if(df_metadata.loc[col,'DTypes'] == 'Categorical')or(df_metadata.loc[col,'DTypes'] == 'Ordinal'):
df[col] = df[col].astype('category' )<count_missing_values>
df = pd.concat([train, test] )
df.isnull().sum()<count_missing_values>
PUNCT_TO_REMOVE = string.punctuation
def remove_punctuation(text):
return text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))
df["text"] = df["text"].apply(lambda text: remove_punctuation(text))
df_test.isnull().sum()<define_variables>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 df["text"] = df["text"].apply(lambda text: remove_emoji(text))
def outlier_processing(df,df_test):
for col in df.columns:
if df[col].dtype.name != 'category':
first_quartile, third_quartile = np.percentile(df[col],[25,75])
 first_percetnile, ninetynine_percentile = np.percentile(df[col],[1,99])
 IQR = third_quartile - first_quartile
lower_bound = first_quartile -(1.5*IQR)
 upper_bound = third_quartile +(1.5*IQR)
 df[col].loc[df[col]>upper_bound] = ninetynine_percentile
df_test[col].loc[df_test[col]>upper_bound] = ninetynine_percentile
df[col].loc[df[col]<lower_bound] = first_percetnile
df_test[col].loc[df_test[col]<lower_bound] = first_percetnile 
<count_unique_values>
def remove_urls(text):
url_pattern = re.compile(r'https?://\S+|www\.\S+')
 try:
return url_pattern.sub(r'', text)
 except:
print(text)
 df["text"] = df["text"].apply(lambda text: remove_urls(text))
ordinal_columns = [col for col in df.columns if df_metadata.loc[col,'DTypes'] == 'Ordinal' and df[col].nunique() > 2]<count_unique_values>
def remove_html(text):
html_pattern = re.compile('<.*?>')
 return html_pattern.sub(r'', text)
 df["text"] = df["text"].apply(lambda text: remove_html(text))
categorical_columns_great_2 = [col for col in df.columns if df_metadata.loc[col,'DTypes'] == 'Categorical' and df[col].nunique() > 2]<categorify>
with open(".. /input/slangtext/slang.txt", "r")as file:
chat_words_str = file.read()
chat_words_map_dict = {}
chat_words_list = []
for line in chat_words_str.split("
"):
if line != "" and "=" in line:
cw = line.split("=")[0]
cw_expanded = line.split("=")[1]
chat_words_list.append(cw)
 chat_words_map_dict[cw] = cw_expanded
chat_words_list = set(chat_words_list )
for col in ordinal_columns:
label_encode = LabelEncoder()
df[col+'label'] = label_encode.fit_transform(df[col])
 df_test[col+'label'] = label_encode.transform(df_test[col])
 df.drop(columns=[col],inplace=True)
 df_test.drop(columns=[col],inplace=True )<categorify>
def chat_words_conversion(text):
new_text = []
for w in text.split() :
if w.upper() in chat_words_list:
new_text.append(chat_words_map_dict[w.upper() ])
 else:
new_text.append(w)
 return " ".join(new_text)
 df["text"] = df["text"].apply(lambda text: chat_words_conversion(text))
df = pd.get_dummies(df,prefix=col,columns=categorical_columns_great_2,drop_first=True)
 df_test = pd.get_dummies(df_test,columns=categorical_columns_great_2,prefix=col,drop_first=True )<normalization>
spell = SpellChecker()
def correct_spellings(text):
corrected_text = []
misspelled_words = spell.unknown(text.split())
 for word in text.split() :
if word in misspelled_words:
corrected_text.append(spell.correction(word))
else:
corrected_text.append(word)
 return " ".join(corrected_text)
 df["text"] = df["text"].apply(lambda text: chat_words_conversion(text))
scaler = MinMaxScaler()<normalization>
train = df[df["target"]!=-1]
test = df[df["target"]==-1]
df_train_scale = scaler.fit_transform(df )<normalization>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
df_test_scale = scaler.transform(df_test )<import_modules>
tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
 save_path = 'distilbert_base_uncased/'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
 fast_tokenizer = BertWordPieceTokenizer('distilbert_base_uncased/vocab.txt', lowercase=True )
from sklearn.model_selection import StratifiedKFold<choose_model_class>
X_test_tokenized = fast_encode(test["text"].astype(str), fast_tokenizer, maxlen=MAX_LEN )
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=17 )<compute_train_metric>
def focal_loss(gamma=2., alpha=.25):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed
def gini(actual,pred,cmpcol = 0,sortcol = 1):
assert(len(actual)== len(pred))
All = np.asarray(np.c_[ actual, pred, np.arange(len(actual)) ], dtype=np.float)
 All = All[ np.lexsort(( All[:,2], -1*All[:,1])) ]
totAllosses = All[:,0].sum()
giniSum = All[:,0].cumsum().sum() / totAllosses
giniSum -=(len(actual)+ 1)/ 2.
 return giniSum / len(actual)
 def gini_normalized(a, p):
return gini(a, p)/ gini(a, a)
 def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = gini_normalized(labels, preds)
 return 'gini', gini_score<import_modules>
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )
import xgboost as xgb<init_hyperparams>
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 BATCH_SIZE = 16 * strategy.num_replicas_in_sync
params = {
'min_child_weight': 10.0,
'objective': 'binary:logistic',
'max_depth': 7,
'max_delta_step': 1.8,
'colsample_bytree': 0.4,
'subsample': 0.8,
'eta': 0.025,
'gamma': 0.65,
'num_boost_round' : 1000
}<train_model>
def build_model(transformer, loss='binary_crossentropy', max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = Dropout(0.35 )(cls_token)
 out = Dense(2, activation='softmax' )(x)
 optimizer = tfa.optimizers.RectifiedAdam(lr=LR)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(optimizer=optimizer, loss=loss, metrics=[tf.keras.metrics.AUC() ])
 return model
model_ensemble = []
for i,(train_index, valid_index)in enumerate(skf.split(df_train_scale,target)) :
print('[FOLD %d/%d]'%(i+1,5))
X_train,X_valid = df_train_scale[train_index],df_train_scale[valid_index]
y_train,y_valid = target.loc[train_index],target.loc[valid_index]
df_train_xgb = xgb.DMatrix(X_train,y_train)
 df_valid_xgb = xgb.DMatrix(X_valid,y_valid)
 valid_list = [(df_train_xgb, 'train'),(df_valid_xgb, 'valid')]
xgb_model = xgb.train(params, df_train_xgb, 3000, valid_list, feval=gini_xgb, maximize=True, early_stopping_rounds=70,verbose_eval=100)
 model_ensemble.append(xgb_model )<predict_on_test>
def build_lrfn(lr_start=0.000001, lr_max=0.000004,
lr_min=0.0000001, lr_rampup_epochs=7,
lr_sustain_epochs=0, lr_exp_decay=.87):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
predict_proba = 0
df_test_xgb = xgb.DMatrix(df_test_scale)
 for i, model in enumerate(model_ensemble):
print('[FOLD %d/%d Prediciton:]'%(i+1,5))
predictions = xgb_model.predict(df_test_xgb)
 predict_proba += predictions
predict_proba = predict_proba/5<save_to_csv>
eas = EarlyStopping(monitor='val_auc', min_delta=0.001, patience=3,
verbose=1, mode='max', baseline=None, restore_best_weights=True)
 lrfn = build_lrfn()
lrs = LearningRateScheduler(lrfn, verbose=1 )
submit = pd.DataFrame({'id':id_test,'target':predict_proba})
 submit.to_csv('xgb_porto.csv',index=False)
 submit.head()<load_from_csv>
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(X_test_tokenized)
.batch(BATCH_SIZE)
 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
num_steps = len(train)// BATCH_SIZE
final_preds = np.zeros(( len(test)))
 total_preds = np.zeros(( len(train)) )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<feature_engineering>
kfold = StratifiedKFold(n_splits=NUM_SPLITS, shuffle=True, random_state=SEED )
for col in train.columns:
train.loc[train[col] == -1, col] = np.nan
for col in test.columns:
test.loc[test[col] == -1, col] = np.nan<init_hyperparams>
for fold,(train_idx, valid_idx)in enumerate(kfold.split(X=train['text'], y=train['target'])) :
print("*"*60)
 print("*"+" "*26+f"FOLD {fold+1}"+" "*26+"*")
 print("*"*60, end="
")
 X_train = train.iloc[train_idx].reset_index(drop=True)
 X_valid = train.iloc[valid_idx].reset_index(drop=True)
 y_train = X_train["target"]
y_valid = X_valid["target"]
X_train_tokenized = fast_encode(X_train.text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 X_valid_tokenized = fast_encode(X_valid.text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 train_dataset = tf.data.Dataset.from_tensor_slices(( X_train_tokenized, to_categorical(y_train)))
 train_dataset = train_dataset.repeat()
train_dataset = train_dataset.shuffle(2048)
 train_dataset = train_dataset.batch(BATCH_SIZE)
 train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)
 valid_dataset = tf.data.Dataset
valid_dataset = valid_dataset.from_tensor_slices(( X_valid_tokenized, to_categorical(y_valid)))
 valid_dataset = valid_dataset.batch(BATCH_SIZE)
 K.clear_session()
with strategy.scope() :
transformer_layer = TFBertModel.from_pretrained('bert-base-uncased')
 model = build_model(transformer_layer, loss=focal_loss(gamma=1.5), max_len=MAX_LEN)
 history = model.fit(train_dataset,
steps_per_epoch=num_steps,
validation_data=valid_dataset,
callbacks=[eas, lrs],
epochs=EPOCHS)
 valid_preds = model.predict(valid_dataset)[:, 1]
total_preds[valid_idx] = valid_preds
test_preds = model.predict(test_dataset)[:, 1]
final_preds += test_preds
seed = 42
param = {'num_leaves': 31,
'boosting_type': 'gbdt',
'learning_rate': 0.01,
'max_bin': 255,
'min_data_in_leaf': 1500,
'feature_fraction': 0.7,
'bagging_freq': 1,
'bagging_fraction': 0.7,
'lambda_l1': 1,
'lambda_l2': 1,
'save_binary': True,
'seed': seed,
'feature_fraction_seed': seed,
'bagging_seed': seed,
'drop_seed': seed,
'data_random_seed': seed,
'objective': 'binary',
'verbose': 1,
'metric': 'binary_logloss',
'is_unbalance': False,
'boost_from_average': True
}<categorify>
actuals = train["target"].values
total_preds[total_preds >= 0.5] = 1
total_preds[total_preds < 0.5] = 0
cat_names = ['ps_ind_02_cat', 'ps_ind_04_cat', 'ps_ind_05_cat', 'ps_car_01_cat',
'ps_car_02_cat', 'ps_car_03_cat', 'ps_car_04_cat', 'ps_car_05_cat',
'ps_car_06_cat', 'ps_car_07_cat', 'ps_car_08_cat', 'ps_car_09_cat',
'ps_car_10_cat', 'ps_car_11_cat']
train = pd.get_dummies(train, columns = cat_names)
 test = pd.get_dummies(test, columns = cat_names )<define_variables>
print(f"AUC: {auc(actuals, total_preds)}")
 print(f"F1 Score: {f1_score(actuals, total_preds)}")
 print(f"MCC: {matthews_corrcoef(actuals, total_preds)}" )
calc_names = ['ps_calc_01', 'ps_calc_02', 'ps_calc_03', 'ps_calc_04',
'ps_calc_05', 'ps_calc_06', 'ps_calc_07', 'ps_calc_08',
'ps_calc_09', 'ps_calc_10', 'ps_calc_11', 'ps_calc_12',
'ps_calc_13', 'ps_calc_14', 'ps_calc_15_bin','ps_calc_16_bin',
'ps_calc_17_bin','ps_calc_18_bin','ps_calc_19_bin',
'ps_calc_20_bin']
train.drop(columns = calc_names, inplace=True)
<drop_column>
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )
predictors = list(train.columns)
 predictors.remove('id')
 predictors.remove('target' )<define_variables>
submission["target"] = final_preds/NUM_SPLITS
submission["target"] = submission["target"].apply(lambda x: 1 if x>=0.5 else 0 )
nfold = 5
target = 'target'
skf = KFold(n_splits=nfold, shuffle=True, random_state=2019)
 oof = np.zeros(len(train))
predictions = np.zeros(len(test))<split>
submission.to_csv("submission.csv", index=False )
<compute_test_metric><EOS> 
submission.to_csv("submission.csv", index=False )