Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<count_values>
import numpy as np
import pandas as pd
import string
import re
from matplotlib import pyplot as plt
import nltk
from nltk.tokenize import TweetTokenizer
from nltk.stem import PorterStemmer
from nltk.corpus import stopwords
data1.Cover_Type.value_counts()<prepare_x_and_y>
nltk.download('stopwords' )
predictor2 = data1.drop(["Id","Cover_Type"],axis = 1)
 target2 = data1.Cover_Type<choose_model_class>
train_filepath = '/kaggle/input/nlp-getting-started/train.csv'
df_train = pd.read_csv(train_filepath, index_col='id')
 df_train.head()
ex = ExtraTreesClassifier(random_state = 123,n_estimators = 500 )<predict_on_test>
df_train.isnull().sum() / len(df_train)* 100
predict = cross_val_predict(ex1,predictor2,target2,cv = 3 )<train_model>
tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True, reduce_len=True)
 stopwords_english = stopwords.words('english')
 stemmer = PorterStemmer()
ex1.fit(predictor2,target2 )<drop_column>
def process_tweet(tweet, tokenizer=tokenizer, stopwords_english=stopwords_english, stemmer=stemmer):
tweet = re.sub(r'^RT[\s]+', '', tweet)
 tweet = re.sub(r'https?:\/\/.*[\r
]*', '', tweet)
 tweet = re.sub(r'
tweet_tokens = tokenizer.tokenize(tweet)
 tweets_clean = []
for word in tweet_tokens:
if(word not in stopwords_english and word not in string.punctuation):
tweets_clean.append(word)
 tweets_stem = []
for word in tweets_clean:
stem_word = stemmer.stem(word)
 tweets_stem.append(stem_word)
 return tweets_stem
test.drop("real",axis = 1,inplace = True )<filter>
def count_tweets(result, tweets, ys):

 for y, tweet in zip(ys, tweets):
for word in process_tweet(tweet):
pair =(word, y)
 result[pair] = result.get(pair, 0)+ 1
return result
dataset = test[test["Cover_Type"] == 1]<predict_on_test>
def test_naive_bayes(test_x, test_y, logprior, loglikelihood):

 accuracy = 0
y_hats = []
for tweet in test_x:
if naive_bayes_predict(tweet, logprior, loglikelihood)> 0:
y_hat_i = 1
else:
y_hat_i = 0
y_hats.append(y_hat_i)
 error = np.mean(np.abs(y_hats - test_y))
accuracy = 1 - error
return accuracy
prediction2 = ex1.predict(dataset.drop(["Id","Cover_Type"],axis = 1))<drop_column>
freqs = count_tweets({}, df_train['text'], df_train['target'])
 logprior, loglikelihood = train_naive_bayes(freqs, df_train['text'], df_train['target'])
 print(logprior)
 print(len(loglikelihood))
dataset.drop("Cover_Type",axis = 1,inplace = True )<feature_engineering>
print("Naive Bayes accuracy:", test_naive_bayes(df_train['text'], df_train['target'], logprior, loglikelihood)* 100, "%" )
dataset["Cover_Type"] = prediction2<drop_column>
print('Truth Predicted Tweet')
 for x, y in zip(df_train['text'], df_train['target']):
y_hat = naive_bayes_predict(x, logprior, loglikelihood)
 if y !=(np.sign(y_hat)> 0):
print('%d\t%0.2f\t%s' %(y, np.sign(y_hat)> 0, ' '.join(
 process_tweet(x)).encode('ascii', 'ignore')) )
data_test.drop("Cover_Type",axis = 1,inplace = True )<feature_engineering>
df_test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
data_test["Cover_Type"] = data_test["real"]<drop_column>
df_test['target'] = df_test['text'].apply(lambda tweet : int(np.sign(naive_bayes_predict(tweet, logprior, loglikelihood)) > 0))
data_test.drop("real",axis = 1,inplace = True )<concatenate>
submission = df_test[['id', 'target']]
submission.to_csv('nb_submission.csv', index=False )
test = pd.concat([dataset,data_test] ).sort_index()<define_variables>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
my_submission3 = test[["Id","Cover_Type"]]<save_to_csv>
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
my_submission3.to_csv("my_submisssion3",index = False )<set_options>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
warnings.simplefilter(action='ignore', category=FutureWarning )<categorify>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
def one_hot_encoder(df, nan_as_category = True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns 
<load_from_csv>
if 'target' in df_test.columns.tolist() :
del df_test['target']
def application_train_test(num_rows = None, nan_as_category = False):
df = pd.read_csv('.. /input/application_train.csv', nrows= num_rows)
 test_df = pd.read_csv('.. /input/application_test.csv', nrows= num_rows)
 print("Train samples: {}, test samples: {}".format(len(df), len(test_df)))
 df = df.append(test_df ).reset_index()
df = df[df['CODE_GENDER'] != 'XNA']
docs = [_f for _f in df.columns if 'FLAG_DOC' in _f]
live = [_f for _f in df.columns if('FLAG_' in _f)&('FLAG_DOC' not in _f)&('_FLAG_' not in _f)]
df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)
 inc_by_org = df[['AMT_INCOME_TOTAL', 'ORGANIZATION_TYPE']].groupby('ORGANIZATION_TYPE' ).median() ['AMT_INCOME_TOTAL']
df['NEW_CREDIT_TO_ANNUITY_RATIO'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']
df['NEW_CREDIT_TO_GOODS_RATIO'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']
df['NEW_DOC_IND_KURT'] = df[docs].kurtosis(axis=1)
 df['NEW_LIVE_IND_SUM'] = df[live].sum(axis=1)
 df['NEW_INC_PER_CHLD'] = df['AMT_INCOME_TOTAL'] /(1 + df['CNT_CHILDREN'])
 df['NEW_INC_BY_ORG'] = df['ORGANIZATION_TYPE'].map(inc_by_org)
 df['NEW_EMPLOY_TO_BIRTH_RATIO'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['NEW_ANNUITY_TO_INCOME_RATIO'] = df['AMT_ANNUITY'] /(1 + df['AMT_INCOME_TOTAL'])
 df['NEW_SOURCES_PROD'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2'] * df['EXT_SOURCE_3']
df['NEW_EXT_SOURCES_MEAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 df['NEW_SCORES_STD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)
 df['NEW_SCORES_STD'] = df['NEW_SCORES_STD'].fillna(df['NEW_SCORES_STD'].mean())
 df['NEW_CAR_TO_BIRTH_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']
df['NEW_CAR_TO_EMPLOY_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']
df['NEW_PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']
df['NEW_PHONE_TO_BIRTH_RATIO_EMPLOYER'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_EMPLOYED']
df['NEW_CREDIT_TO_INCOME_RATIO'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']
for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:
df[bin_feature], uniques = pd.factorize(df[bin_feature])
 df, cat_cols = one_hot_encoder(df, nan_as_category)
 dropcolum=['FLAG_DOCUMENT_2','FLAG_DOCUMENT_4',
'FLAG_DOCUMENT_5','FLAG_DOCUMENT_6','FLAG_DOCUMENT_7',
'FLAG_DOCUMENT_8','FLAG_DOCUMENT_9','FLAG_DOCUMENT_10',
'FLAG_DOCUMENT_11','FLAG_DOCUMENT_12','FLAG_DOCUMENT_13',
'FLAG_DOCUMENT_14','FLAG_DOCUMENT_15','FLAG_DOCUMENT_16',
'FLAG_DOCUMENT_17','FLAG_DOCUMENT_18','FLAG_DOCUMENT_19',
'FLAG_DOCUMENT_20','FLAG_DOCUMENT_21']
df= df.drop(dropcolum,axis=1)
 del test_df
gc.collect()
return df<load_from_csv>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
def bureau_and_balance(num_rows = None, nan_as_category = True):
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows)
 bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)
 bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(['SK_ID_BUREAU'], axis=1, inplace= True)
 del bb, bb_agg
gc.collect()
num_aggregations = {
'DAYS_CREDIT': [ 'mean', 'var'],
'DAYS_CREDIT_ENDDATE': [ 'mean'],
'DAYS_CREDIT_UPDATE': ['mean'],
'CREDIT_DAY_OVERDUE': ['mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'AMT_CREDIT_SUM': [ 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': [ 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'AMT_ANNUITY': ['max', 'mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ])
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACTIVE_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLOSED_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 del closed, closed_agg, bureau
gc.collect()
return bureau_agg<load_from_csv>
train_input = bert_encode(df_train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(df_test.text.values, tokenizer, max_len=160)
 train_labels = df_train.target.values
def previous_applications(num_rows = None, nan_as_category = True):
prev = pd.read_csv('.. /input/previous_application.csv', nrows = num_rows)
 prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)
 prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']
num_aggregations = {
'AMT_ANNUITY': [ 'max', 'mean'],
'AMT_APPLICATION': [ 'max','mean'],
'AMT_CREDIT': [ 'max', 'mean'],
'APP_CREDIT_PERC': [ 'max', 'mean'],
'AMT_DOWN_PAYMENT': [ 'max', 'mean'],
'AMT_GOODS_PRICE': [ 'max', 'mean'],
'HOUR_APPR_PROCESS_START': [ 'max', 'mean'],
'RATE_DOWN_PAYMENT': [ 'max', 'mean'],
'DAYS_DECISION': [ 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
}
cat_aggregations = {}
for cat in cat_cols:
cat_aggregations[cat] = ['mean']
prev_agg = prev.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 prev_agg.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist() ])
 approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
approved_agg = approved.groupby('SK_ID_CURR' ).agg(num_aggregations)
 approved_agg.columns = pd.Index(['APPROVED_' + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist() ])
 prev_agg = prev_agg.join(approved_agg, how='left', on='SK_ID_CURR')
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
refused_agg = refused.groupby('SK_ID_CURR' ).agg(num_aggregations)
 refused_agg.columns = pd.Index(['REFUSED_' + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist() ])
 prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')
 del refused, refused_agg, approved, approved_agg, prev
gc.collect()
return prev_agg<load_from_csv>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
def pos_cash(num_rows = None, nan_as_category = True):
pos = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows = num_rows)
 pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)
 aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size'],
'SK_DPD': ['max', 'mean'],
'SK_DPD_DEF': ['max', 'mean']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
pos_agg = pos.groupby('SK_ID_CURR' ).agg(aggregations)
 pos_agg.columns = pd.Index(['POS_' + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR' ).size()
del pos
gc.collect()
return pos_agg<load_from_csv>
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=2,
callbacks=[checkpoint],
batch_size=16
)
def installments_payments(num_rows = None, nan_as_category = True):
ins = pd.read_csv('.. /input/installments_payments.csv', nrows = num_rows)
 ins, cat_cols = one_hot_encoder(ins, nan_as_category= True)
 ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0)
 aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DPD': ['max', 'mean', 'sum','min','std' ],
'DBD': ['max', 'mean', 'sum','min','std'],
'PAYMENT_PERC': [ 'max','mean', 'var','min','std'],
'PAYMENT_DIFF': [ 'max','mean', 'var','min','std'],
'AMT_INSTALMENT': ['max', 'mean', 'sum','min','std'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum','std'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum','std']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
ins_agg = ins.groupby('SK_ID_CURR' ).agg(aggregations)
 ins_agg.columns = pd.Index(['INSTAL_' + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist() ])
 ins_agg['INSTAL_COUNT'] = ins.groupby('SK_ID_CURR' ).size()
del ins
gc.collect()
return ins_agg<load_from_csv>
model.load_weights('model.h5')
 test_pred = model.predict(test_input )
def credit_card_balance(num_rows = None, nan_as_category = True):
cc = pd.read_csv('.. /input/credit_card_balance.csv', nrows = num_rows)
 cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)
 cc.drop(['SK_ID_PREV'], axis= 1, inplace = True)
 cc_agg = cc.groupby('SK_ID_CURR' ).agg([ 'max', 'mean', 'sum', 'var'])
 cc_agg.columns = pd.Index(['CC_' + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist() ])
 cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR' ).size()
del cc
gc.collect()
return cc_agg 
<categorify>
df_test['target'] = test_pred.round().astype(int)
 submission = df_test[['id', 'target']]
submission.to_csv('bert_submission.csv', index=False )