Kaggle competition: Natural Language Processing with Disaster Tweets
null_entries_train[null_entries_train.project_is_approved==0].project_resource_summary.iloc[1]<feature_engineering>
convert_abbrev = False
model_type = 'default'
Dropout_num = 0.1
learning_rate = 1e-5
max_len = 160
layers = []
activation = 'relu'
model_name = 'model_bert_cv_default_{}_shape_{}'.format(Dropout_num,'_'.join([str(i)for i in layers]))
def get_text_stats(text_col):

 title="Text Stats of " + text_col
target_col='project_is_approved'
text_stats=pd.DataFrame()
text_stats[target_col]=train[target_col]
text_stats['word_count']=train[text_col].apply(lambda x: len(str(x ).split()))
text_stats['count_unique_word']=train[text_col].apply(lambda x: len(set(str(x ).split())))
 text_stats['word_unique_percent']=(text_stats['count_unique_word']*100)/text_stats['word_count']
temp_df = pd.melt(text_stats, value_vars=['word_count', 'count_unique_word'], id_vars=target_col)
 print("------ Sample from an Approved project ------
")
 print(approvals[text_col].iloc[0])
 print("
------ Sample from a Rejected project ------
")
 print(rejects[text_col].iloc[0])
 plt.figure(figsize=(16,5))
plt.subplot(121)
 plt.suptitle(title,fontsize=16)
 plt.title("Word Count")
 sns.violinplot(x='variable', y='value', hue=str(target_col), data=temp_df,inner='quartile')
 plt.ylabel('
plt.subplot(122)
 plt.title("Percentage of Unique words - effect on Approval")
 ax=sns.kdeplot(text_stats[text_stats.project_is_approved == 0].word_unique_percent, label="Not Approved",shade=True,color='r')
 ax=sns.kdeplot(text_stats[text_stats.project_is_approved == 1].word_unique_percent, label="Approved")
 plt.legend()
plt.xlabel('Percent unique words', fontsize=12)
 plt.ylabel('
plt.show()<categorify>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
gender_mapping = {"Ms.": "Female", "Mrs.":"Female", "Mr.":"Male", "Teacher":"Unknown", "Dr.":"Unknown", np.nan:"Unknown" }
train["gender"] = train.teacher_prefix.map(gender_mapping)
 approvals["gender"] = approvals.teacher_prefix.map(gender_mapping)
 rejects["gender"] = rejects.teacher_prefix.map(gender_mapping)
 test['gender'] = test.teacher_prefix.map(gender_mapping)
 make_custom_plot('gender','Analyzing Gender' )<groupby>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )
subject_cats=','.join(train['project_subject_categories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_categories'])
 cats.project_subject_categories=cats.project_subject_categories.str.strip()
x=cats.project_subject_categories.value_counts()
print("There are",len(x),"different subject categories after cleaning")
 subject_cats=','.join(approvals['project_subject_categories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_categories'])
 cats.project_subject_categories=cats.project_subject_categories.str.strip()
x1=cats.project_subject_categories.value_counts()
subject_cats=','.join(rejects['project_subject_categories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_categories'])
 cats.project_subject_categories=cats.project_subject_categories.str.strip()
x2=cats.project_subject_categories.value_counts() 
<string_transform>
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input,Dropout
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
from sklearn.metrics import f1_score,precision_score,recall_score,roc_auc_score,log_loss,accuracy_score
subject_cats=','.join(train['project_subject_subcategories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_subcategories'])
 cats.project_subject_subcategories=cats.project_subject_subcategories.str.strip()
x=cats.project_subject_subcategories.value_counts()
print("There are ",len(x)," different subject sub-categories after Cleaning")
 print("They are:- ",x.index.values)
 subject_cats=','.join(approvals['project_subject_subcategories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_subcategories'])
 cats.project_subject_subcategories=cats.project_subject_subcategories.str.strip()
x1=cats.project_subject_subcategories.value_counts()
subject_cats=','.join(rejects['project_subject_subcategories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_subcategories'])
 cats.project_subject_subcategories=cats.project_subject_subcategories.str.strip()
x2=cats.project_subject_subcategories.value_counts()<define_variables>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
if model_type=='default':
for layer in layers:
clf_output = Dense(layer, activation=activation )(clf_output)
 out = Dense(1, activation='sigmoid' )(clf_output)
 elif model_type=='dropout':
for layer in layers:
x = Dropout(Dropout_num )(clf_output)
 clf_output = Dense(layer, activation=activation )(x)
 x = Dropout(Dropout_num )(clf_output)
 out = Dense(1, activation='sigmoid' )(x)
 elif model_type=='GlobalAveragePooling1D':
for layer in layers:
if Dropout_num>0:
clf_output = Dropout(Dropout_num )(clf_output)
 clf_output = Dense(layer, activation=activation )(clf_output)
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model
end_preprocess=time.time()
print("Time till sub-category:",end_preprocess-start_time,"s" )<groupby>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
state_vals=train.groupby(['school_state'])['project_is_approved','teacher_id'].agg(
 {'project_is_approved':['sum','count'],'teacher_id':['nunique']} ).reset_index()
state_vals.columns=['state','approved','total','teacher_count']
state_vals['approval_perc']=(state_vals.approved*100)/state_vals.total
state_vals['proj_per_teacher']=(state_vals.total)/state_vals.teacher_count
state_vals=state_vals.round(2)
 print("Top States")
 state_vals.sort_values('total',ascending=False ).head()<data_type_conversions>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
py.init_notebook_mode(connected=True)
 state_vals['text'] = 'Approved Projects: '+state_vals['approved'].astype(str)+ '
'+'Total Projects:'+state_vals['total'].astype(str)+'
'+\
'Total Teachers:'+state_vals['teacher_count'].astype(str)+ '
'+'
'+\
'Approval Rate: '+state_vals['approval_perc'].astype(str)
 scl = [[0.0, 'rgb(0,39,143)'],[0.2, 'rgb(0,107,177)'],[0.4, 'rgb(0,154,200)'],\
[0.6, 'rgb(0,189,220)'],[0.8, 'rgb(0,218,235)'],[1.0, 'rgb(0,240,247)']]
data = [ dict(
 type='choropleth',
colorscale = scl,
autocolorscale = False,
locations = state_vals['state'],
z = state_vals['approval_perc'],
locationmode = 'USA-states',
text = state_vals['text'],
marker = dict(
 line = dict(
 color = 'rgb(0,200,255)',
width = 2 
)) ,
colorbar = dict(
 title = "Approval Percentage")
)]
layout = dict(
 title = 'State wide Approval Analysis 
(Hover for breakdown)',
geo = dict(
 scope='usa',
projection=dict(type='albers usa'),
showlakes = True,
lakecolor = 'rgb(255, 255, 255)'), 
)
 fig = dict(data=data, layout=layout)
 py.iplot(fig, validate=False,filename='d3-cloropleth-map' )<feature_engineering>
train_input = bert_encode(train.text.values, tokenizer, max_len=max_len)
 test_input = bert_encode(test.text.values, tokenizer, max_len=max_len)
 train_labels = train.target.values
print(train_input[0].shape,test_input[0].shape )
approvals['project_submitted_datetime'] = pd.to_datetime(approvals['project_submitted_datetime'])
 approvals['datetime_dow'] = approvals['project_submitted_datetime'].dt.dayofweek
approvals['datetime_year'] = approvals['project_submitted_datetime'].dt.year
approvals['datetime_month'] = approvals['project_submitted_datetime'].dt.month
approvals['datetime_hour'] = approvals['project_submitted_datetime'].dt.hour
approvals['datetime_day'] = approvals['project_submitted_datetime'].dt.day
approvals['datetime_date'] = approvals['project_submitted_datetime'].dt.date
rejects['project_submitted_datetime'] = pd.to_datetime(rejects['project_submitted_datetime'])
 rejects['datetime_dow'] = rejects['project_submitted_datetime'].dt.dayofweek
rejects['datetime_year'] = rejects['project_submitted_datetime'].dt.year
rejects['datetime_month'] = rejects['project_submitted_datetime'].dt.month
rejects['datetime_hour'] = rejects['project_submitted_datetime'].dt.hour
rejects['datetime_day'] = rejects['project_submitted_datetime'].dt.day
rejects['datetime_date'] = rejects['project_submitted_datetime'].dt.date
app_date=approvals.groupby('datetime_date')['datetime_date'].count()
rej_date=rejects.groupby('datetime_date')['datetime_date'].count()<feature_engineering>
from sklearn.model_selection import StratifiedKFold
text_col='project_title'
target_col='project_is_approved'
text_stats=pd.DataFrame()
text_stats[target_col]=train[target_col]
text_stats['word_count']=train[text_col].apply(lambda x: len(str(x ).split()))
text_stats['count_unique_word']=train[text_col].apply(lambda x: len(set(str(x ).split())))
<sort_values>
skf = StratifiedKFold(n_splits=5 )
temp=train.groupby('project_title')['project_is_approved'].agg(['sum','count'])
 temp['approval_rate']=(temp['sum']*100)/temp['count']
temp.columns=['
temp=temp.sort_values(by='
temp=temp.iloc[0:20]
temp<define_variables>
train_labels = train.target.values
approved_mask=np.array(Image.open(".. /input/imagesfordonorchoose/1_approve_heart.png"))
rejected_mask=np.array(Image.open(".. /input/imagesfordonorchoose/2_reject_heart.png"))
approved_mask=~approved_mask[:,:,1]
rejected_mask=~rejected_mask[:,:,1]
make_word_clouds('project_title',approved_mask,rejected_mask,"What does a good title contain?" )<feature_engineering>
import gc
train['student_description']=train['project_essay_1']
train.loc[train.project_essay_3.notnull() ,'student_description']=train.loc[train.project_essay_3.notnull() ,'project_essay_1']+train.loc[train.project_essay_3.notnull() ,'project_essay_2']
test['student_description']=test['project_essay_1']
test.loc[test.project_essay_3.notnull() ,'student_description']=test.loc[test.project_essay_3.notnull() ,'project_essay_1']+test.loc[test.project_essay_3.notnull() ,'project_essay_2']<feature_engineering>
infold_pred,outfold_pred = np.zeros(train.shape[0]),np.zeros(train.shape[0])
 test_pred = np.zeros(test.shape[0])
 for dev_index, val_index in skf.split(train.text.values,train_labels):
print("TRAIN:", dev_index, "TEST:", val_index)
 dev = train.text.values[dev_index]
dev_dv = train_labels[dev_index]
val = train.text.values[val_index]
val_dv = train_labels[val_index]
dev_input = bert_encode(dev, tokenizer, max_len=max_len)
 val_input = bert_encode(val, tokenizer, max_len=max_len)
 test_input = bert_encode(test.text.values, tokenizer, max_len=max_len)
 model = build_model(bert_layer, max_len=max_len)
 model.summary()
checkpoint = ModelCheckpoint('{}.h5'.format(model_name), monitor='val_loss', save_best_only=True)
 dev_history = model.fit(
 dev_input, dev_dv,
validation_split=0.2,
epochs=3,
callbacks=[checkpoint],
batch_size=16 
)
 model.load_weights('{}.h5'.format(model_name))
infold_pred[dev_index] += model.predict(dev_input)[:,0]
outfold_pred[val_index] += model.predict(val_input)[:,0]
test_pred += model.predict(test_input)[:,0]
os.remove('{}.h5'.format(model_name))
gc.collect()
train['project_description']=train['project_essay_2']
train.loc[train.project_essay_3.notnull() ,'project_description']=train.loc[train.project_essay_3.notnull() ,'project_essay_3']+train.loc[train.project_essay_3.notnull() ,'project_essay_4']
test['project_description']=test['project_essay_2']
test.loc[test.project_essay_3.notnull() ,'project_description']=test.loc[test.project_essay_3.notnull() ,'project_essay_3']+test.loc[test.project_essay_3.notnull() ,'project_essay_4']
test[test.project_essay_3.notnull() ].head(1 ).project_description.values<drop_column>
infold_pred = infold_pred/4
test_pred = test_pred/5
del(train['project_essay_1'])
 del(train['project_essay_2'])
 del(train['project_essay_3'])
 del(train['project_essay_4'])
 del(test['project_essay_1'])
 del(test['project_essay_2'])
 del(test['project_essay_3'])
 del(test['project_essay_4'])
 approvals=train[train.project_is_approved==1]
rejects=train[train.project_is_approved==0]<load_pretrained>
import pickle
rejected_mask=np.array(Image.open(".. /input/imagesfordonorchoose/1_approve_hand.png"))
approved_mask=np.array(Image.open(".. /input/imagesfordonorchoose/2_reject_hand.png"))
approved_mask=~approved_mask[:,:,1]
rejected_mask=~rejected_mask[:,:,1]
make_word_clouds('student_description',approved_mask,rejected_mask,"How are Students described in Approved VS Rejected projects?" )<load_pretrained>
with open('predictions.pkl','wb')as f:
pickle.dump({'infold_pred':infold_pred,'outfold_pred':outfold_pred,'test_pred':test_pred},f )
approved_mask=np.array(Image.open(".. /input/imagesfordonorchoose/1_approve_tick.png"))
rejected_mask=np.array(Image.open(".. /input/imagesfordonorchoose/2_reject_tick.png"))
approved_mask=~approved_mask[:,:,1]
rejected_mask=~rejected_mask[:,:,1]
make_word_clouds('project_description',approved_mask,rejected_mask,"How are Projects described in Approved VS Rejected projects?" )<train_model>
print('Train score:{}, test score:{}'.format(f1_score(train_labels,infold_pred.round().astype(int)) ,
f1_score(train_labels,outfold_pred.round().astype(int))))
end_time=time.time()
print("Time till plotting section",end_time-start_time,"s" )<data_type_conversions>
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('{}.csv'.format(model_name), index=False )