Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<categorify>
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K
from transformers import *
import tokenizers
import os
from tokenizers import BertWordPieceTokenizer
from transformers import BertTokenizer, TFBertModel, BertConfig
from sklearn.model_selection import StratifiedShuffleSplit
from transformers import BertTokenizer, TFBertForQuestionAnswering
from sklearn.model_selection import train_test_split
import gc
from keras.callbacks import ModelCheckpoint
from sklearn.model_selection import StratifiedKFold
from keras.utils import plot_model
%%markdown
1)fill the *NaN* values with mean(*Age feature*)our more frequent values(*Embarked feature*)
 2)Add *titles* of passengers from names then delete names
3)Encode categorical features into integers
4)make 4 bins of age to categorize it
5)Normalize our datasets(training_set and test_set)
 6)Split our training set into training and validation sets(80%/20%)
<data_type_conversions>
max_len = 168
train_mode = False
tokenizer = tokenizers.ByteLevelBPETokenizer(vocab_file = '.. /input/tf-roberta/vocab-roberta-base.json',
merges_file = '.. /input/tf-roberta/merges-roberta-base.txt',
add_prefix_space = True,
lowercase = True )
for i in range(len(dataset)) :
freq_port = dataset[i]['Embarked'].dropna().mode() [0]
dataset[i]['Embarked'] = dataset[i]['Embarked'].fillna(freq_port)
 dataset[i] = dataset[i].fillna(dataset[i].mean() )<feature_engineering>
sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
x_train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 x_train.head()
print("Titels of passengers by sex")
 dataset[0]['Title'] = dataset[0].Name.str.extract('([A-Za-z]+)\.', expand=False)
 display(pd.crosstab(dataset[0]['Sex'], dataset[0]['Title']))<feature_engineering>
x_train['text'].replace('', np.nan, inplace=True)
 x_train.dropna(subset=['text'], inplace=True)
 x_train.reset_index(drop=True, inplace=True )
for i, data in enumerate(dataset):
dataset[i]['Title'] = data.Name.str.extract('([A-Za-z]+)\.', expand=False)
 dataset[i]['Title'] = data['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev',
'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset[i]['Title'] = data['Title'].replace(['Mlle', 'Ms'], 'Miss')
 dataset[i]['Title'] = data['Title'].replace('Mme', 'Mrs')
 print(dataset[i]['Title'].unique() )<categorify>
tweet = 'WHY DO WE FALL? SO THAT WE CAN LEARN TO PICK OURSELVES BACK UP.'
selected_text = 'LEARN TO PICK OURSELVES BACK UP'
sentiment = 'positive'
print('text',tweet)
 print('select_text:',selected_text)
 print('sentiment:', sentiment)
 idx0 = None
idx1 = None
st_len = len(selected_text)
 tweet = " " + tweet
for i in range(len(tweet)) :
if(tweet[i:i+st_len]==selected_text):
idx0 = i
idx1 = i + st_len -1
break
char_targets = [0]*len(tweet)
 for i in range(len(tweet)) :
if idx0 != None and idx1!=None:
if i>=idx0 and i<=idx1:
char_targets[i] = 1
print('char_targets:',char_targets)
 tok_tweet = tokenizer.encode(tweet)
 input_ids = tok_tweet.ids
offsets = tok_tweet.offsets
print('input_ids:',input_ids)
 print('offsets:',offsets)
 targets_index = []
for i,(off1,off2)in enumerate(offsets):
if sum(char_targets[off1:off2])>0:
targets_index.append(i)
 target_start = targets_index[0]
target_end = targets_index[-1]
print('target_start:',target_start)
 print('target_end:', target_end)
 ids = [0] + input_ids + [2,2] + [sentiment_id[sentiment]] + [2]
token_type_ids = [0]*(len(ids))
mask = [1] * len(ids)
 target_start+=1
target_end+=1
offsets = [(0,0)]*1 + offsets + [(0,0)]*4
padding_length = max_len - len(ids)
 if padding_length > 0:
ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
encoder = LabelEncoder()
categoricalFeatures = dataset[0].select_dtypes(include=['object'] ).columns
for i, data in enumerate(dataset):
data[categoricalFeatures]=data[categoricalFeatures].astype(str)
 encoded = data[categoricalFeatures].apply(encoder.fit_transform)
 for j in categoricalFeatures:
dataset[i][j]=encoded[j]
dataset[0].head()<feature_engineering>
def text_process(tweet, sentiment, tokenizer, max_len, selected_text=None):

 if selected_text!=None:
tweet = " " + tweet
idx0 = None
idx1 = None
st_len = len(selected_text)
 for i in range(len(tweet)) :
if(tweet[i:i+st_len]==selected_text):
idx0 = i
idx1 = i + st_len -1
break
char_targets = [0]*len(tweet)
 for i in range(len(tweet)) :
if idx0 != None and idx1!=None:
if i>=idx0 and i<=idx1:
char_targets[i] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids = tok_tweet.ids
offsets = tok_tweet.offsets
targets_index = []
for i,(off1,off2)in enumerate(offsets):
if sum(char_targets[off1:off2])>0:
targets_index.append(i)
 target_start = targets_index[0]
target_end = targets_index[-1]
ids = [0] + input_ids + [2,2] + [sentiment_id[sentiment]] + [2]
token_type_ids = [0]*(len(ids))
mask = [1] * len(ids)
 target_start+=1
target_end+=1
offsets = [(0,0)]*1 + offsets + [(0,0)]*4
padding_length = max_len - len(ids)
 if padding_length > 0:
ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
 return {
'ids': ids,
'token_type_ids':token_type_ids,
'mask':mask,
'target_start':target_start,
'target_end':target_end,
'offsets':offsets
}
else:
tweet = " " + tweet
tok_tweet = tokenizer.encode(tweet)
 input_ids = tok_tweet.ids
offsets = tok_tweet.offsets
ids = [0] + input_ids + [2,2] + [sentiment_id[sentiment]] + [2]
token_type_ids = [0]*(len(ids))
mask = [1] * len(ids)
 offsets = [(0,0)]*1 + offsets + [(0,0)]*4
padding_length = max_len - len(ids)
 if padding_length > 0:
ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
 return {
'ids': ids,
'token_type_ids':token_type_ids,
'mask':mask,
'offsets':offsets
}
bins = [0,18,60,80]
labels = [1,2,3]
for i, data in enumerate(dataset):
dataset[i] = dataset[i].drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1)
 dataset[i]['Age']=pd.cut(dataset[i]['Age'],bins=bins ,labels=labels)
 dataset[i]['Age']=dataset[i]['Age'].astype('int64')
 print('training dataset:')
 display(dataset[0].head())
 print('testing dataset:')
 display(dataset[1].head() )<prepare_x_and_y>
rows = x_train.shape[0]
train_ids = np.zeros(( rows,max_len), dtype = 'int32')
 train_mask = np.zeros(( rows,max_len), dtype = 'int32')
 train_type_ids = np.zeros(( rows,max_len), dtype = 'int32')
 train_start_idx = np.zeros(( rows,max_len), dtype = 'int32')
 train_end_idx = np.zeros(( rows,max_len), dtype = 'int32')
 for i in range(x_train.shape[0]):
encoding = text_process(x_train.loc[i,'text'], x_train.loc[i,'sentiment'], tokenizer, max_len,x_train.loc[i,'selected_text'])
 train_ids[i] = encoding['ids']
train_start_idx[i,encoding['target_start']] = 1
train_end_idx[i, encoding['target_end']] = 1
train_type_ids[i] = encoding['token_type_ids']
train_mask[i] = encoding['mask']
X=dataset[0].iloc[:, 1:]
Y=dataset[0].iloc[:, 0]
x_test=dataset[1].iloc[:, 0:]
normalized_data = X
normalized_data=normalized_data.append(x_test)
 normalized_x_train = normalized_data.values
normalized_x_train /= np.max(np.abs(normalized_x_train),axis=0)
 X = pd.DataFrame(normalized_x_train[:891,:],
columns=['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Title'])
 print(X.head())
 print(len(X))
x_test = pd.DataFrame(normalized_x_train[891:,:],
columns=['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Title'])
 display(x_test.head())
 print(len(x_test))
display(X.columns )<split>
i = 10
encoding = text_process(x_train.loc[i,'text'], x_train.loc[i,'sentiment'], tokenizer, max_len,x_train.loc[i,'selected_text'])
 train_ids[i] = encoding['ids']
train_start_idx[i,encoding['target_start']] = 1
train_end_idx[i, encoding['target_end']] = 1
train_type_ids[i] = encoding['token_type_ids']
train_mask[i] = encoding['mask']
tweet = x_train.loc[i,'text']
select_text = x_train.loc[i,'selected_text']
target_start = np.argmax(train_start_idx[i,])
 target_end = np.argmax(train_end_idx[i, ])
 offsets = encoding['offsets']
print('tweet:',tweet)
 print('selected_text:',select_text)
 print(tweet[offsets[target_start][0]:offsets[target_end][-1]] )
X_train, X_val, y_train, y_val = train_test_split(X, Y, test_size = 0.20)
 print("Training set shape: "+str(X_train.shape))
print("Validation set shape: "+str(X_val.shape))<set_options>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0):
return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
%%markdown
1)Logistic Regression
2)Decision Tree
3)Random Forest
4)XGBoost 
<define_variables>
config = RobertaConfig.from_pretrained('.. /input/tf-roberta/config-roberta-base.json')
 print(config )
accuracies_list = list()
accuracies = namedtuple('accuracies',('Model', 'accuracy'))<set_options>
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 roberta = TFRobertaForQuestionAnswering.from_pretrained('.. /input/tf-roberta/pretrained-roberta-base.h5', config = config)
 x = roberta(ids, attention_mask = att, token_type_ids = tok)
 x1 = tf.keras.layers.Dropout(0.3 )(x[0])
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.3 )(x[1])
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs = [ids, att, tok], outputs=[x1, x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate = 3e-5)
 model.compile(loss= 'categorical_crossentropy', optimizer=optimizer)
 return model
%%markdown 
<choose_model_class>
def lr_scheduler(epoch):
return 3e-5 * 0.2**epoch
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
 Y_pred = logreg.predict(X_val)
 acc_log = round(logreg.score(X_train, y_train)* 100, 2)
 print('accuracy: {}'.format(acc_log))
accuracies_list.append(accuracies('Logistic Regression', acc_log))<set_options>
gc.collect()
%%markdown 
<choose_model_class>
x_test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 rows = x_test.shape[0]
preds_start = np.zeros(( rows,max_len))
preds_end = np.zeros(( rows,max_len))
decisiontree = DecisionTreeClassifier()
decisiontree.fit(X_train, y_train)
 y_pred = decisiontree.predict(X_val)
 acc_decisiontree = round(accuracy_score(y_pred, y_val)* 100, 2)
 print('accuracy: {}'.format(acc_decisiontree))
accuracies_list.append(accuracies('Decision Tree', acc_decisiontree))<set_options>
rows = x_test.shape[0]
test_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_mask = np.zeros(( rows,max_len), dtype = 'int32')
 test_type_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_start_idx = np.zeros(( rows,max_len), dtype = 'int32')
 test_end_idx = np.zeros(( rows,max_len), dtype = 'int32')
 for i in range(x_test.shape[0]):
encoding = text_process(x_test.loc[i,'text'], x_test.loc[i,'sentiment'], tokenizer, max_len)
 test_ids[i] = encoding['ids']
test_type_ids[i] = encoding['token_type_ids']
test_mask[i] = encoding['mask']
%%markdown 
<compute_train_metric>
for i in range(n_splits):
model.load_weights(f'.. /input/roberta-folds/roberta-{i}.h5')
 preds = model.predict([test_ids, test_mask, test_type_ids], verbose = True)
 preds_start += preds[0]/n_splits
preds_end += preds[1] /n_splits
clf = RandomForestClassifier(max_depth=10, max_leaf_nodes =20,random_state=0)
 clf.fit(X_train,y_train)
 y_pred=clf.predict(X_val)
 acc_random_forest = round(accuracy_score(y_pred, y_val)* 100, 2)
 print('accuracy: {}'.format(acc_random_forest))
accuracies_list.append(accuracies('Random Forest', acc_random_forest))<set_options>
submission = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv' )
%%markdown 
<compute_train_metric>
for k in range(x_test.shape[0]):
if x_test.loc[k,'sentiment']!='neutral':
encoding = text_process(x_test.loc[k,'text'], x_test.loc[k,'sentiment'], tokenizer, max_len)
 offsets = encoding['offsets']
targets_start = np.argmax(preds_start[k,])
 targets_end = np.argmax(preds_end[k,])
 if targets_start<=targets_end:
pred = x_test.loc[k,'text'][offsets[targets_start][0]:offsets[targets_end][-1]]
submission.loc[k,'selected_text'] = pred
else:
submission.loc[k,'selected_text'] = x_test.loc[k,'text']
else:
submission.loc[k,'selected_text'] = x_test.loc[k,'text']
<train_model><EOS> 
submission.to_csv('submission.csv', index = False )