Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<feature_engineering>
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K
from sklearn.model_selection import StratifiedKFold
from keras.layers import Dense, Flatten, Conv1D, Dropout, Input
from keras.models import Model, load_model
from keras.callbacks import ModelCheckpoint, EarlyStopping
from transformers import *
import tokenizers
housing_df[categorical_col[17]]=housing_df[categorical_col[17]].replace(np.nan,'TA' )<define_variables>
data_path = '.. /input/tweet-sentiment-extraction'
path = '.. /input/roberta-base'
path2 = '.. /input/tf-roberta'
max_len = 128
vocab_file = path + '/vocab.json'
merges_file = path + '/merges.txt'
config_file = path2 + '/config-roberta-base.json'
pretrained_file = path2 + '/pretrained-roberta-base.h5'
print("LotFrontage contains %s missing values"%(mis_cols.LotFrontage))<categorify>
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file,
merges_file,
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive':tokenizer.encode('positive' ).ids[0],
'negative':tokenizer.encode('negative' ).ids[0],
'neutral':tokenizer.encode('neutral' ).ids[0]}
train_df['LotFrontage']=train_df.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
test_df['LotFrontage']=test_df.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))<filter>
train_data = pd.read_csv(data_path + '/train.csv')
 train_data.dropna(axis = 0,inplace=True)
 train_data.head()
Garage_cols = int_mis_cols[int_mis_cols.str.contains('Gar')]
Basement_cols = int_mis_cols[int_mis_cols.str.contains('Bsmt')]
print(Garage_cols,"
",Basement_cols )<categorify>
test_data = pd.read_csv(data_path + '/test.csv')
 test_data.head()
housing_df[Garage_cols]=housing_df[Garage_cols].replace(np.nan,0)
 housing_df[Basement_cols]=housing_df[Basement_cols].replace(np.nan,0)
 housing_df['MasVnrArea']=housing_df['MasVnrArea'].replace(np.nan,0 )<filter>
train_data.reset_index(inplace=True)
 tot_tw = train_data.shape[0]
input_ids = np.ones(( tot_tw, max_len), dtype='int32')
 attention_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 token_type_ids = np.zeros(( tot_tw, max_len), dtype='int32')
 start_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 end_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 for i in range(tot_tw):
set1 = " "+" ".join(train_data.loc[i,'text'].split())
 set2 = " ".join(train_data.loc[i,'selected_text'].split())
 idx = set1.find(set2)
 set2_loc = np.zeros(( len(set1)))
 set2_loc[idx:idx+len(set2)]=1
if set1[idx-1]==" ":
set2_loc[idx-1]=1
enc_set1 = tokenizer.encode(set1)
 selected_text_token_idx=[]
for k,(a,b)in enumerate(enc_set1.offsets):
sm = np.sum(set2_loc[a:b])
 if sm > 0:
selected_text_token_idx.append(k)
 senti_token = sentiment_id[train_data.loc[i,'sentiment']]
input_ids[i,:len(enc_set1.ids)+5] = [0]+enc_set1.ids+[2,2]+[senti_token]+[2]
attention_mask[i,:len(enc_set1.ids)+5]=1
if len(selected_text_token_idx)> 0:
start_mask[i,selected_text_token_idx[0]+1]=1
end_mask[i, selected_text_token_idx[-1]+1]=1
mis_val_percent = round(( 100 * housing_df.isnull().sum() / len(housing_df)) ,2)
 mis_val_percent[mis_val_percent>0]<count_unique_values>
tot_test_tw = test_data.shape[0]
input_ids_t = np.ones(( tot_test_tw,max_len), dtype='int32')
 attention_mask_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 token_type_ids_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 for i in range(tot_test_tw):
set1 = " "+" ".join(test_data.loc[i,'text'].split())
 enc_set1 = tokenizer.encode(set1)
 s_token = sentiment_id[test_data.loc[i,'sentiment']]
input_ids_t[i,:len(enc_set1.ids)+5]=[0]+enc_set1.ids+[2,2]+[s_token]+[2]
attention_mask_t[i,:len(enc_set1.ids)+5]=1
housing_df.columns.nunique()<feature_engineering>
def custom_loss(y_true, y_pred):
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, from_logits = False, label_smoothing = 0.20)
 loss = tf.reduce_mean(loss)
 return loss
train_df.SalePrice=np.log(train_df.SalePrice )<feature_engineering>
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 config_path = RobertaConfig.from_pretrained(config_file)
 roberta_model = TFRobertaModel.from_pretrained(pretrained_file, config=config_path)
 x = roberta_model(ids, attention_mask = att, token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.05 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.05 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=custom_loss, optimizer=optimizer)
 return model
<count_unique_values><EOS> 
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(pred_start[k,])
 b = np.argmax(pred_end[k,])
 if a>b:
st = test_data.loc[k,'text']
else:
text1 = " "+" ".join(test_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 test_data['selected_text']=all
test_data.head(20)
 test_data[['textID','selected_text']].to_csv('submission.csv', index=False )