Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<string_transform>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import StratifiedKFold
from transformers import *
import tokenizers
import tensorflow as tf
import tensorflow.keras.backend as K
tokenizer = Tokenizer()
num_rows = df_train.shape[0]
tokenizer.fit_on_texts(df_train['Phrase'].values)
 row_max_length = max([len(x.split())for x in df_train['Phrase'].values])
 vocabulary_size = len(tokenizer.word_index)+ 1
X_train_tokens = tokenizer.texts_to_sequences(X_train)
 X_test_tokens = tokenizer.texts_to_sequences(X_test)
 X_sub_tokens = tokenizer.texts_to_sequences(df_submission['Phrase'].values)
 X_train_pad = pad_sequences(X_train_tokens, maxlen=row_max_length, padding='post')
 X_test_pad = pad_sequences(X_test_tokens, maxlen=row_max_length, padding='post')
 X_sub_pad = pad_sequences(X_sub_tokens, maxlen=row_max_length, padding='post' )<categorify>
y_train_cat = to_categorical(y_train)
 y_test_cat = to_categorical(y_test)
 target_length = y_train_cat.shape[1]
print('Original vector size: {}'.format(y_train.shape))
print('Converted vector size: {}'.format(y_train_cat.shape))<train_model>
tw_pal = ['
tw_pal_1 = ['
sns.set_style("whitegrid" )
EMBEDDING_DIM = 256
model = Sequential()
model.add(Embedding(vocabulary_size, EMBEDDING_DIM, input_length=row_max_length))
model.add(SpatialDropout1D(0.2))
model.add(Bidirectional(GRU(128)))
 model.add(Dense(128, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(target_length, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=1)
 history = model.fit(X_train_pad, y_train_cat, epochs=5, validation_data=(X_test_pad, y_test_cat), batch_size=128, callbacks=[callback] )<save_to_csv>
train = pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv")
 test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv" )
y_sub_hat_ = model.predict(X_sub_pad)
 y_sub_hat = [np.argmax(x)for x in y_sub_hat_]
df_save = pd.read_csv('/kaggle/input/sentiment-analysis-on-movie-reviews/sampleSubmission.csv')
 df_save['Sentiment'] = y_sub_hat
df_save.to_csv('Submission.csv', index = False)
 print('Submission saved!' )<set_options>
print(train.isna().sum())
 print("")
 print(test.isna().sum())
 print("")
 print(sample.isna().sum() )
sns.set_style('whitegrid')
 warnings.simplefilter(action='ignore')
<load_from_csv>
train[train.isna().any(axis=1)]
a = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 b = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<set_options>
train = train.dropna()
print(train.isna().sum().sum() )
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None )<train_model>
print(train.nunique())
 print('')
 print(test.nunique())
 print('')
 print(sample.nunique() )
print('The shape of our training set: %s houses and %s features'%(a.shape[0],a.shape[1]))
print('The shape of our training set: %s houses and %s features'%(b.shape[0],b.shape[1]))
print('The testing set has 1 feature less than the training set, which is SalePrice, the target to predict ' )<prepare_x_and_y>
train['no_words'] = train['text'].str.split().apply(len)
 train['no_words_sel_text'] = train['selected_text'].str.split().apply(len)
 col = ['no_words', 'no_words_sel_text']
col_names = ['Mean no.of words in text', 'Mean no.of words in selected text']
plot_bar(col, col_names )
na = a.shape[0]
nb = b.shape[0]
y_train = a['SalePrice'].to_frame()
c1 = pd.concat(( a, b), sort=False ).reset_index(drop=True)
 c1.drop(['SalePrice'], axis=1, inplace=True)
 c1.drop(['Id'], axis=1, inplace=True)
 print(f"Total size is {c1.shape}" )<compute_test_metric>
def hash_count(tweet):
w = tweet.split()
return len([word for word in w if word.startswith('
def mention_count(tweet):
w = tweet.split()
return len([word for word in w if word.startswith('@')])
 def web_add(tweet):
w = tweet.split()
return len([word for word in w if word.startswith('http')] )
msv1(c1, 20, color=('black', 'deeppink'))<drop_column>
print('TF version',tf.__version__ )
c=c1.dropna(thresh=len(c1)*0.8, axis=1)
 print(f"We dropped {c1.shape[1]-c.shape[1]} features in the combined set" )<compute_test_metric>
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()
msv2(c )<data_type_conversions>
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
NA=c[allna.index.to_list() ]<data_type_conversions>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
c['MasVnrArea']=c.MasVnrArea.fillna(0)
 c['LotFrontage']=c.LotFrontage.fillna(c.LotFrontage.median())
 c['GarageYrBlt']=c["GarageYrBlt"].fillna(1980 )<categorify>
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
ffill_cols = ['Electrical', 'SaleType', 'KitchenQual', 'Exterior1st',
'Exterior2nd', 'Functional', 'Utilities', 'MSZoning']
def filling_NA(data, columns, METHOD='ffill'):
fill_cols = columns
for col in data[fill_cols]:
data[col]= data[col].fillna(method=METHOD)
 return data<compute_test_metric>
msv2(c )<concatenate>
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
d=filling_NA(c, ffill_cols )<compute_test_metric>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model
msv2(d )<data_type_conversions>
n_splits = 5
fill_cols = ['Electrical', 'SaleType', 'KitchenQual', 'Exterior1st',
'Exterior2nd', 'Functional', 'Utilities', 'MSZoning']
for col in c[fill_cols]:
c[col] = c[col].fillna(method='ffill' )<data_type_conversions>
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(10):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits
NAcols=c.columns
for col in NAcols:
if c[col].dtype == "object":
c[col] = c[col].fillna("None" )<data_type_conversions>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_df.loc[k,'text']
else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )
<feature_engineering><EOS> 
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )