Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<categorify>
!pip install transformers
le = preprocessing.LabelEncoder()
train['province_encoder'] = le.fit_transform(train['Province_State'])
 test['province_encoder'] = le.transform(test['Province_State'] )<count_duplicates>
import tensorflow as tf
import keras
import transformers
from transformers import *
import pandas as pd
import numpy as np
import tokenizers
<load_from_csv>
print(tf.__version__ )
train_df = pd.read_csv(Path('/kaggle/input/covid19-train-df', 'train_df.csv'), index_col = 0, parse_dates = ['date'])
 train_df = train_df[train_df['date_int']>=301]
train_df['weekday'] = train_df['date'].dt.weekday
train_df[train_df['country'] == 'Italy'].tail(10 )<split>
vocab_file = '.. /input/tf-roberta/vocab-roberta-base.json'
merge_file = '.. /input/tf-roberta/merges-roberta-base.txt'
tokenizer = tokenizers.ByteLevelBPETokenizer(vocab_file, merge_file,lowercase = True )
def split_train_val(df, val_ratio):
val_len = int(len(df)* val_ratio)
 train_set = df[:-val_len]
val_set = df[-val_len:]
return train_set, val_set<prepare_x_and_y>
MAX_LEN = 100
sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()
test_fixed_cols = ['ForecastId', 'Province_State', 'Country_Region', 'Date']
fixed_cols = ['Id', 'province', 'country', 'date']
output_cols = ['cc_cases', 'ft_cases']
input_cols = list(set(train_df.columns.to_list())- set(fixed_cols)- set(output_cols))
print('output columns are ', output_cols)
 print('input columns are ', input_cols)
 X = train_df[input_cols]
y = train_df[output_cols]<categorify>
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
cc_input = ['country_encoder', 'province_encoder', 'weekday', 'date_int','cc_cases_1', 'cc_cases_2', 'cc_cases_3', 'cc_cases_4', 'cc_cases_5', 'cc_cases_6', 'cc_cases_7']
ft_input = ['country_encoder', 'province_encoder', 'weekday' , 'date_int', 'ft_cases_1', 'ft_cases_2', 'ft_cases_3', 'ft_cases_4', 'ft_cases_5', 'ft_cases_6', 'ft_cases_7']
cc_output = ['cc_cases']
ft_output = ['ft_cases']
val_ratio = 0.05
X_cc = X[cc_input]
X_ft = X[ft_input]
y_cc = y[cc_output]
y_ft = y[ft_output]
train_X_cc, val_X_cc = split_train_val(df = X_cc, val_ratio = val_ratio)
 train_y_cc, val_y_cc = split_train_val(df = y_cc, val_ratio = val_ratio)
 train_X_ft, val_X_ft = split_train_val(df = X_ft, val_ratio = val_ratio)
 train_y_ft, val_y_ft = split_train_val(df = y_ft, val_ratio = val_ratio )<normalization>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
<normalization>
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained('.. /input/tf-roberta/config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained('.. /input/tf-roberta/pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 drop1 = tf.keras.layers.Dropout(0.2 )(x[0])
 layer2 = tf.keras.layers.Conv1D(50 ,kernel_size = 1 )(drop1)
 drop2 = tf.keras.layers.Dropout(0.2 )(layer2)
 layer3 = tf.keras.layers.Conv1D(1 ,kernel_size = 1 )(drop2)
 layer4 = tf.keras.layers.Flatten()(layer3)
 output_1 = tf.keras.layers.Activation('softmax' )(layer4)
 drop1_ = tf.keras.layers.Dropout(0.2 )(x[0])
 layer1_ = tf.keras.layers.Conv1D(50 ,kernel_size = 1 )(drop1_)
 drop2_ = tf.keras.layers.Dropout(0.2 )(layer1_)
 layer2_ = tf.keras.layers.Conv1D(1 ,kernel_size = 1 )(drop2_)
 layer3_ = tf.keras.layers.Flatten()(layer2_)
 output_2 = tf.keras.layers.Activation('softmax' )(layer3_)
 model = tf.keras.Model(inputs = [ids ,att ,tok] ,outputs = [output_1 ,output_2])
 model.summary()
<data_type_conversions>
def my_loss(alpha ,gamma):

 def focal_loss(y_true ,y_pred):
y_true = tf.cast(y_true ,dtype = tf.float32)
 y_pred = tf.cast(y_pred ,dtype = tf.float32)
 log_lik = y_true*tf.keras.backend.log(y_pred)
 log_lik = alpha*(( 1-y_pred)**gamma)*log_lik
return -tf.keras.backend.sum(log_lik)
 return focal_loss
X_train_cc = train_X_cc.to_numpy()
X_val_cc = val_X_cc.to_numpy()
X_train_ft = train_X_ft.to_numpy()
X_val_ft = val_X_ft.to_numpy()
y_train_cc = train_y_cc.to_numpy()
y_val_cc = val_y_cc.to_numpy()
y_train_ft = train_y_ft.to_numpy()
y_val_ft = val_y_ft.to_numpy()<compute_test_metric>
optimizer = tf.keras.optimizers.Adam(learning_rate=0.000001)
 model.compile(loss = my_loss(1.0 ,2.0),optimizer = optimizer )
def root_mean_squared_log_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(K.log(y_pred + 1)- K.log(y_true + 1))))<choose_model_class>
history = model.fit([input_ids[800:], attention_mask[800:],token_type_ids[800:]], [start_tokens[800:], end_tokens[800:]] ,epochs = 30 ,
validation_split = 0.1 ,batch_size = 32 )
def LSTM_model(n_1, input_dim, output_dim):
model = Sequential()
model.add(LSTM(n_1,input_shape=(1, input_dim), activation='relu'))
model.add(Dense(output_dim, activation='relu'))
model.compile(loss=root_mean_squared_log_error, optimizer='adam')
 return model<choose_model_class>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
K.clear_session()
model_cc = LSTM_model(4, X_train_cc.shape[-1], y_train_cc.shape[-1])
 model_ft = LSTM_model(4, X_train_ft.shape[-1], y_train_ft.shape[-1])
 early_stop_cc = EarlyStopping(monitor='val_loss', patience=5, verbose=0, mode='min')
 early_stop_ft = EarlyStopping(monitor='val_loss', patience=5, verbose=0, mode='min' )<train_model>
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_start,oof_end = model.predict([input_ids[:800],attention_mask[:800],token_type_ids[:800]],verbose=1 )
print('Start model training')
 start_time = time.time()
history_cc = model_cc.fit(X_train_cc, y_train_cc, batch_size = 16, epochs = 100,validation_data =(X_val_cc, y_val_cc), verbose = 2, callbacks=[early_stop_cc])
 model_cc.save("model_cc.h5")
 print('Time spent for model training is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<train_model>
all = []
jac = []
for k in range(800):
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('Start model training')
 start_time = time.time()
history_ft = model_ft.fit(X_train_ft, y_train_ft, batch_size = 16, epochs = 8,validation_data =(X_val_ft, y_val_ft), verbose = 2, callbacks=[early_stop_ft])
 model_ft.save("model_ft.h5")
 print('Time spent for model training is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<predict_on_test>
preds = model.predict([input_ids_t ,attention_mask_t ,token_type_ids_t] ,verbose = 1)
 preds_start = preds[0]
preds_end = preds[1]
yhat_val_cc = model_cc.predict(X_val_cc)
 print(yhat_val_cc[50:70] )<predict_on_test>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax([preds_start[k ,]])
 b = np.argmax(preds_end[k ,])
 if a>b:
st = test.loc[k ,'text']
else:
text1 = " " + " ".join(test.loc[k ,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )
yhat_val_ft = model_ft.predict(X_val_ft)
 print(yhat_val_ft[60:70] )<create_dataframe>
test['selected_text'] = all
test[['textID' ,'selected_text']].to_csv('submission.csv' ,index = False)
 pd.set_option('max_colwidth' ,60)
 test.sample(25 )