Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<set_options>
print('TF version',tf.__version__ )
pd.set_option('display.max_columns', 30)
 sns.set() 
<load_from_csv>
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<count_missing_values>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
print('Number of records:')
 print(len(df_train.index))
print('{:=<70}'.format(''))
print('Missing values in the training dataset:')
 print(df_train.isnull().sum())
 print('{:=<70}'.format(''))<count_missing_values>
MAX_LEN = 196
EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 42
N_SPLITS = 5
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
print('Number of records:')
 print(len(df_test.index))
print('{:=<70}'.format(''))
print('Missing values in the test dataset:')
 print(df_test.isnull().sum())
 print('{:=<70}'.format(''))<feature_engineering>
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1
df_train['Dataset'] = 'train'
df_test['Dataset'] = 'test'
df_test.insert(loc=1, column='Survived', value=np.nan)
 df_all = df_train.append(df_test )<count_missing_values>
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1
print('Number of records:')
 print(len(df_all.index))
print('{:=<70}'.format(''))
print('Missing values in the test dataset:')
 print(df_all.isnull().sum())
 print('{:=<70}'.format(''))<count_values>
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
print('Percentage of passengers who survived(1)and died(0):')
 print(df_train['Survived'].value_counts(normalize=True))<count_values>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.15 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.15 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model
sex = df_train['Sex'].value_counts(normalize=True)
 print('{:=<70}'.format(''))
print('Proportion of passengers by gender:')
 print(sex)
 print('{:=<70}'.format(''))
f_passengers = sex['female']
m_passengers = sex['male']<create_dataframe>
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=N_SPLITS,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train_df.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train_df.loc[k,'text']
else:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train_df.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
proportions_col = ['category', '%passengers', '%survived']
proportions_cat = ['female', 'male', 'class 1', 'class 2', 'class 3']
proportions_survived = [f_survived, m_survived, c1_survived, c2_survived, c3_survived]
proportions_passengers =[f_passengers, m_passengers, c1_passengers, c2_passengers, c3_passengers]
df_proportions = pd.DataFrame(
 list(zip(proportions_cat,
proportions_passengers,
proportions_survived)
),
columns=proportions_col)
 df_proportions['variance'] = df_proportions['%passengers'] - df_proportions['%survived']
df_proportions_melted = df_proportions.melt(id_vars=['category', 'variance'],
var_name='ratio type',
value_name='ratio value')
 my_red = '
my_green = '
l_colours = [my_red if i >=0 else my_green for i in df_proportions['variance'].tolist() ]
d_colours = dict(zip(df_proportions['category'],l_colours))
fig, ax = plt.subplots(figsize=(6,8))
ax = sns.pointplot(x='ratio type',
y='ratio value',
hue='category',
data=df_proportions_melted,
palette=d_colours)
 for i in range(0, len(df_proportions)) :
lbl_offset = 1.1
if(df_proportions['%passengers'] - df_proportions['%survived'])[i] < 0:
ax.text(x=lbl_offset,
y=df_proportions['%survived'][i],
s=df_proportions['category'][i],
ha='left',
color=my_green)
 else:
ax.text(x=lbl_offset,
y=df_proportions['%survived'][i],
s=df_proportions['category'][i],
ha='left',
color=my_red)
 ax.set(title='Proportions of passengers in the total and survivors population',
ylabel='Percentage')
 ax.xaxis.label.set_visible(False)
 ax.legend_.set_visible(False)
 plt.show()<create_dataframe>
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
summary = pd.pivot_table(data=df_train,
index=['Survived'],
columns=['Sex', 'Pclass'],
values=['Name'],
aggfunc=('count'),
margins=True,
margins_name='Total')
 print('{:=<70}'.format(''))
print(summary.div(summary.iloc[-1]))
print('{:=<70}'.format(''))<statistical_test>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_df.loc[k,'text']
else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )
def get_chi2(crosstab, HasMargin=True, proba=0.95):

 var1 = crosstab.index.name
var2 = crosstab.columns.name
if HasMargin==True:
i = -1
else:
i = ""
stat, p, dof, expected = chi2_contingency(
 observed=crosstab.iloc[:i,:i],
correction=True)
 critical = chi2.ppf(proba, dof)
 if stat >= critical:
chi2_independence = 'The variables {} and {} are dependent(X2>eX2)'.format(var1, var2)
 else:
chi2_independence = 'The variables {} and {} are independent(X2<eX2)'.format(var1, var2)
 alpha = 1 - proba
if p <= alpha:
chi2_significance = 'The test is statistically significant(p<alpha)'
else:
chi2_significance = 'The test is not statistically significant(p>alpha)'
print('{:^70}'.format('Chi2 test of independence'))
print('{:=<70}'.format(''))
print('Tested variables')
 print(var1)
 print(var2)
 print('{:=<70}'.format(''))
print('{}'.format('Test of independence'))
print('Chi-square statistic(X2): {:.3f}'.format(stat))
print('Minimum expected chi-square statistic(eX2): {:.3f}'.format(critical))
print(chi2_independence)
 print('{:=<70}'.format(''))
print('Significance of the test')
 print('p-value(p): {:.3f}'.format(p))
print('Accepted significance level(alpha): {:.3f}'.format(alpha))
print(chi2_significance)
 print('{:=<70}'.format(''))<compute_train_metric>
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test_df.sample(25 )
<compute_train_metric><EOS> 
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )