Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<load_from_csv>
Image(".. /input/cycliclearningrate-1/keras_clr_triangular.png" )
train_df_raw = pd.read_csv(path_train)
 train_df_raw.head()<count_missing_values>
print('TF version',tf.__version__ )
draw_missing_data_table(train_df_raw )<data_type_conversions>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train_neg = train[train.sentiment == "negative"].reset_index()
def preprocess_data(df):
processed_df = df
processed_df['Embarked'].fillna('C', inplace=True)
 processed_df['Age'] = processed_df.groupby(['Pclass','Sex','Parch','SibSp'])['Age'].transform(lambda x: x.fillna(x.mean()))
processed_df['Age'] = processed_df.groupby(['Pclass','Sex','Parch'])['Age'].transform(lambda x: x.fillna(x.mean()))
processed_df['Age'] = processed_df.groupby(['Pclass','Sex'])['Age'].transform(lambda x: x.fillna(x.mean()))
processed_df['Fare'] = processed_df['Fare'].interpolate()
processed_df['Cabin'].fillna('U', inplace=True)
 processed_df['Title'] = pd.Series(( name.split('.')[0].split(',')[1].strip() for name in train_df_raw['Name']), index=train_df_raw.index)
 processed_df['Title'] = processed_df['Title'].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 processed_df['Title'] = processed_df['Title'].replace(['Mlle', 'Ms'], 'Miss')
 processed_df['Title'] = processed_df['Title'].replace('Mme', 'Mrs')
 processed_df['Title'] = processed_df['Title'].map({"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5})
 processed_df['Age'] = processed_df.groupby(['Title'])['Age'].transform(lambda x: x.fillna(x.mean()))
processed_df['Sex'] = processed_df['Sex'].map({'male': 0, 'female': 1})
 processed_df['Embarked'] = processed_df['Embarked'].map({'S': 0, 'C': 1, 'Q': 2})
 processed_df['FamillySize'] = processed_df['SibSp'] + processed_df['Parch'] + 1
processed_df['FamillySize'][processed_df['FamillySize'].between(1, 5, inclusive=False)] = 2
processed_df['FamillySize'][processed_df['FamillySize']>5] = 3
processed_df['IsAlone'] = np.where(processed_df['FamillySize']!=1, 0, 1)
 processed_df['IsChild'] = processed_df['Age'] < 18
processed_df['IsChild'] = processed_df['IsChild'].astype(int)
 processed_df['Cabin'] = processed_df['Cabin'].str[:1]
processed_df['Cabin'] = processed_df['Cabin'].map({cabin: p for p, cabin in enumerate(set(cab for cab in processed_df['Cabin'])) })
 processed_df['TicketSurvivor'] = pd.Series(0, index=processed_df.index)
 tickets = processed_df['Ticket'].value_counts().to_dict()
for t, occ in tickets.items() :
if occ != 1:
table = train_df_raw['Survived'][train_df_raw['Ticket'] == t]
if sum(table)!= 0:
processed_df['TicketSurvivor'][processed_df['Ticket'] == t] = 1
processed_df = processed_df.drop(['Name', 'Ticket', 'PassengerId'], 1)
 return processed_df<split>
e = tokenizer.encode("going")
 print(e.ids )
train_df = train_df_raw.copy()
X = train_df.drop(['Survived'], 1)
 Y = train_df['Survived']
X = preprocess_data(X)
 sc = StandardScaler()
X = pd.DataFrame(sc.fit_transform(X.values), index=X.index, columns=X.columns)
 X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)
 X_train.head()<compute_train_metric>
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(ct):
text1 = " "+" ".join(train.loc[k,'text'].split() ).strip()
text2 = " ".join(train.loc[k,'selected_text'].split() ).strip()
text1 = text1.replace("_","")
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 enc2 = tokenizer.encode(text2)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 print(offsets)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
lg = LogisticRegression(solver='lbfgs', random_state=42)
 lg.fit(X_train, Y_train)
 logistic_prediction = lg.predict(X_test)
 score = metrics.accuracy_score(Y_test, logistic_prediction)
 display_confusion_matrix(Y_test, logistic_prediction, score=score )<compute_train_metric>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 text1 = text1.replace("_","")
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
dt = DecisionTreeClassifier(min_samples_split=15, min_samples_leaf=20, random_state=42)
 dt.fit(X_train, Y_train)
 dt_prediction = dt.predict(X_test)
 score = metrics.accuracy_score(Y_test, dt_prediction)
 display_confusion_matrix(Y_test, dt_prediction, score=score )<compute_train_metric>
def build_model() :

 ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x_bert = bert_model([ids,att,tok])
 logits = tf.keras.layers.Dense(2, activation="linear", kernel_initializer="he_normal" )(x_bert[0])
 print(x_bert[0].shape)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, axis=-1)
 end_logits = tf.squeeze(end_logits, axis=-1)
 start_scores = tf.keras.layers.Activation('softmax' )(start_logits)
 end_scores = tf.keras.layers.Activation('softmax' )(end_logits)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[start_scores,end_scores])
 optimizer = tf.keras.optimizers.Adam(learning_rate=0.00003)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer,metrics=['accuracy'])
 return model
svm = SVC(gamma='auto', random_state=42)
 svm.fit(X_train, Y_train)
 svm_prediction = svm.predict(X_test)
 score = metrics.accuracy_score(Y_test, svm_prediction)
 display_confusion_matrix(Y_test, svm_prediction, score=score )<compute_train_metric>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
self.scale_mode = 'cycle'
print(self.scale_mode)
 if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):
if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
print("Setting the lr to {}".format(self.base_lr))
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
rf = RandomForestClassifier(n_estimators=200, random_state=42)
 rf.fit(X_train, Y_train)
 rf_prediction = rf.predict(X_test)
 score = metrics.accuracy_score(Y_test, rf_prediction)
 display_confusion_matrix(Y_test, rf_prediction, score=score )<choose_model_class>
clr = CyclicLR(
 	mode='triangular',
	base_lr= 1e-6,
	max_lr= 4e-5,
	step_size= 320,
scale_mode='cycle' )
def build_ann(optimizer='adam'):
ann = Sequential()
ann.add(Dense(units=32, kernel_initializer='glorot_uniform', activation='relu', input_shape=(13,)))
 ann.add(Dense(units=64, kernel_initializer='glorot_uniform', activation='relu'))
ann.add(Dropout(rate=0.5))
ann.add(Dense(units=64, kernel_initializer='glorot_uniform', activation='relu'))
ann.add(Dropout(rate=0.5))
ann.add(Dense(units=1, kernel_initializer='glorot_uniform', activation='sigmoid'))
ann.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
 return ann<train_model>
jac = []; VER='v4'; DISPLAY=1
preds_start = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids.shape[0],MAX_LEN))
def scheduler(epoch):
if epoch <1:
return 0.00003
else:
return 0.00003 * tf.math.exp(-0.1 *(10 - epoch))
reduce_lr = tf.keras.callbacks.LearningRateScheduler(scheduler,verbose=1)
 n_splits = 5
skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=42)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print(idxT,idxV)
 print("Training for fold {}".format(fold))
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 hist = model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=32, verbose=DISPLAY, callbacks=[sv,reduce_lr],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
model.load_weights('%s-roberta-%i.h5'%(VER,fold))
preds_start[idxV,],preds_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=1)
 all = []
for k in idxV:
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
opt = optimizers.Adam(lr=0.001)
 ann = build_ann(opt)
 history = ann.fit(X_train, Y_train, batch_size=16, epochs=30, validation_data=(X_test, Y_test))<compute_train_metric>
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(n_splits):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('v4-robertafulldata-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits
ann_prediction = ann.predict(X_test)
 ann_prediction =(ann_prediction > 0.5)
 score = metrics.accuracy_score(Y_test, ann_prediction)
 display_confusion_matrix(Y_test, ann_prediction, score=score )<compute_train_metric>
all = []
jaccard_score = 0
for k in range(input_ids_t.shape[0]):
score_array = np.zeros(( MAX_LEN,MAX_LEN),dtype='float32')
 for i in range(MAX_LEN):
score_array[i,:] = preds_start[k,i] + preds_end[k,]
result = np.argmax(score_array)
(a,b)= np.unravel_index(result, score_array.shape)
 if a>b:
print("Result {}".format(a))
print("Truth {}".format(np.argmax(start_tokens[k,])))
 st = test.loc[k,'text']
else:
if test.loc[k, 'sentiment'] == "neutral":
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )
<create_dataframe><EOS> 
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.head(25 )