Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<choose_model_class>
SEED = 88888
os.environ['PYTHONHASHSEED']=str(SEED)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
random.seed(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED)
 print('TF version',tf.__version__ )
model_cat = CatBoostClassifier(random_state = 51,eval_metric = 'AUC')
 random_grid_cat = {'learning_rate': [0.05, 0.08, 0.1, 0.15, 0.2, 0.3],
'depth': [4, 6,10,15,20,30,40,50,60,70],
'l2_leaf_reg': [1, 3, 5, 7, 9]}
randomized_search_cat = model_cat.randomized_search(random_grid_cat, train_pool_cat, cv=5,plot=True)
<train_model>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 100
BATCH_SIZE = 32
PAD_ID = 1
LABEL_SMOOTHING = 0.15
sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
randomized_search_cat<train_model>
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 pre_enc = 4
post_enc = 1
tot_enc = pre_enc+post_enc
for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx =(text1 + " " ).find(" "+ text2 + " ")
 if idx < 0:
idx = text1.find(text2)
 else:
idx += 1
chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+ tot_enc] = [0,s_tok,s_tok,2] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+ tot_enc] = 1
if len(toks)>0:
start_tokens[k,toks[0]+ pre_enc ] = 1
end_tokens[k,toks[-1]+ pre_enc] = 1
model_train = CatBoostClassifier(iterations=170, learning_rate=0.1, random_state = 51,eval_metric = 'AUC',loss_function ='CrossEntropy')
 model_train.fit(train_pool_cat)
 acc_catboost = round(model_train.score(X_train_cat, y_train)* 100, 2 )<find_best_params>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
if len(enc.ids)> MAX_LEN - tot_enc:
continue
input_ids_t[k,:len(enc.ids)+ tot_enc] = [0,s_tok,s_tok,2] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+ tot_enc] = 1
cv_params = model_train.get_params()
cv_data = cv(train_pool_cat,
cv_params,
fold_count=5,
plot=True)
 acc_cv_catboost = round(np.max(cv_data['test-AUC-mean'])* 100, 2)
<compute_test_metric>
def create_optimizer(
 init_lr,
num_train_steps,
num_warmup_steps,
min_lr_ratio=0.0,
adam_epsilon=1e-8,
weight_decay_rate=0.0,
include_in_weight_decay=None, 
):

 lr_schedule = tf.keras.optimizers.schedules.PolynomialDecay(
 initial_learning_rate=init_lr,
decay_steps=num_train_steps - num_warmup_steps,
end_learning_rate=init_lr * min_lr_ratio, 
)
 if num_warmup_steps:
lr_schedule = transformers.optimization_tf.WarmUp(
 initial_learning_rate=init_lr, decay_schedule_fn=lr_schedule, warmup_steps=num_warmup_steps 
)
 if weight_decay_rate > 0.0:
AdamWeightDecay.apply_gradients = partialmethod(AdamWeightDecay.apply_gradients, clip_norm=1.0)
 optimizer = AdamWeightDecay(
 learning_rate=lr_schedule,
weight_decay_rate=weight_decay_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=adam_epsilon,
exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"],
include_in_weight_decay=include_in_weight_decay, 
)
 else:
optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule, epsilon=adam_epsilon)
 return optimizer, lr_schedule
def scheduler(epoch):
if epoch > 3:
return 3e-5 * 0.5**(3)
 return 3e-5 * 0.5**(epoch)
 def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.5 )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Dropout(0.4 )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.5 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.BatchNormalization()(x2)
 x2 = tf.keras.layers.Dropout(0.4 )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 lr_schedule = tf.keras.experimental.CosineDecayRestarts(initial_learning_rate=3e-5, \
first_decay_steps=4550, t_mul=2.0, m_mul=0.85, alpha=0.1)
 AdamWeightDecay.apply_gradients = partialmethod(AdamWeightDecay.apply_gradients, clip_norm=1.0)
 test_optimizer = AdamWeightDecay(
 learning_rate=lr_schedule,
weight_decay_rate=0.003,
include_in_weight_decay=None, 
)
 model.compile(loss=loss_fn, optimizer=test_optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model
print('train accuracy: ' , acc_catboost)
 print("CV Accuracy: " ,acc_cv_catboost)
 cv_data.head()<save_to_csv>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
y_pred=model_train.predict_proba(X_test_cat)
 pred=pd.DataFrame(y_pred[:,1])
 sub_df=pd.read_csv('.. /input/summeranalytics2020/Sample_submission.csv')
 datasets=pd.concat([sub_df['Id'],pred],axis=1)
 datasets.columns=['Id','Attrition']
datasets.to_csv('Catboost_submission_temp.csv',index=False )<choose_model_class>
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
train["predicted"] = ""
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
best_vall_loss = 10000
no_loss_improvement=0
patience = 20
reduce_lr = tf.keras.callbacks.LearningRateScheduler(scheduler,\
verbose=DISPLAY)
 early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001,
patience=1, verbose=DISPLAY, mode='auto')
 inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '.. /models/test/%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
history = model.fit(inpT, targetT,\
epochs=epoch, initial_epoch=epoch - 1, \
batch_size=BATCH_SIZE, \
verbose=DISPLAY, callbacks=[],\
validation_data=(inpV, targetV), shuffle=False)
 if(history.history['val_loss'][-1]<best_vall_loss):
no_loss_improvement = 0
print("
saving as val loss reduced:",best_vall_loss," to ",history.history['val_loss'][-1])
 best_vall_loss = history.history['val_loss'][-1]
save_weights(model, weight_fn)
 else :
no_loss_improvement +=1
if no_loss_improvement == patience:
print("
early stopping")
 break
print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],\
attention_mask[idxV,],\
token_type_ids[idxV,]],\
verbose=DISPLAY)
 all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 if(a-pre_enc < 0): a= pre_enc
if(b-pre_enc+1 < 0): b= pre_enc
if a>b:
st = tokenizer.decode(enc.ids[a-pre_enc:]+enc.ids[:b-pre_enc+1])
 else:
st = tokenizer.decode(enc.ids[a-pre_enc:b-pre_enc+1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
train.loc[k,"predicted"] = st
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits * np.mean(all)
 preds_end += preds[1]/skf.n_splits * np.mean(all)
 print()
train["pred_jac"] = train.apply(lambda x: jaccard(x["selected_text"],x["predicted"]),axis=1 )
xbg_classifier = xgboost.XGBClassifier(scoring = 'roc_auc', random_state = 51)
 booster=['gbtree']
hyperparameter_grid = {
'n_estimators' : [100, 500, 900, 1100, 1500],
'min_child_weight': [1, 2, 3, 5, 7, 9],
'gamma': [0.5, 1, 1.5, 2, 5],
'subsample': [0.6, 0.8, 1.0],
'colsample_bytree': [0.6, 0.8, 1.0],
'max_depth': [5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120],
'learning_rate' : [0.005,0.01,0.03, 0.05, 0.15,0.3, 0.45, 0.55]
}
folds = 5
skf = StratifiedKFold(n_splits=folds, shuffle = True, random_state = 51)
 random_cv = RandomizedSearchCV(estimator=xbg_classifier,
param_distributions=hyperparameter_grid,
cv=skf.split(X_train,y_train), n_iter=50,
scoring = 'roc_auc',n_jobs = 4,
verbose = 5,
return_train_score = True )<train_model>
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
print(jac )
random_cv.fit(X_train, y_train )<train_model>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 if(a-pre_enc < 0): a= pre_enc
if(b-pre_enc+1 < 0): b=pre_enc
if a>b:
st = tokenizer.decode(enc.ids[a-pre_enc:]+enc.ids[:b-pre_enc+1])
 else:
st = tokenizer.decode(enc.ids[a-pre_enc:b-pre_enc+1])
 all.append(st )
classifier_t = random_cv.best_estimator_
print(classifier_t)
 classifier_t.fit(X_train,y_train )<find_best_model_class>
test['selected_text'] = all
test['selected_text'] = test[['selected_text','text']].apply(\
lambda x: re.sub(r'\.\.+$', ".. ", x[0])\
if(len(x[0].split())==1 and len(x[1].split())!=1)\
else x , axis =1)
 test['selected_text'] = test[['selected_text','text']].apply(\
lambda x: re.sub(r'!!!!+$', "!", x[0])\
if(len(x[0].split())==1 and len(x[1].split())!=1)\
else x , axis =1)
 test[['textID','selected_text']].to_csv('submission.csv',index=False )
acc_xgb = round(classifier_t.score(X_train, y_train)* 100, 2)
 train_pred = model_selection.cross_val_predict(classifier_t,
X_train,
y_train,
cv=5,
n_jobs = -1)
 acc_cv = round(metrics.accuracy_score(y_train, train_pred)* 100, 2)
 print(acc_xgb)
 print(acc_cv )<save_to_csv>
def display_vals(x):
print("text=",x['text'])
 print("selected_text=",x['selected_text'])
 print("predicted=",x['predicted'])
 print("sentiment=",x['sentiment'])
 print("jaccard=",str(x['pred_jac']))
print("test_pred=",str(x['pred_test']))
print("pred_test_jac=",str(x['pred_test_jac']))
print("
")
 train["select_eq_text"] = train.apply(lambda x: jaccard(x["selected_text"],x["text"]),axis=1)
 train.loc[train["text"]=='',"text"]=" "
train.loc[train["selected_text"]=='',"selected_text"]=" "
<set_options><EOS> 
train["pred_test"] = train.predicted
train['pred_test'] = train[['pred_test','text']].apply(\
lambda x: re.sub(r'\.\.+$', ".. ", x[0])\
if(len(x[0].split())==1 and len(x[1].split())!=1)\
else x , axis =1)
 train['pred_test'] = train[['pred_test','text']].apply(\
lambda x: re.sub(r'!!!!+$', "!", x[0])\
if(len(x[0].split())==1 and len(x[1].split())!=1)\
else x , axis =1)
 train["pred_test_jac"] = train.apply(lambda x: jaccard(x["selected_text"],x["pred_test"]),axis=1)
 print(train.pred_test_jac.mean() ,train.pred_jac.mean())
 print("overall gain =",(train["pred_test_jac"] - train["pred_jac"] ).sum() )