Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<compute_test_metric>
print('TF version',tf.__version__ )
reg_results = []
for regre in reg :
reg_results.append(np.mean(np.sqrt(-cross_val_score(regre, X_rbst, y = target,scoring = 'neg_mean_squared_error',
cv = kfold, n_jobs=-4))))<create_dataframe>
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()
reg_re = pd.DataFrame({"CrossValMeans":reg_means,"CrossValerrors": reg_std})
 reg_re<train_on_grid>
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
GBC = GradientBoostingRegressor()
gb_param_grid = {'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="neg_mean_squared_error", n_jobs= 4, verbose = 1)
 gsGBC.fit(X_rbst,target)
 GBC_best = gsGBC.best_estimator_
gsGBC.best_score_<define_search_space>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
<train_on_grid>
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
XGB = XGBRegressor()
xgb_param_grid = {'learning_rate': [1,0.1,0.01,0.001],
'n_estimators': [50, 100, 200, 500, 1000],
'max_depth' : [1,3,5,10,50]}
gsXGB = GridSearchCV(XGB,param_grid = xgb_param_grid, cv=kfold, scoring="neg_mean_squared_error", n_jobs= 4, verbose = 1)
 gsXGB.fit(X_rbst,target)
 XGB_best = gsXGB.best_estimator_
gsXGB.best_score_<train_on_grid>
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
LGB = LGBMRegressor()
lgb_param_grid = {
'num_leaves' : [1,5,10],
'learning_rate': [1,0.1,0.01,0.001],
'n_estimators': [50, 100, 200, 500, 1000,5000],
'max_depth': [15,20,25],
'num_leaves': [50, 100, 200],
'min_split_gain': [0.3, 0.4],
}
gsLGB = GridSearchCV(LGB,param_grid = lgb_param_grid, cv=kfold, scoring="neg_mean_squared_error", n_jobs= 4, verbose = 1)
 gsLGB.fit(X_rbst,target)
 LGB_best = gsLGB.best_estimator_
gsLGB.best_score_<prepare_output>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.3 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.3 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model
ensemble = np.expm1(0.1*test_Survived_GBC + 0.8*test_Survived_XGB + 0.1*test_Survived_LGB)
 submission = pd.DataFrame({
"Id" :test['Id'],
"SalePrice": ensemble
})
 submission.head()<import_modules>
n_splits = 5
from sklearn.ensemble import VotingRegressor<train_model>
votingC = VotingRegressor(estimators=[('XGB', XGB_best),('LGB', LGB_best),('GBC',GBC_best)], n_jobs=4)
 votingC = votingC.fit(X_rbst, target )<predict_on_test>
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(15):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('/kaggle/input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits
test_SalePrice = pd.Series(votingC.predict(test_rbst), name="SalePrice" )<prepare_output>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_df.loc[k,'text']
else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )
submission = pd.DataFrame({
"Id" :test['Id'],
"SalePrice": np.expm1(test_SalePrice)
 })
 submission.head()<import_modules>
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )
<define_search_space><EOS> 
sub_df = pd.read_csv('submission.csv')
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.... ', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()