Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<define_variables>
print('TF version',tf.__version__ )
string_features = all_houses.select_dtypes(include=['object'] ).columns
numeric_features = all_houses.select_dtypes(include=['int64', 'float64'] ).columns<categorify>
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()
all_houses[numeric_features] = all_houses[numeric_features].apply(normal_minmax )<filter>
train_list = train_df.to_dict('records' )
skewed_features = all_houses[numeric_features].apply(lambda x: skew(x))
skewed_features = skewed_features[abs(skewed_features)> 0.75]<feature_engineering>
MAX_SEQUENCE_LENGTH = 0
for image_data in train_list:
MAX_SEQUENCE_LENGTH = max(MAX_SEQUENCE_LENGTH, len(image_data['text'].split(' ')))
 print(MAX_SEQUENCE_LENGTH )
all_houses[skewed_features] = np.log1p(all_houses[skewed_features] )<categorify>
MAX_SEQUENCE_LENGTH = 0
for image_data in train_list:
MAX_SEQUENCE_LENGTH = max(MAX_SEQUENCE_LENGTH, len(image_data['selected_text'].split(' ')))
 print(MAX_SEQUENCE_LENGTH )
all_houses = pd.get_dummies(all_houses, columns=string_features, drop_first=True )<train_model>
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
df_houses_train = all_houses[:train_size]
df_houses_test = all_houses[train_size:]
X_train = df_houses_train
X_test = df_houses_test
print('Train:', X_train.shape)
 print('Test :', X_test.shape )<split>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
seed = 42
n_folds = 5
kfold = KFold(n_folds, shuffle=True, random_state=seed ).get_n_splits(X_train )<compute_train_metric>
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
def get_rmse(model, x, y):
rmse = np.sqrt(-cross_val_score(model,
x,
y=y,
scoring="neg_mean_squared_error",
cv=kfold))
return rmse<choose_model_class>
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
ridgeCV = RidgeCV(alphas=np.arange(0.001, 0.1, 0.01))
lassoCV = LassoCV(alphas=np.arange(0.001, 0.1, 0.01), eps=np.arange(1e-7, 1e-5, 1e-6))
lassoLarsCV = LassoLarsCV(max_iter=1000, eps=1e-7)
 elasticNetCV = ElasticNetCV(eps=1e-15)
 gBoost = GradientBoostingRegressor(random_state=seed)
 xgbReg = xgb.XGBRegressor(random_state=seed)
 lgbmReg = lgb.LGBMRegressor()
clfs = dict()
clfs['RidgeCV'] = ridgeCV
clfs['LassoCV'] = lassoCV
clfs['LassoLarsCV'] = lassoLarsCV
clfs['ElasticNetCV'] = elasticNetCV
clfs['GBoosting'] = gBoost
clfs['XGBRegressor'] = xgbReg
clfs['LGBMRegressor'] = lgbmReg<train_on_grid>
from keras import regularizers
def tuning(model, param_grid, X, y):
grid_result = GridSearchCV(model,
param_grid=param_grid,
cv=kfold,
scoring="neg_mean_squared_error",
n_jobs=-1,
verbose=1)
 grid_result.fit(X, y)
 model_best = grid_result.best_estimator_
print('Best score:', np.round(grid_result.best_score_, 4))
print('Best estimator:', model_best)
 return model_best<train_model>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same',activation='relu', kernel_regularizer=regularizers.l2(0.01))(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same',activation='relu', kernel_regularizer=regularizers.l2(0.01))(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(32, 2,padding='same',activation='relu', kernel_regularizer=regularizers.l2(0.01))(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Dropout(0.1 )(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2,padding='same',activation='relu', kernel_regularizer=regularizers.l2(0.01))(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2,padding='same',activation='relu', kernel_regularizer=regularizers.l2(0.01))(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(32, 2,padding='same',activation='relu', kernel_regularizer=regularizers.l2(0.01))(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Dropout(0.1 )(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model
parameters = {'n_estimators':[1800, 1900],
'learning_rate': [0.05],
'max_depth': [2, 3],
'min_samples_leaf': [14, 16],
'max_features': ['sqrt'],
'min_samples_split': [5, 6],
'loss': ['huber', 'ls']}
gBoost_best = tuning(clfs['GBoosting'], parameters, X_train, y_train )<train_model>
n_splits = 5
parameters = {'learning_rate': [0.01, 0.04],
'max_depth': [2, 3],
'min_child_weight': [2, 3],
'silent': [1],
'subsample': [0.7],
'colsample_bytree': [0.7],
'n_estimators': [1300, 1500]}
xgbReg_best = tuning(clfs['XGBRegressor'], parameters, X_train, y_train )<train_model>
'
parameters = {'objective': ['regression'],
'num_leaves': [4, 5],
'max_depth': [3, 4],
'bagging_freq': [5, 6],
'bagging_fraction': [0.8, 0.9],
'learning_rate': [0.01, 0.05],
'n_estimators': [700, 1000]}
lgbmReg_best = tuning(clfs['LGBMRegressor'], parameters, X_train, y_train )<prepare_output>
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/roberta-kim-cnn/v7-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits
clfs['GBoostingBest'] = gBoost_best
clfs['XGBRegressorBest'] = xgbReg_best
clfs['LGBMRegressorBest'] = lgbmReg_best
get_results(list(clfs.values()), list(clfs.keys()), X_train, y_train )<train_model>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_df.loc[k,'text']
else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
<choose_model_class><EOS> 
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False)