Kaggle competition: Tweet Sentiment Extraction
all_data = pd.concat(( train,test)).reset_index(drop = True)
 all_data.isnull().sum().sort_values(ascending = False)[all_data.isnull().sum().sort_values(ascending = False)!= 0]<define_variables>
import re
import string
import os
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
from nltk.corpus import stopwords
from tqdm import tqdm
import spacy
import random
from spacy.util import compounding
from spacy.util import minibatch
miss_val_1 = ["Alley",
"PoolQC",
"MiscFeature",
"Fence",
"FireplaceQu",
"GarageType",
"GarageFinish",
"GarageQual",
"GarageCond",
'BsmtQual',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'MasVnrType']
miss_val_2=['BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'TotalBsmtSF',
'BsmtFullBath',
'BsmtHalfBath',
'GarageYrBlt',
'GarageArea',
'GarageCars',
'MasVnrArea']
for i in miss_val_1:
all_data[i]=all_data[i].fillna('None')
 for i in miss_val_2:
all_data[i]=all_data[i].fillna(0 )<sort_values>
print('TF version',tf.__version__ )
all_data.isnull().sum().sort_values(ascending = False)[all_data.isnull().sum().sort_values(ascending = False)!= 0]<data_type_conversions>
def load_data_of_train() :
train=pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 train['text'] = train['text'].astype(str)
 train['selected_text'] = train['selected_text'].astype(str)
 return train
def load_data_of_test() :
test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 return test
def load_data_of_submission() :
sub = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv')
 return sub
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.mean()))
all_data['MSSubClass'] = all_data['MSSubClass'].astype(str)
 all_data['MSZoning'] = all_data.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str)
 all_data['Functional'] = all_data['Functional'].fillna('Typ')
 all_data['Utilities'] = all_data['Utilities'].fillna('AllPub')
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna("TA")
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['Electrical'] = all_data['Electrical'].fillna("SBrkr" )<sort_values>
train_data= load_data_of_train()
train_data.head()
all_data.isnull().sum().sort_values(ascending = False)[all_data.isnull().sum().sort_values(ascending = False)!= 0]<filter>
train_data.sentiment.value_counts()
numerical = all_data.dtypes[all_data.dtypes != "object"].index<feature_engineering>
train_data.dropna(inplace=True )
skew = all_data[numerical].apply(lambda x: skew(x)).sort_values(ascending=False)
 skew_index = skew[abs(skew)>0.5].index
for feat in skew_index:
all_data[feat] = boxcox1p(all_data[feat], boxcox_normmax(all_data[feat] + 1))<categorify>
test_data = load_data_of_test()
sample_submission_data = load_data_of_submission()
cat = all_data.dtypes[all_data.dtypes == "object"].index
for i in cat:
all_data[i].dtype
le = LabelEncoder()
all_data[i] = le.fit_transform(all_data[i])
 all_data[i].dtype<prepare_x_and_y>
def jaccard(text1, text2):
a = set(text1.split())
 b = set(text2.split())
 intresection = a.intersection(b)
 IOU =(float )(len(intresection)) /(len(a)+len(b)-len(intresection))
return IOU
X = all_data.iloc[:len(y), :]
X_sub = all_data.iloc[len(y):, :]<compute_train_metric>
jaccard_list=[]
def calc_jaccard() :
for row in train_data.itertuples() :
jaccard_list.append(jaccard(row.text, row.selected_text))
return jaccard_list;
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )<define_search_space>
jac = calc_jaccard()
train_data['jaccard'] = jac
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003,))
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
<compute_test_metric>
train_data['no_words'] = train_data['text'].apply(lambda x:len(str(x ).split()))
lessThanThree = train_data[train_data['no_words']<=2]
score = cv_rmse(ridge)
 print("Ridge: {:.4f}
".format(score.mean()))
score = cv_rmse(lasso)
 print("LASSO: {:.4f}
".format(score.mean()))
score = cv_rmse(elasticnet)
 print("elastic net: {:.4f}
".format(score.mean()))
score = cv_rmse(svr)
 print("SVR: {:.4f}
".format(score.mean()))
score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}
".format(score.mean()))
score = cv_rmse(xgboost)
 print("xgboost: {:.4f}
".format(score.mean()))<train_model>
lessThanThree[lessThanThree['sentiment']=='negative']
elastic = elasticnet.fit(X, y)
 lasso = lasso.fit(X, y)
 ridge = ridge.fit(X, y)
 svr = svr.fit(X, y)
 xgb = xgboost.fit(X, y)
 lgb = lightgbm.fit(X, y )<predict_on_test>
final_train=load_data_of_train()
final_test=load_data_of_test()
final_sample_submission=load_data_of_submission()
final_train
def models_predict(X):
return(( 0.15 * elastic.predict(X)) + \ 
(0.15 * lasso.predict(X)) + \ 
(0.15 * ridge.predict(X)) + \ 
(0.15 * svr.predict(X)) + \ 
(0.20 * xgb.predict(X)) + \ 
(0.20 * lgb.predict(X)) )<save_to_csv>
final_train['no_words'] = final_train['text'].apply(lambda x:len(str(x ).split()))
final_train = final_train[final_train['no_words']>=3]
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.expm1(models_predict(X_sub))
submission.to_csv("submission111.csv", index=False )<load_from_csv>
MAX_LEN = 96
PATH = ''
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/tf-roberta/vocab-roberta-base.json',
merges_file='.. /input/tf-roberta/merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
<load_from_csv>
def definitions(Count,flag):
d = dict()
d['input_ids'] = np.ones(( Count,MAX_LEN),dtype='int32')
 d['attention_mask'] = np.zeros(( Count,MAX_LEN),dtype='int32')
 d['token_type_ids'] = np.zeros(( Count,MAX_LEN),dtype='int32')
 if(flag):
d['start_tokens'] = np.zeros(( Count,MAX_LEN),dtype='int32')
 d['end_tokens'] = np.zeros(( Count,MAX_LEN),dtype='int32')
 return d
train = pd.read_csv('.. /input/covid19-global-forecasting-week-3/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-3/test.csv')
 submission = pd.read_csv('.. /input/covid19-global-forecasting-week-3/submission.csv')
 clean_train = pd.read_csv('.. /input/clean-data/clean_train.csv')
 clean_test = pd.read_csv('.. /input/clean-data/clean_test.csv' )<feature_engineering>
count_row = final_train.shape[0]
data_definitions = definitions(count_row, True)
 input_ids = data_definitions['input_ids']
attention_mask = data_definitions['attention_mask']
token_type_ids = data_definitions['token_type_ids']
start_tokens = data_definitions['start_tokens']
end_tokens = data_definitions['end_tokens']
tr_p = train[train['Province_State'].notnull() ]
tr_p['Country_Province'] = tr_p['Country_Region'] + '_' + tr_p['Province_State']
tr_np = train[train['Province_State'].isnull() ]
tr_np['Country_Province'] = tr_np['Country_Region']<concatenate>
iterate=0
for k,col in final_train.iterrows() :
text1 = " "+" ".join(col['text'].split())
 text2 = " ".join(col['selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[final_train.loc[k]['sentiment']]
input_ids[iterate][:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[iterate][:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[iterate][toks[0]+1] = 1
end_tokens[iterate][toks[-1]+1] = 1
iterate = iterate + 1
train1 = pd.concat([tr_np,tr_p])
 train1.drop(['Province_State','Country_Region'], axis=1,inplace = True )<feature_engineering>
count_row = final_test.shape[0]
data_definitions = definitions(count_row,False)
 input_ids_t = data_definitions['input_ids']
token_type_ids_t = data_definitions['token_type_ids']
attention_mask_t = data_definitions['attention_mask']
for k,col in final_test.iterrows() :
text1 = " "+" ".join(col['text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[col['sentiment']]
input_ids_t[k][:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k][:len(enc.ids)+5] = 1
te_p = test[test['Province_State'].notnull() ]
te_p['Country_Province'] = te_p['Country_Region'] + '_' + te_p['Province_State']
te_np = test[test['Province_State'].isnull() ]
te_np['Country_Province'] = te_np['Country_Region']<concatenate>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained('.. /input/tf-roberta/config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained('.. /input/tf-roberta/pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model
test1 = pd.concat([te_np,te_p])
 test1.drop(['Province_State','Country_Region'], axis=1,inplace = True )<data_type_conversions>
%%time
n_splits = 5
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('/kaggle/input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits
train1['Date'] = pd.to_datetime(train1['Date'])
 test1['Date'] = pd.to_datetime(test1['Date'] )<data_type_conversions>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = final_test.loc[k,'text']
else:
text1 = " "+" ".join(final_test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )
train1['Country_Province'] = train1['Country_Province'].astype('category')
 test1['Country_Province'] = test1['Country_Province'].astype('category' )<drop_column>
final_test['selected_text'] = all
final_test[['textID','selected_text']].to_csv('submission.csv',index=False )