Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<sort_values>
train_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 test_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )
pd.DataFrame(abs(train.corr() ['Survived'] ).sort_values(ascending = False))<compute_train_metric>
print(train_data.notnull().sum())
 print(train_data.isnull().sum() )
male_mean = train[train['Sex'] == 1].Survived.mean()
female_mean = train[train['Sex'] == 0].Survived.mean()
print("Male survival mean: " + str(male_mean))
print("female survival mean: " + str(female_mean))
print("The mean difference between male and female survival rate: " + str(female_mean - male_mean))<feature_engineering>
train_data.dropna(axis = 0,inplace=True )
male = train[train['Sex'] == 1]
female = train[train['Sex'] == 0]
m_mean_samples = []
f_mean_samples = []
for i in range(50):
m_mean_samples.append(np.mean(random.sample(list(male['Survived']),50,)))
 f_mean_samples.append(np.mean(random.sample(list(female['Survived']),50,)))
 print(f"Male mean sample mean: {round(np.mean(m_mean_samples),2)}")
 print(f"Male mean sample mean: {round(np.mean(f_mean_samples),2)}")
 print(f"Difference between male and female mean sample mean: {round(np.mean(f_mean_samples)- np.mean(m_mean_samples),2)}" )<categorify>
print(test_data.notnull().sum())
 print(test_data.isnull().sum() )
train['name_length'] = [len(i)for i in train.Name]
test['name_length'] = [len(i)for i in test.Name]
def name_length_group(size):
a = ''
if(size <=20):
a = 'short'
elif(size <=35):
a = 'medium'
elif(size <=45):
a = 'good'
else:
a = 'long'
return a
train['nLength_group'] = train['name_length'].map(name_length_group)
 test['nLength_group'] = test['name_length'].map(name_length_group)
<feature_engineering>
def remove_punctuation(text):
no_punct = "".join([c for c in text if c not in string.punctuation])
 return no_punct
train_data['s_text_clean'] = train_data['selected_text'].apply(str ).apply(lambda x: remove_punctuation(x.lower()))
train_data.head(20 )
train["title"] = [i.split('.')[0] for i in train.Name]
train["title"] = [i.split(',')[1] for i in train.title] 
<feature_engineering>
tokenizer = RegexpTokenizer(r'\w+')
 train_data['s_text_tokens'] = train_data['s_text_clean'].apply(str ).apply(lambda x: tokenizer.tokenize(x))
train_data.head(20 )
test['title'] = [i.split('.')[0].split(',')[1].strip() for i in test.Name] 
<feature_engineering>
def remove_stopwords(text):
words = [w for w in text if(w not in stopwords.words('english')or w not in 'im')]
return words
train_data['s_text_tokens_NOTstop'] = train_data['s_text_tokens'].apply(lambda x: remove_stopwords(x))
train_data.head(20 )
train["title"] = [i.replace('Ms', 'Miss')for i in train.title]
train["title"] = [i.replace('Mlle', 'Miss')for i in train.title]
train["title"] = [i.replace('Mme', 'Mrs')for i in train.title]
train["title"] = [i.replace('Dr', 'rare')for i in train.title]
train["title"] = [i.replace('Col', 'rare')for i in train.title]
train["title"] = [i.replace('Major', 'rare')for i in train.title]
train["title"] = [i.replace('Don', 'rare')for i in train.title]
train["title"] = [i.replace('Jonkheer', 'rare')for i in train.title]
train["title"] = [i.replace('Sir', 'rare')for i in train.title]
train["title"] = [i.replace('Lady', 'rare')for i in train.title]
train["title"] = [i.replace('Capt', 'rare')for i in train.title]
train["title"] = [i.replace('the Countess', 'rare')for i in train.title]
train["title"] = [i.replace('Rev', 'rare')for i in train.title] 
<feature_engineering>
lemmatizer = WordNetLemmatizer()
def word_lemmatizer(text):
lem_text = [lemmatizer.lemmatize(i)for i in text]
return lem_text
train_data['s_text_lemma'] = train_data['s_text_tokens_NOTstop'].apply(lambda x: word_lemmatizer(x))
train_data.head(20 )
train['family_size'] = train.SibSp + train.Parch+1
test['family_size'] = test.SibSp + test.Parch+1<categorify>
stemmer = PorterStemmer()
def word_stemmer(text):
stem_text = " ".join([stemmer.stem(i)for i in text])
 return stem_text
train_data['s_text_stem'] = train_data['s_text_lemma'].apply(lambda x: word_stemmer(x))
train_data.head(20 )
train['family_group'] = train['family_size'].map(family_group)
 test['family_group'] = test['family_size'].map(family_group )<feature_engineering>
positive_train = train_data[train_data['sentiment']=='positive']
neutral_train = train_data[train_data['sentiment']=='neutral']
negative_train = train_data[train_data['sentiment']=='negative']
train['is_alone'] = [1 if i<2 else 0 for i in train.family_size]
test['is_alone'] = [1 if i<2 else 0 for i in test.family_size]<count_values>
max_len = 128
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file = '/kaggle/input/roberta-base/vocab.json',
merges_file = '/kaggle/input/roberta-base/merges.txt',
lowercase =True,
add_prefix_space=True 
)
 sentiment_id = {'positive':tokenizer.encode('positive' ).ids[0],
'negative':tokenizer.encode('negative' ).ids[0],
'neutral':tokenizer.encode('neutral' ).ids[0]}
train_data.reset_index(inplace=True)
 tot_tw = train_data.shape[0]
input_ids = np.ones(( tot_tw, max_len), dtype='int32')
 attention_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 token_type_ids = np.zeros(( tot_tw, max_len), dtype='int32')
 start_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 end_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 for i in range(tot_tw):
set1 = " "+" ".join(train_data.loc[i,'text'].split())
 set2 = " ".join(train_data.loc[i,'selected_text'].split())
 idx = set1.find(set2)
 set2_loc = np.zeros(( len(set1)))
 set2_loc[idx:idx+len(set2)]=1
if set1[idx-1]==" ":
set2_loc[idx-1]=1
enc_set1 = tokenizer.encode(set1)
 selected_text_token_idx=[]
for k,(a,b)in enumerate(enc_set1.offsets):
sm = np.sum(set2_loc[a:b])
 if sm > 0:
selected_text_token_idx.append(k)
 senti_token = sentiment_id[train_data.loc[i,'sentiment']]
input_ids[i,:len(enc_set1.ids)+5] = [0]+enc_set1.ids+[2,2]+[senti_token]+[2]
attention_mask[i,:len(enc_set1.ids)+5]=1
if len(selected_text_token_idx)> 0:
start_mask[i,selected_text_token_idx[0]+1]=1
end_mask[i, selected_text_token_idx[-1]+1]=1
train.Ticket.value_counts().sample(10 )<drop_column>
def custom_loss(y_true, y_pred):
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, from_logits = False, label_smoothing = 0.20)
 loss = tf.reduce_mean(loss)
 return loss
train.drop(['Ticket'], axis=1, inplace=True)
 test.drop(['Ticket'], axis=1, inplace=True )<feature_engineering>
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 config_path = RobertaConfig.from_pretrained('/kaggle/input/tf-roberta/config-roberta-base.json')
 roberta_model = TFRobertaModel.from_pretrained('/kaggle/input/tf-roberta/pretrained-roberta-base.h5', config=config_path)
 x = roberta_model(ids, attention_mask = att, token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.05 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.05 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=custom_loss, optimizer=optimizer)
 return model
train['calculated_fare'] = train.Fare/train.family_size
test['calculated_fare'] = test.Fare/test.family_size<categorify>
tot_test_tw = test_data.shape[0]
input_ids_t = np.ones(( tot_test_tw,max_len), dtype='int32')
 attention_mask_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 token_type_ids_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 for i in range(tot_test_tw):
set1 = " "+" ".join(test_data.loc[i,'text'].split())
 enc_set1 = tokenizer.encode(set1)
 s_token = sentiment_id[test_data.loc[i,'sentiment']]
input_ids_t[i,:len(enc_set1.ids)+5]=[0]+enc_set1.ids+[2,2]+[s_token]+[2]
attention_mask_t[i,:len(enc_set1.ids)+5]=1
def fare_group(fare):

 a= ''
if fare <= 4:
a = 'Very_low'
elif fare <= 10:
a = 'low'
elif fare <= 20:
a = 'mid'
elif fare <= 45:
a = 'high'
else:
a = "very_high"
return a
train['fare_group'] = train['calculated_fare'].map(fare_group)
 test['fare_group'] = test['calculated_fare'].map(fare_group)
<drop_column>
pred_start= np.zeros(( input_ids_t.shape[0],max_len))
pred_end= np.zeros(( input_ids_t.shape[0],max_len))
for i in range(5):
print('--'*20)
 print('-- MODEL %i --'%(i+1))
print('--'*20)
 K.clear_session()
model = build_model()
model.load_weights('/kaggle/input/model4/v4-roberta-%i.h5'%(i))
pred = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=1)
 pred_start = pred_start +(pred[0]/5)
 pred_end = pred_end +(pred[1]/5 )
train.drop(['PassengerId'], axis=1, inplace=True)
 test.drop(['PassengerId'], axis=1, inplace=True )<categorify>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(pred_start[k,])
 b = np.argmax(pred_end[k,])
 if a>b:
st = test_data.loc[k,'text']
else:
text1 = " "+" ".join(test_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 test_data['selected_text']=all
test_data.head(20 )
train = pd.get_dummies(train, columns=['title',"Pclass", 'Cabin','Embarked','nLength_group', 'family_group', 'fare_group'], drop_first=False)
 test = pd.get_dummies(test, columns=['title',"Pclass",'Cabin','Embarked','nLength_group', 'family_group', 'fare_group'], drop_first=False)
 train.drop(['family_size','Name', 'Fare','name_length'], axis=1, inplace=True)
 test.drop(['Name','family_size',"Fare",'name_length'], axis=1, inplace=True )<predict_on_test>
test_data[['textID','selected_text']].to_csv('submission.csv', index=False)
 print("Submission successful" )
train = pd.concat([train[["Survived", "Age", "Sex","SibSp","Parch"]], train.loc[:,"is_alone":]], axis=1)
 test = pd.concat([test[["Age", "Sex"]], test.loc[:,"SibSp":]], axis=1 )<choose_model_class>
train_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 test_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )
def completing_age(df):
age_df = df.loc[:,"Age":]
temp_train = age_df.loc[age_df.Age.notnull() ]
temp_test = age_df.loc[age_df.Age.isnull() ]
y = temp_train.Age.values
x = temp_train.loc[:, "Sex":].values
rfr = RandomForestRegressor(n_estimators=1500, n_jobs=-1)
 rfr.fit(x, y)
 predicted_age = rfr.predict(temp_test.loc[:, "Sex":])
 df.loc[df.Age.isnull() , "Age"] = predicted_age
return df
completing_age(train)
 completing_age(test);<define_variables>
print(train_data.notnull().sum())
 print(train_data.isnull().sum() )
def age_group_fun(age):

 a = ''
if age <= 1:
a = 'infant'
elif age <= 4:
a = 'toddler'
elif age <= 13:
a = 'child'
elif age <= 18:
a = 'teenager'
elif age <= 35:
a = 'Young_Adult'
elif age <= 45:
a = 'adult'
elif age <= 55:
a = 'middle_aged'
elif age <= 65:
a = 'senior_citizen'
else:
a = 'old'
return a
train['age_group'] = train['Age'].map(age_group_fun)
 test['age_group'] = test['Age'].map(age_group_fun)
 train = pd.get_dummies(train,columns=['age_group'], drop_first=True)
 test = pd.get_dummies(test,columns=['age_group'], drop_first=True);<prepare_x_and_y>
train_data.dropna(axis = 0,inplace=True )
X = train.drop(['Survived'], axis = 1)
 y = train["Survived"]<split>
print(test_data.notnull().sum())
 print(test_data.isnull().sum() )
X_train, X_test, y_train, y_test = train_test_split(X, y,test_size =.33, random_state=0 )<normalization>
def remove_punctuation(text):
no_punct = "".join([c for c in text if c not in string.punctuation])
 return no_punct
train_data['s_text_clean'] = train_data['selected_text'].apply(str ).apply(lambda x: remove_punctuation(x.lower()))
train_data.head(20 )
std_scale = StandardScaler()
X_train = std_scale.fit_transform(X_train)
 X_test = std_scale.transform(X_test)
<train_model>
tokenizer = RegexpTokenizer(r'\w+')
 train_data['s_text_tokens'] = train_data['s_text_clean'].apply(str ).apply(lambda x: tokenizer.tokenize(x))
train_data.head(20 )
logreg = LogisticRegression(solver='liblinear',
penalty= 'l1',random_state = 42 
)
 logreg.fit(X_train,y_train)
 y_pred = logreg.predict(X_test)
<create_dataframe>
def remove_stopwords(text):
words = [w for w in text if(w not in stopwords.words('english')or w not in 'im')]
return words
train_data['s_text_tokens_NOTstop'] = train_data['s_text_tokens'].apply(lambda x: remove_stopwords(x))
train_data.head(20 )
pd.DataFrame(confusion_matrix(y_test,y_pred),\
columns=["Predicted Not-Survived", "Predicted Survived"],\
index=["Not-Survived","Survived"] )<compute_train_metric>
lemmatizer = WordNetLemmatizer()
def word_lemmatizer(text):
lem_text = [lemmatizer.lemmatize(i)for i in text]
return lem_text
train_data['s_text_lemma'] = train_data['s_text_tokens_NOTstop'].apply(lambda x: word_lemmatizer(x))
train_data.head(20 )
y_pred = logreg.predict(X_test)
 cm = confusion_matrix(y_test, y_pred)
 cm_display = ConfusionMatrixDisplay(cm ).plot()<compute_test_metric>
stemmer = PorterStemmer()
def word_stemmer(text):
stem_text = " ".join([stemmer.stem(i)for i in text])
 return stem_text
train_data['s_text_stem'] = train_data['s_text_lemma'].apply(lambda x: word_stemmer(x))
train_data.head(20 )
accuracy_score(y_test, y_pred )<compute_test_metric>
positive_train = train_data[train_data['sentiment']=='positive']
neutral_train = train_data[train_data['sentiment']=='neutral']
negative_train = train_data[train_data['sentiment']=='negative']
recall_score(y_test, y_pred )<compute_test_metric>
max_len = 128
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file = '/kaggle/input/roberta-base/vocab.json',
merges_file = '/kaggle/input/roberta-base/merges.txt',
lowercase =True,
add_prefix_space=True 
)
 sentiment_id = {'positive':tokenizer.encode('positive' ).ids[0],
'negative':tokenizer.encode('negative' ).ids[0],
'neutral':tokenizer.encode('neutral' ).ids[0]}
train_data.reset_index(inplace=True)
 tot_tw = train_data.shape[0]
input_ids = np.ones(( tot_tw, max_len), dtype='int32')
 attention_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 token_type_ids = np.zeros(( tot_tw, max_len), dtype='int32')
 start_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 end_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 for i in range(tot_tw):
set1 = " "+" ".join(train_data.loc[i,'text'].split())
 set2 = " ".join(train_data.loc[i,'selected_text'].split())
 idx = set1.find(set2)
 set2_loc = np.zeros(( len(set1)))
 set2_loc[idx:idx+len(set2)]=1
if set1[idx-1]==" ":
set2_loc[idx-1]=1
enc_set1 = tokenizer.encode(set1)
 selected_text_token_idx=[]
for k,(a,b)in enumerate(enc_set1.offsets):
sm = np.sum(set2_loc[a:b])
 if sm > 0:
selected_text_token_idx.append(k)
 senti_token = sentiment_id[train_data.loc[i,'sentiment']]
input_ids[i,:len(enc_set1.ids)+5] = [0]+enc_set1.ids+[2,2]+[senti_token]+[2]
attention_mask[i,:len(enc_set1.ids)+5]=1
if len(selected_text_token_idx)> 0:
start_mask[i,selected_text_token_idx[0]+1]=1
end_mask[i, selected_text_token_idx[-1]+1]=1
precision_score(y_test, y_pred )<compute_test_metric>
def custom_loss(y_true, y_pred):
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, from_logits = False, label_smoothing = 0.20)
 loss = tf.reduce_mean(loss)
 return loss
print(classification_report(y_test, y_pred))<compute_train_metric>
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 config_path = RobertaConfig.from_pretrained('/kaggle/input/tf-roberta/config-roberta-base.json')
 roberta_model = TFRobertaModel.from_pretrained('/kaggle/input/tf-roberta/pretrained-roberta-base.h5', config=config_path)
 x = roberta_model(ids, attention_mask = att, token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.05 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.05 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=custom_loss, optimizer=optimizer)
 return model
cv = StratifiedShuffleSplit(n_splits = 10, test_size =.25, random_state = 0)
 column_names = X.columns
X = std_scale.fit_transform(X)
 accuracies = cross_val_score(LogisticRegression(solver='liblinear'), X,y, cv = cv)
 print("Cross-Validation accuracy scores:{}".format(accuracies))
print("Mean Cross-Validation accuracy score: {}".format(round(accuracies.mean() ,5)) )<choose_model_class>
tot_test_tw = test_data.shape[0]
input_ids_t = np.ones(( tot_test_tw,max_len), dtype='int32')
 attention_mask_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 token_type_ids_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 for i in range(tot_test_tw):
set1 = " "+" ".join(test_data.loc[i,'text'].split())
 enc_set1 = tokenizer.encode(set1)
 s_token = sentiment_id[test_data.loc[i,'sentiment']]
input_ids_t[i,:len(enc_set1.ids)+5]=[0]+enc_set1.ids+[2,2]+[s_token]+[2]
attention_mask_t[i,:len(enc_set1.ids)+5]=1
C_vals = [0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]
cv = StratifiedShuffleSplit(n_splits = 10, test_size =.25)
 param = {'C': C_vals}
logreg = LogisticRegression()
grid = GridSearchCV(
 estimator=LogisticRegression() ,
param_grid = param,
scoring = 'accuracy',
n_jobs =-1,
cv = cv 
)
 grid.fit(X, y )<find_best_params>
pred_start= np.zeros(( input_ids_t.shape[0],max_len))
pred_end= np.zeros(( input_ids_t.shape[0],max_len))
for i in range(5):
print('--'*20)
 print('-- MODEL %i --'%(i+1))
print('--'*20)
 K.clear_session()
model = build_model()
model.load_weights('/kaggle/input/model4/v4-roberta-%i.h5'%(i))
pred = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=1)
 pred_start = pred_start +(pred[0]/5)
 pred_end = pred_end +(pred[1]/5 )
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_)
<compute_train_metric>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(pred_start[k,])
 b = np.argmax(pred_end[k,])
 if a>b:
st = test_data.loc[k,'text']
else:
text1 = " "+" ".join(test_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 test_data['selected_text']=all
test_data.head(20 )
<compute_train_metric><EOS> 
test_data[['textID','selected_text']].to_csv('submission.csv', index=False)
 print("Submission successful" )