Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<find_best_params>
train_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 test_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )
alphas = [0.1, 0.15, 0.2, 0.005, 0.0032, 0.0030, 0.0024,]
lasso_cv = []
for alpha in alphas:
lasso = make_pipeline(StandardScaler() , RobustScaler() , Lasso(alpha = alpha))
lasso_cv.append(RMSE_CV(lasso ).mean())
 lasso_cv = pd.Series(lasso_cv, index = alphas)
 print('The minimum error is {:.4f} at {:.4f}'.format(lasso_cv.min() , lasso_cv.idxmin()))<define_search_space>
print(train_data.notnull().sum())
 print(train_data.isnull().sum() )
alphas = [0.1, 0.15, 0.2, 0.005, 0.0032, 0.0030, 0.0024, 0.0028]
ENet_cv = []
for alpha in alphas:
ENet = make_pipeline(StandardScaler() , RobustScaler() , ElasticNet(alpha = alpha, l1_ratio = 0.9))
ENet_cv.append(RMSE_CV(ENet ).mean())
 ENet_cv = pd.Series(lasso_cv, index = alphas)
 print('The minimum error is {:.4f} at {:.4f}'.format(ENet_cv.min() , ENet_cv.idxmin()))<predict_on_test>
train_data.dropna(axis = 0,inplace=True )
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_train_metric>
print(test_data.notnull().sum())
 print(test_data.isnull().sum() )
averaged_models = AveragingModels(models =(ENet, ridge, lasso))
score = RMSE_CV(averaged_models)
 print('Averaged base models score: {:.4f}'.format(score.mean()))<save_to_csv>
def remove_punctuation(text):
no_punct = "".join([c for c in text if c not in string.punctuation])
 return no_punct
train_data['s_text_clean'] = train_data['selected_text'].apply(str ).apply(lambda x: remove_punctuation(x.lower()))
train_data.head(20 )
averaged_models.fit(traindf, target)
 pred = averaged_models.predict(testdf)
 pred = np.expm1(pred)
 sub = pd.DataFrame({'Id': Id, 'SalePrice': pred})
 sub.to_csv('submission.csv',index = False )<load_from_csv>
tokenizer = RegexpTokenizer(r'\w+')
 train_data['s_text_tokens'] = train_data['s_text_clean'].apply(str ).apply(lambda x: tokenizer.tokenize(x))
train_data.head(20 )
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<import_modules>
def remove_stopwords(text):
words = [w for w in text if(w not in stopwords.words('english')or w not in 'im')]
return words
train_data['s_text_tokens_NOTstop'] = train_data['s_text_tokens'].apply(lambda x: remove_stopwords(x))
train_data.head(20 )
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.utils import np_utils
from keras.optimizers import RMSprop
from keras.callbacks import ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split<prepare_x_and_y>
lemmatizer = WordNetLemmatizer()
def word_lemmatizer(text):
lem_text = [lemmatizer.lemmatize(i)for i in text]
return lem_text
train_data['s_text_lemma'] = train_data['s_text_tokens_NOTstop'].apply(lambda x: word_lemmatizer(x))
train_data.head(20 )
df_train_x = df_train.iloc[:,1:]
df_train_y = df_train.iloc[:,:1]<choose_model_class>
stemmer = PorterStemmer()
def word_stemmer(text):
stem_text = " ".join([stemmer.stem(i)for i in text])
 return stem_text
train_data['s_text_stem'] = train_data['s_text_lemma'].apply(lambda x: word_stemmer(x))
train_data.head(20 )
def cnn_model(result_class_size):
model = Sequential()
model.add(Conv2D(32,(5, 5), input_shape=(28,28,1), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(16,(3, 3), activation='relu'))
model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(130, activation='relu'))
model.add(Dense(50, activation='relu'))
model.add(Dense(result_class_size, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=RMSprop() , metrics=['accuracy'])
 return model<categorify>
positive_train = train_data[train_data['sentiment']=='positive']
neutral_train = train_data[train_data['sentiment']=='neutral']
negative_train = train_data[train_data['sentiment']=='negative']
df_test = df_test / 255
df_train_x = df_train_x / 255<prepare_x_and_y>
max_len = 128
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file = '/kaggle/input/roberta-base/vocab.json',
merges_file = '/kaggle/input/roberta-base/merges.txt',
lowercase =True,
add_prefix_space=True 
)
 sentiment_id = {'positive':tokenizer.encode('positive' ).ids[0],
'negative':tokenizer.encode('negative' ).ids[0],
'neutral':tokenizer.encode('neutral' ).ids[0]}
train_data.reset_index(inplace=True)
 tot_tw = train_data.shape[0]
input_ids = np.ones(( tot_tw, max_len), dtype='int32')
 attention_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 token_type_ids = np.zeros(( tot_tw, max_len), dtype='int32')
 start_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 end_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 for i in range(tot_tw):
set1 = " "+" ".join(train_data.loc[i,'text'].split())
 set2 = " ".join(train_data.loc[i,'selected_text'].split())
 idx = set1.find(set2)
 set2_loc = np.zeros(( len(set1)))
 set2_loc[idx:idx+len(set2)]=1
if set1[idx-1]==" ":
set2_loc[idx-1]=1
enc_set1 = tokenizer.encode(set1)
 selected_text_token_idx=[]
for k,(a,b)in enumerate(enc_set1.offsets):
sm = np.sum(set2_loc[a:b])
 if sm > 0:
selected_text_token_idx.append(k)
 senti_token = sentiment_id[train_data.loc[i,'sentiment']]
input_ids[i,:len(enc_set1.ids)+5] = [0]+enc_set1.ids+[2,2]+[senti_token]+[2]
attention_mask[i,:len(enc_set1.ids)+5]=1
if len(selected_text_token_idx)> 0:
start_mask[i,selected_text_token_idx[0]+1]=1
end_mask[i, selected_text_token_idx[-1]+1]=1
arr_train_x_28x28 = np.reshape(df_train_x.values,(df_train_x.values.shape[0], 28, 28, 1))
arr_test_x_28x28 = np.reshape(df_test.values,(df_test.values.shape[0], 28, 28, 1))<split>
def custom_loss(y_true, y_pred):
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, from_logits = False, label_smoothing = 0.20)
 loss = tf.reduce_mean(loss)
 return loss
random_seed = 3
split_train_x, split_val_x, split_train_y, split_val_y, = train_test_split(arr_train_x_28x28, arr_train_y, test_size = 0.08, random_state=random_seed )<choose_model_class>
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 config_path = RobertaConfig.from_pretrained('/kaggle/input/tf-roberta/config-roberta-base.json')
 roberta_model = TFRobertaModel.from_pretrained('/kaggle/input/tf-roberta/pretrained-roberta-base.h5', config=config_path)
 x = roberta_model(ids, attention_mask = att, token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.05 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.05 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=custom_loss, optimizer=optimizer)
 return model
reduce_lr = ReduceLROnPlateau(monitor='val_acc',
factor=0.5,
patience=3,
min_lr=0.00001 )<train_model>
tot_test_tw = test_data.shape[0]
input_ids_t = np.ones(( tot_test_tw,max_len), dtype='int32')
 attention_mask_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 token_type_ids_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 for i in range(tot_test_tw):
set1 = " "+" ".join(test_data.loc[i,'text'].split())
 enc_set1 = tokenizer.encode(set1)
 s_token = sentiment_id[test_data.loc[i,'sentiment']]
input_ids_t[i,:len(enc_set1.ids)+5]=[0]+enc_set1.ids+[2,2]+[s_token]+[2]
attention_mask_t[i,:len(enc_set1.ids)+5]=1
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1 
)
 datagen.fit(split_train_x )<train_model>
pred_start= np.zeros(( input_ids_t.shape[0],max_len))
pred_end= np.zeros(( input_ids_t.shape[0],max_len))
for i in range(2):
print('--'*20)
 print('-- MODEL %i --'%(i+1))
print('--'*20)
 K.clear_session()
model = build_model()
model.load_weights('/kaggle/input/model4/v4-roberta-%i.h5'%(i+3))
pred = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=1)
 pred_start = pred_start +(pred[0]/2)
 pred_end = pred_end +(pred[1]/2 )
model.fit_generator(datagen.flow(split_train_x,split_train_y, batch_size=64),
epochs = 30, validation_data =(split_val_x,split_val_y),
verbose = 2, steps_per_epoch=700
, callbacks=[reduce_lr] )<save_to_csv>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(pred_start[k,])
 b = np.argmax(pred_end[k,])
 if a>b:
st = test_data.loc[k,'text']
else:
text1 = " "+" ".join(test_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 test_data['selected_text']=all
test_data.head(20 )
prediction = model.predict_classes(arr_test_x_28x28, verbose=0)
 data_to_submit = pd.DataFrame({"ImageId": list(range(1,len(prediction)+1)) , "Label": prediction})
 data_to_submit.to_csv("result.csv", header=True, index = False )<define_variables>
test_data[['textID','selected_text']].to_csv('submission.csv', index=False)
 print("Submission successful" )