Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<load_from_csv>
print('TF version',tf.__version__ )
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv" )<concatenate>
datapath= "/kaggle/input/tweet-sentiment-extraction"
train = pd.read_csv(datapath+"/train.csv" ).fillna('')
 test = pd.read_csv(datapath+"/test.csv" ).fillna('' )
full_dataset = pd.concat([train_data, test_data])
 full_dataset.head()<categorify>
print(train[train.isna().any(axis=1)])
 print(test[test.isna().any(axis=1)] )
class KnownCabinTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- KnownCabin transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X.Cabin.fillna(0, inplace=True)
 X["KnownCabin"] = 0
X.loc[X.Cabin != 0, 'KnownCabin'] = 1
print("- KnownCabin transformer applied -")
 return X 
<categorify>
def pre_process(text):

 text = str(text ).lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text
class TitleTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- Title transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X['Title'] = X.Name.str.extract('([A-Za-z]+)\.', expand=False)
 X['Title'] = X['Title'].replace(['Lady', 'Countess','Capt', 'Col',
'Don', 'Dr', 'Major', 'Rev',
'Sir', 'Jonkheer', 'Dona'],
'Rare')
 X['Title'] = X['Title'].replace(['Mlle', 'Ms'], 'Miss')
 X['Title'] = X['Title'].replace('Mme', 'Mrs')
 X['Title'] = X['Title'].fillna('Rare')
 title_dummies = pd.get_dummies(X['Title'], prefix="Title")
 X = pd.concat([X, title_dummies], axis=1)
 X = X.drop('Title', axis=1)
 print("- Title transformer applied -")
 return X<categorify>
eda['text2'] = eda['text'].apply(lambda x: pre_process(x))
eda['selectedtxt2'] = eda['selected_text'].apply(lambda x: pre_process(x))
eda.head(10 )
class MissingFareTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- MissingFare transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X.loc[(X.Pclass == 1)&(X.Fare.isnull()), 'Fare'] = X.loc[X.Pclass == 1]["Fare"].median()
X.loc[(X.Pclass == 2)&(X.Fare.isnull()), 'Fare'] = X.loc[X.Pclass == 2]["Fare"].median()
X.loc[(X.Pclass == 3)&(X.Fare.isnull()), 'Fare'] = X.loc[X.Pclass == 3]["Fare"].median()
print("- MissingFare transformer applied -")
 return X
class EmbarkedTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- MissingEmbarked transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
self.most_frequent = X["Embarked"].value_counts().index[0]
X["Embarked"].fillna(self.most_frequent, inplace=True)
 embarked_dummies = pd.get_dummies(X['Embarked'], prefix="Embarked")
 X = pd.concat([X, embarked_dummies], axis=1)
 X = X.drop('Embarked', axis=1)
 print("- MissingEmbarked transformer applied -")
 return X
class HypotheticalMissingsTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- HypotheticalMissings transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
self.most_frequent_class = X["Pclass"].value_counts().index[0]
self.unknown_name = "unknown"
self.horizontal_family = 0
self.vertical_family = 0
X["Pclass"].fillna(self.most_frequent_class, inplace=True)
 X["Name"].fillna(self.unknown_name, inplace=True)
 X["SibSp"].fillna(self.horizontal_family, inplace=True)
 X["Parch"].fillna(self.vertical_family, inplace=True)
 X.loc[(X.Sex.isnull())&(X.Title_Master == 1), "Sex"] == "male"
X.loc[(X.Sex.isnull())&(X.Title_Mr == 1), "Sex"] == "male"
X.loc[(X.Sex.isnull())&(X.Title_Rare == 1), "Sex"] == "male"
X.loc[(X.Sex.isnull())&(X.Title_Miss == 1), "Sex"] == "female"
X.loc[(X.Sex.isnull())&(X.Title_Mrs == 1), "Sex"] == "female"
print("- HypotheticalMissings transformer applied -")
 return X
class GenderTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- Gender transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X['Sex'] = X['Sex'].map({'male': 0, 'female': 1})
 print("- Gender transformer applied -")
 return X
class PclassTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- Pclass transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
pclass_dummies = pd.get_dummies(X['Pclass'], prefix='Pclass')
 X = pd.concat([X, pclass_dummies], axis=1)
 X = X.drop('Pclass', axis=1)
 print("- Pclass transformer applied -")
 return X
class FamilySizeTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- FamilySize transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X["FamilySize"] = X["SibSp"] + X["Parch"] + 1
print("- FamilySize transformer applied -")
 return X<categorify>
test['text2'] = test['text'].apply(lambda x: pre_process(x))
test.head(10 )
class AgeStatusTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- AgeStatus transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X["AgeStatus"] = "known"
X.loc[(X["Age"] > 1)&(( X["Age"]*2)%2 != 0), "AgeStatus"] = "estimated"
X.loc[X["Age"].isnull() , "AgeStatus"] = "guessed"
age_status_dummies = pd.get_dummies(X['AgeStatus'], prefix="AgeStatus")
 X = pd.concat([X, age_status_dummies], axis=1)
 X = X.drop('AgeStatus', axis=1)
 print("- AgeStatus transformer applied -")
 return X 
<categorify>
model_path = '/kaggle/input/tf-roberta/'
MAX_LEN = 96
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=model_path+'vocab-roberta-base.json',
merges_file=model_path+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
class TicketGroupingTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- TicketGrouping transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
cat_feature = ['Ticket']
count_enc = ce.CountEncoder(cols=cat_feature)
 count_enc.fit(X[cat_feature])
 grouping_col = count_enc.transform(X[cat_feature] ).add_suffix("_grouping")
 X = pd.concat([X, grouping_col], axis=1)
 print("- TicketGrouping transformer applied -")
 return X<categorify>
trainrows = train.shape[0]
input_ids = np.ones(( trainrows, MAX_LEN),dtype='int32')
 attention_mask= np.zeros(( trainrows,MAX_LEN),dtype='int32')
 token_type_ids= np.zeros(( trainrows,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( trainrows,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( trainrows,MAX_LEN),dtype='int32')
 for k in range(trainrows):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1
class AgeGuessingTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- AgeGuessing transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X.loc[(X.Age.isnull() == True)&(X.Title_Master == 1), "Age"] = X.loc[X.Title_Master == 1]["Age"].median()
X.loc[(X.Age.isnull() == True)&(X.SibSp > 0)&(X.Title_Miss == 1), "Age"] = X.loc[(X.Title_Miss == 1)&(X.SibSp > 0)]["Age"].median()
X.loc[(X.Age.isnull() == True)&(X.Ticket_grouping == 1), "Age"] = X.loc[(X.Ticket_grouping == 1)]["Age"].median()
X.loc[(X.Age.isnull() == True)&(X.SibSp > 1), "Age"] = X.loc[(X.Age.isnull() == False)&(X.SibSp > 1)]["Age"].median()
X.loc[(X.Age.isnull()), "Age"] = X.loc[(X.Age.isnull() == False)&(X.Ticket_grouping > 1)]["Age"].median()
print("- AgeGuessing transformer applied -")
 return X<categorify>
testrows = test.shape[0]
input_ids_t = np.ones(( testrows, MAX_LEN), dtype='int32')
 attention_mask_t= np.zeros(( testrows, MAX_LEN), dtype='int32')
 token_type_ids_t= np.zeros(( testrows, MAX_LEN), dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1
class AgeBinningTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- AgeBinning transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
self.bins = [0, 2, 16, 25, 40, 50, np.inf]
self.age_cat = ["0_2", "2_16", "16_25", "25_40", "30_50", "50+"]
X["AgeBins"] = pd.cut(X["Age"], bins=self.bins, labels=self.age_cat)
 age_bins_dummies = pd.get_dummies(X['AgeBins'], prefix="AgeBins")
 X = pd.concat([X, age_bins_dummies], axis=1)
 print("- AgeBinning transformer applied -")
 return X<categorify>
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(model_path+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(model_path+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model
custom_pipeline = Pipeline([ 
("cabin_trans", KnownCabinTransformer()), 
("title_trans", TitleTransformer()), 
("hypo_missing_trans", HypotheticalMissingsTransformer()), 
("missing_fare_trans", MissingFareTransformer()), 
("embarked_trans", EmbarkedTransformer()), 
("gender_trans", GenderTransformer()), 
("class_trans", PclassTransformer()), 
("family_size_trans", FamilySizeTransformer()), 
("ticket_grp_trans", TicketGroupingTransformer()), 
("age_status_trans", AgeStatusTransformer()), 
("age_guessing_trans", AgeGuessingTransformer()), 
("age_binning_trans", AgeBinningTransformer()),
])
 full_dataset = custom_pipeline.fit_transform(full_dataset )<feature_engineering>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
full_dataset["TS_Tragedy"] = "null"
i = 0
unique_tickets_subset = full_dataset.loc[full_dataset["Ticket_grouping"] > 2]["Ticket"].unique()
while i < len(unique_tickets_subset):
subset = full_dataset.loc[full_dataset.Ticket == unique_tickets_subset[i]]
try:
ratio = len(subset.loc[subset["Survived"] == 0])/ len(subset.loc[subset.Survived.isnull() == False])
 if ratio > 0.65 :
full_dataset.loc[full_dataset.Ticket == unique_tickets_subset[i], "TS_Tragedy"] = "yes"
else:
full_dataset.loc[full_dataset.Ticket == unique_tickets_subset[i], "TS_Tragedy"] = "no"
i += 1
except ZeroDivisionError:
i += 1
TST_dummies = pd.get_dummies(full_dataset['TS_Tragedy'], prefix="TST")
 full_dataset = pd.concat([full_dataset, TST_dummies], axis=1)
 full_dataset = full_dataset.drop('TS_Tragedy', axis=1)
 full_dataset.head(3)
<prepare_output>
SEED = 42
EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
LABEL_SMOOTHING = 0.1
np.random.seed(SEED)
 jac = []
VER='v0'
DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=3,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) ,
key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3),
reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],
token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
full_dataset.set_index(full_dataset.PassengerId, verify_integrity = True, inplace=True)
 full_dataset.tail()<prepare_x_and_y>
selectedtext = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 selectedtext.append(st )
<normalization><EOS> 
test['selected_text'] = selectedtext
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 test.sample(10 )