Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<train_model>
internet_on = False
class EsemblingClassifier:
def __init__(self, verbose=True):
self.ann = build_ann(optimizer=optimizers.Adam(lr=0.001))
self.rf = RandomForestClassifier(n_estimators=300, max_depth=11, random_state=42)
 self.svm = SVC(random_state=42)
 self.trained = False
self.verbose = verbose
def fit(self, X, y):
if self.verbose:
print('-------- Fitting models --------')
 self.ann.fit(X, y, epochs=30, batch_size=16, verbose=0)
 self.rf.fit(X, y)
 self.svm.fit(X, y)
 self.trained = True
def predict(self, X):
if self.trained == False:
raise NotFittedError('Please train the classifier before making a prediction')
 if self.verbose:
print('-------- Making and combining predictions --------')
 predictions = list()
pred_ann = self.ann.predict(X)
 pred_ann =(pred_ann > 0.5)*1
pred_rf = self.rf.predict(X)
 pred_svm = self.svm.predict(X)
 for n in range(len(pred_ann)) :
combined = pred_ann[n] + pred_rf[n] + pred_svm[n]
p = 0 if combined == 1 or combined == 0 else 1
predictions.append(p)
 return predictions<compute_train_metric>
if internet_on==True:
!pip install fairseq fastBPE
ens = EsemblingClassifier()
ens.fit(X_train, Y_train)
 ens_prediction = ens.predict(X_test)
 score = metrics.accuracy_score(Y_test, ens_prediction)
 display_confusion_matrix(Y_test, ens_prediction, score=score )<load_from_csv>
if internet_on==True:
!mv /root/.cache/pip/wheels/df/60/ff/1764bce64cccd9d2c06ba19e5f6f4108ad29e2d48e1068c684/* /kaggle/working
!mv /root/.cache/pip/wheels/fb/85/9b/286072121774d5b8b0253ab66271b558069189cbe795bc6084/* /kaggle/working
test_df_raw = pd.read_csv(path_test)
 test = test_df_raw.copy()
test = preprocess_data(test)
 test = pd.DataFrame(sc.fit_transform(test.values), index=test.index, columns=test.columns)
 test.head()<save_to_csv>
if internet_on==False:
!pip install.. /input/fairseq-and-fastbpe/sacrebleu-1.4.9-py3-none-any.whl
model_test = EsemblingClassifier()
model_test.fit(X, Y)
 prediction = model_test.predict(test)
 result_df = test_df_raw.copy()
result_df['Survived'] = prediction
result_df.to_csv('submission.csv', columns=['PassengerId', 'Survived'], index=False )<define_variables>
if internet_on==False:
!pip install.. /input/v1-fairseq-fastbpe/fastBPE-0.1.0-cp36-cp36m-linux_x86_64.whl
!pip install.. /input/v1-fairseq-fastbpe/fairseq-0.9.0-cp36-cp36m-linux_x86_64.whl
start_time = time.time()<set_options>
print('TF version',tf.__version__ )
%matplotlib inline 
<install_modules>
class BERTweetTokenizer() :
def __init__(self,pretrained_path = 'pretrained_models/BERTweet_base_transformers/'):
self.bpe = fastBPE(SimpleNamespace(bpe_codes= pretrained_path + "bpe.codes"))
self.vocab = Dictionary()
self.vocab.add_from_file(pretrained_path + "dict.txt")
 self.cls_token_id = 0
self.pad_token_id = 1
self.sep_token_id = 2
self.pad_token = '<pad>'
self.cls_token = '
'
self.sep_token = '
'
def bpe_encode(self,text):
return self.bpe.encode(text)
 def encode(self,text,add_special_tokens=False):
subwords = self.bpe.encode(text)
 input_ids = self.vocab.encode_line(subwords, append_eos=False, add_if_not_exist=False ).long().tolist()
return input_ids
def tokenize(self,text):
return self.bpe_encode(text ).split()
def convert_tokens_to_ids(self,tokens):
input_ids = self.vocab.encode_line(' '.join(tokens), append_eos=False, add_if_not_exist=False ).long().tolist()
return input_ids
def decode_id(self,id):
return self.vocab.string(id, bpe_symbol = '@@')
 def decode_id_nospace(self,id):
return self.vocab.string(id, bpe_symbol = '@@ ' )
!pip install tensorflow-text==2.0.0 --user<import_modules>
tokenizer = BERTweetTokenizer('/kaggle/input/bertweet-base-transformers/' )
import tensorflow as tf
import tensorflow_hub as hub<import_modules>
tokenizer.encode('positive' )
import tensorflow_text as text<load_from_csv>
tokenizer.encode('negative' )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )<categorify>
tokenizer.encode('neutral' )
def clean(text):
text = re.sub(r"http\S+", " ", text)
 text = re.sub(r"RT ", " ", text)
 text = re.sub(r"[^a-zA-Z
text = re.sub(r"[0-9]", " ", text)
 text = re.sub(r"\s+", " ", text)
 text = text.strip()
return text<feature_engineering>
tokenizer.decode_id([14058] )
train.text = train.text.apply(clean)
 test.text = test.text.apply(clean )<load_pretrained>
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()
use = hub.load("https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3" )<categorify>
train_df.sentiment.value_counts(dropna=False )
X_train = []
for r in tqdm(train.text.values):
emb = use(r)
 review_emb = tf.reshape(emb, [-1] ).numpy()
X_train.append(review_emb)
 X_train = np.array(X_train)
 y_train = train.target.values
X_test = []
for r in tqdm(test.text.values):
emb = use(r)
 review_emb = tf.reshape(emb, [-1] ).numpy()
X_test.append(review_emb)
 X_test = np.array(X_test )<split>
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
train_arrays, test_arrays, train_labels, test_labels = train_test_split(X_train,
y_train,
random_state =42,
test_size=0.05 )<train_on_grid>
MAX_LEN = 96
PATH = '.. /input/bertweet-base-transformers/'
sentiment_id = {'positive': 1809, 'negative': 3392, 'neutral': 14058}
def svc_param_selection(X, y, nfolds):
Cs = [1.07]
gammas = [2.075]
param_grid = {'C': Cs, 'gamma' : gammas}
grid_search = GridSearchCV(SVC(kernel='rbf'), param_grid, cv=nfolds, n_jobs=8)
 grid_search.fit(X, y)
 grid_search.best_params_
return grid_search
model = svc_param_selection(train_arrays,train_labels, 5 )<find_best_params>
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc:
w = tokenizer.decode_id([t])
 if text1[text1.find(w,idx)-1] == " ":
idx+=1
offsets.append(( idx,idx+len(w)))
 idx += len(w)
 else:
offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
if len(enc)<92:
input_ids[k,:len(enc)+5] = [0] + enc + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
if len(enc)>91:
input_ids[k,:96] = [0] + enc[:91] + [2,2] + [s_tok] + [2]
attention_mask[k,:96] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,96-1] = 1
model.best_params_<predict_on_test>
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
if len(enc)<92:
input_ids_t[k,:len(enc)+5] = [0] + enc + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc)+5] = 1
if len(enc)>91:
input_ids_t[k,:96] = [0] + enc[:91] + [2,2] + [s_tok] + [2]
attention_mask_t[k,:96] = 1
pred = model.predict(test_arrays )<compute_test_metric>
all=[]
count=0
for k in range(train_df.shape[0]):
a = np.argmax(start_tokens[k,])
 b = np.argmax(end_tokens[k,])
 text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:b])
 st = st.replace('<unk>','')
 all.append(jaccard(st,train_df.loc[k,'selected_text']))
print('>>>> Jaccard =',np.mean(all))
cm = confusion_matrix(test_labels,pred)
 cm<compute_test_metric>
improve_jacc_review = False
accuracy = accuracy_score(test_labels,pred)
 accuracy<predict_on_test>
if improve_jacc_review == True:
all=[]
count=0
for k in range(train_df.shape[0]):
a = np.argmax(start_tokens[k,])
 b = np.argmax(end_tokens[k,])
 text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:b])
 st = st.replace('<unk>','')
 if jaccard(st,train_df.loc[k,'selected_text'])<.3:
print(k)
 print(st)
 print(train_df.loc[k,'selected_text'])
 print()
all.append(jaccard(st,train_df.loc[k,'selected_text']))
print('>>>> Jaccard =',np.mean(all))
test_pred = model.predict(X_test)
 submission['target'] = test_pred.round().astype(int)
<define_variables>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'model.bin',config=config,from_pt=True)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x[0])
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x[0])
 x2 = tf.keras.layers.BatchNormalization()(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model
count = 2
prob_disaster = 0.9
keyword_list_disaster = list(ag[(ag['Count']>count)&(ag['Disaster Probability']>=prob_disaster)].index)
 keyword_list_disaster<feature_engineering>
n_splits = 5
ids_disaster = test['id'][test.keyword.isin(keyword_list_disaster)].values
submission['target'][submission['id'].isin(ids_disaster)] = 1<save_to_csv>
n_seeds = 1
submission.to_csv("submission.csv", index=False)
 submission.head(10 )<set_options>
trainModel=False
%matplotlib inline 
<load_from_csv>
if trainModel==True:
for x in range(n_seeds):
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=777+x)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train_df.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
reduce_lr = tf.keras.callbacks.LearningRateScheduler(scheduler)
 sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i-%x.h5'%(VER,fold,x), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 hist = model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=8, verbose=DISPLAY, callbacks=[sv, reduce_lr],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i-%x.h5'%(VER,fold,x))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/(n_splits*n_seeds)
 preds_end += preds[1]/(n_splits*n_seeds)
 all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:a+3])
 else:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:b])
 st = st.replace('<unk>','')
 all.append(jaccard(st,train_df.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
train_file_path = ".. /input/titanic/train.csv"
test_file_path = ".. /input/titanic/test.csv"
train_data = pd.read_csv(train_file_path)
 test_data = pd.read_csv(test_file_path )<groupby>
if trainModel==False:
DISPLAY=1
for x in range(n_seeds):
jac = []
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
print('
print('
print('
skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=777+x)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train_df.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/bertweet-files/v0-roberta-%i-%x.h5'%(fold,x))
print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/(n_splits*n_seeds)
 preds_end += preds[1]/(n_splits*n_seeds)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 all = []; pos = []; neg = []; nue = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:a+3])
 else:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:b])
 st = st.replace('<unk>','')
 if train_df.loc[k,'sentiment']=='positive':
pos.append(jaccard(st,train_df.loc[k,'selected_text']))
if train_df.loc[k,'sentiment']=='negative':
neg.append(jaccard(st,train_df.loc[k,'selected_text']))
if train_df.loc[k,'sentiment']=='neutral':
st = text1
nue.append(jaccard(st,train_df.loc[k,'selected_text']))
all.append(jaccard(st,train_df.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print('>>>> FOLD %i Neutral Jaccard ='%(fold+1),np.mean(nue))
print('>>>> FOLD %i Positive Jaccard ='%(fold+1),np.mean(pos))
print('>>>> FOLD %i Negative Jaccard ='%(fold+1),np.mean(neg))
print()
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
train_data.groupby(by=['Pclass'] ).count()<sort_values>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:a+3])
 else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:b])
 st = st.replace('<unk>','')
 if test_df.loc[k,'sentiment']=='neutral':
st = text1
all.append(st )
perc = train_data[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 perc*100 
<feature_engineering>
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )