Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<define_variables>
df = pd.read_csv('.. /input/nlp-getting-started/train.csv',index_col=0)
 df_test = pd.read_csv('.. /input/nlp-getting-started/test.csv',index_col=0)
 temp = [(x,y)for x,y in zip(list(df['text']),list(df['target'])) ]
random.shuffle(temp)
 tweets = [t[0] for t in temp]
y = [t[1] for t in temp]
y = np.array(y ).astype('float32' )
%%time
!python.. /input/tweet-inference-scripts/inference_bert_wwm.py<define_variables>
print('Observations in training set')
 print(df['target'].count())
 print()
print('Label proportion in training set')
 print(df['target'].value_counts() /(sum(df['target'].value_counts())))
 print()
print('Observations in test set')
 print(df_test['text'].count() )
%%time
!python.. /input/tweet-inference-scripts/inference_albert.py<install_modules>
import tensorflow as tf
from transformers import RobertaTokenizerFast, TFRobertaForSequenceClassification
%%time
!pip install /kaggle/input/bertweet-libs/sacrebleu-1.4.10-py3-none-any.whl
!cp -R /kaggle/input/bertweet-libs/fairseq-0.9.0/fairseq-0.9.0 /kaggle/working
!cp -R /kaggle/input/bertweet-libs/fastBPE-0.1.0/fastBPE-0.1.0/ /kaggle/working
!pip install /kaggle/working/fairseq-0.9.0/
!pip install /kaggle/working/fastBPE-0.1.0/
!python.. /input/tweet-inference-scripts/inference_bertweet.py<define_variables>
model_name = 'roberta-large'
roberta_tokenizer = RobertaTokenizerFast.from_pretrained(model_name)
 roberta_seq = TFRobertaForSequenceClassification.from_pretrained(model_name )
%%time
!python.. /input/tweet-inference-scripts/inference_roberta_anton.py<load_pretrained>
for t in tweets:
if '&' in re.sub(r'(&amp|&gt|&lt)','',t):
print(t )
%%time
!python.. /input/tweet-inference-scripts/inference_roberta_large_hiki.py<define_variables>
for t in tweets:
if any([x in t for x in [' btw ',' omg ',' lol ',' thx ']]):
print(t )
%%time
!python.. /input/tweet-inference-scripts/inference_roberta_hiki.py<categorify>
def process_tweets(tweets):
r = tweets
r = [re.sub(r'https?://t.co/\w+','',t)for t in r]
r = [re.sub('&amp;','&',t)for t in r]
r = [re.sub('&gt;','gt',t)for t in r]
r = [re.sub('&lt;','lt',t)for t in r]
return r
tweets = process_tweets(tweets )
def string_from_preds_char_level(texts, preds):
selected_texts = []
n_models = len(preds)
 for idx in range(len(texts)) :
data = texts[idx]
start_probas = np.mean(
 [preds[i][0][idx] for i in range(n_models)], 0)
 end_probas = np.mean(
 [preds[i][1][idx] for i in range(n_models)], 0)
 start_idx = np.argmax(start_probas)
 end_idx = np.argmax(end_probas)
 if end_idx < start_idx:
selected_text = data
else:
selected_text = data[start_idx: end_idx]
selected_texts.append(selected_text.strip())
 return selected_texts
df_test = pd.read_csv(
 '.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 df_test['selected_text'] = ''
sub = pd.read_csv(
 '.. /input/tweet-sentiment-extraction/sample_submission.csv')
 preds_1 = np.load('preds_char_test_cnn_0_2.npy')
 preds_2 = np.load('preds_char_test_cnn_1_2.npy')
 preds_3 = np.load('preds_char_test_wavenet_2.npy')
 preds_4 = np.load('preds_char_test_wavenet_6_2.npy')
 test_preds =(preds_1 + preds_2 + preds_3 + preds_4)/ 4
selected_texts = string_from_preds_char_level(
 df_test['text'].values, test_preds)
 sub['selected_text'] = selected_texts
df_test['selected_text'] = selected_texts
sub.to_csv('submission.csv', index=False )<import_modules>
temp = roberta_tokenizer(tweets[:5],padding='max_length',max_length=50)
 temp.keys()
print('TF version',tf.__version__ )<load_from_csv>
print('Original tweet:')
 print(tweets[0])
 print('Encoded tweet:')
 print(temp['input_ids'][0])
 print('Decoded tweet:')
 print(roberta_tokenizer.decode(temp['input_ids'][0]))
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()<string_transform>
all_tweets = list(pd.concat([df,df_test],axis=0)['text'])
 all_tweets = process_tweets(all_tweets)
 max_len = max([len(t)for t in roberta_tokenizer(all_tweets)['input_ids']])
 print(max_len )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
X_train, X_test, y_train, y_test = train_test_split(tweets,y,test_size=0.30)
 X_train = roberta_tokenizer(X_train,padding='max_length',max_length=max_len,return_tensors='tf')
 X_test = roberta_tokenizer(X_test,padding='max_length',max_length=max_len,return_tensors='tf' )
MAX_LEN = 196
EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 42
N_SPLITS = 5
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<define_variables>
batch_size = 8
train_dataset = tf.data.Dataset.from_tensor_slices(( dict(X_train),y_train))
train_dataset = train_dataset.batch(batch_size)
 test_dataset = tf.data.Dataset.from_tensor_slices(( dict(X_test),y_test))
test_dataset = test_dataset.batch(batch_size )
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1<categorify>
temp_x, temp_y = next(iter(test_dataset))
temp = roberta_seq(temp_x,temp_y)
 temp
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1<train_model>
optimizer = tf.keras.optimizers.Adam(learning_rate=5e-6)
 loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
 roberta_seq.compile(optimizer=optimizer,loss=loss,metrics=['accuracy'] )
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss<categorify>
history = roberta_seq.fit(train_dataset,epochs=3,
validation_data=test_dataset,
callbacks=[callback_chkpt] )
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.15 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.15 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<split>
roberta_seq.load_weights(chkpt )
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=N_SPLITS,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train_df.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train_df.loc[k,'text']
else:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train_df.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print() 
<feature_engineering>
outputs = roberta_seq.predict(test_dataset)
 y_pred = outputs[0].argmax(axis=1 )
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<string_transform>
print('Confusion matrix:')
 print(confusion_matrix(y_test,y_pred,labels=[0,1]))
print()
print('Classification report:')
 print(classification_report(y_test,y_pred,labels=[0,1],target_names=['not a disaster','disaster']))
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_df.loc[k,'text']
else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )<save_to_csv>
tweets_test = list(df_test['text'])
 tweets_test = process_tweets(tweets_test)
 X_real_test = roberta_tokenizer(tweets_test,padding='max_length',max_length=max_len,return_tensors='tf')
 real_test_dataset = tf.data.Dataset.from_tensor_slices(dict(X_real_test))
real_test_dataset = real_test_dataset.batch(batch_size)
 real_test_dataset
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test_df.sample(25 )<save_to_csv>
outputs_test = roberta_seq.predict(real_test_dataset)
 y_pred_test = outputs_test[0].argmax(axis=1 )
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )<set_options>
results = pd.Series(y_pred_test,index=df_test.index,name='target')
 results.to_csv('./submission.csv' )