Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_pretrained>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 data.sample(10 )
def scheduler(epoch):
return 3e-5 * 0.2**epoch
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model 
<predict_on_test>
text = data.text
duplicates = data[text.isin(text[text.duplicated() ])].sort_values(by='text')
 conflicting_check = pd.DataFrame(duplicates.groupby(['text'] ).target.mean())
 conflicting_check.sample(10 )
n_splits=5
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits<save_to_csv>
conflicting = conflicting_check.loc[(conflicting_check.target != 1)&(conflicting_check.target != 0)].index
data = data.drop(data[text.isin(conflicting)].index)
 print('Conflicting samples count:', conflicting.shape[0] )
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 test['selected_text'] = all
submission=test[['textID','selected_text']]
submission.to_csv('submission.csv',index=False)
 submission.head(5 )<import_modules>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )
import numpy as np
import pandas as pd
import random
import math
import argparse
import pickle
import tokenizers
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras import layers as L
from transformers import TFBertMainLayer,TFBertModel,TFBertPreTrainedModel,BertConfig,BertTokenizer
from transformers import TFRobertaModel, RobertaTokenizer,RobertaConfig
from transformers.modeling_tf_utils import get_initializer
from sklearn.model_selection import StratifiedKFold
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences<define_variables>
!pip install transformers
def set_seed(args):
random.seed(args.seed)
 np.random.seed(args.seed)
 tf.random.set_seed(args.seed)
 set_seed(args )<load_from_csv>
sentences = data.text.values
labels =data.target.values
test_df=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/test.csv")
 train_df=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/train.csv")
 train_df.dropna(inplace=True)
 train_df=train_df.reset_index()<define_variables>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )
def load_data_roberta(df,tokenizer, is_train_eval):
sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
ct = df.shape[0]
MAX_LEN_WORD=args.max_seq_length
input_ids = np.ones(( ct,MAX_LEN_WORD),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN_WORD),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN_WORD),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN_WORD),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN_WORD),dtype='int32')
 offsets=[]
tweets=[]
for k in range(ct):
text1 = " "+" ".join(df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 sentiment_token = sentiment_id[df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, sentiment_token] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
offsets.append([(0,0)]* 2 + enc.offsets + [(0,0)])
 tweets.append(text1)
 if is_train_eval:
chars = np.zeros(( len(text1)))
 text2 = " ".join(df.loc[k,'selected_text'].split())
 start_idx = text1.find(text2)
 chars[start_idx:start_idx+len(text2)]=1
if text1[start_idx-1]==' ':
chars[start_idx-1] = 1
selected_toks_index = []
for i,(a,b)in enumerate(enc.offsets):
sm = np.sum(chars[a:b])
 if sm>0: selected_toks_index.append(i)
 if len(selected_toks_index)>0:
start_tokens[k,selected_toks_index[0]+2] = 1
end_tokens[k,selected_toks_index[-1]+2] = 1
if is_train_eval:
dataset={'input_ids':input_ids,
'attention_mask':attention_mask,
'token_type_ids':token_type_ids,
'start_tokens':start_tokens,
'end_tokens':end_tokens,
'tweets':tweets,
'offsets':offsets}
else:
dataset={'input_ids':input_ids,
'attention_mask':attention_mask,
'token_type_ids':token_type_ids,
'tweets':tweets,
'offsets':offsets}
return dataset<load_pretrained>
print(' Original: ', sentences[0])
 print('Tokenized: ', tokenizer.tokenize(sentences[0]))
print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[0])) )
model_path=".. /input/robertatransformer/roberta-base-tf_model.h5"
model_class=TFRobertaModel
config = RobertaConfig.from_pretrained(".. /input/robertatransformer/roberta-base-config.json")
 tokenizer = tokenizers.ByteLevelBPETokenizer(vocab_file=".. /input/robertatransformer/roberta-base-vocab.json",
merges_file=".. /input/robertatransformer/roberta-base-merges.txt",
lowercase=True,
add_prefix_space=True )<load_pretrained>
max_len = 0
for sent in sentences:
input_ids = tokenizer.encode(sent, add_special_tokens=True)
 max_len = max(max_len, len(input_ids))
print('Max tweet length: ', max_len )
trainset=load_data_roberta(train_df,tokenizer, is_train_eval=True)
 testset=load_data_roberta(test_df,tokenizer, is_train_eval=False )<categorify>
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 labels = torch.tensor(labels)
 print('Original: ', sentences[0])
 print('Token IDs:', input_ids[0] )
PAD_ID=1
lr=args.learning_rate
def build_model_cnn(model_class,model_path,config):
MAX_LEN_WORD=args.max_seq_length
ids = tf.keras.layers.Input(( MAX_LEN_WORD,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN_WORD,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN_WORD,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN_WORD - tf.reduce_sum(padding, -1)
 max_len_word = tf.reduce_max(lens)
 ids_ = ids[:, :max_len_word]
att_ = att[:, :max_len_word]
tok_ = tok[:, :max_len_word]
bertmodel = model_class.from_pretrained(model_path,config=config)
 x = bertmodel(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768,2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=lr)
 model.compile(loss=loss_fn,optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN_WORD - max_len_word]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN_WORD - max_len_word]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<compute_train_metric>
SPLIT = 0.999
dataset = TensorDataset(input_ids, attention_masks, labels)
 train_size = int(SPLIT * len(dataset))
val_size = len(dataset)- train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
 print('{:>5,} training samples'.format(train_size))
print('{:>5,} validation samples'.format(val_size))
LABEL_SMOOTHING=0.1
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss 
<load_pretrained>
batch_size = 32
train_dataloader = DataLoader(
 train_dataset,
sampler = RandomSampler(train_dataset),
batch_size = batch_size 
)
 validation_dataloader = DataLoader(
 val_dataset,
sampler = SequentialSampler(val_dataset),
batch_size = batch_size
)
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f )<load_pretrained>
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()
def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model 
<compute_test_metric>
optimizer = AdamW(model.parameters() ,
lr = 2e-5,
eps = 1e-8
)
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
epochs = 2
total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )
jac = [];
VER='v0';
DISPLAY=1
n_splits=args.cv_splits
lr=args.learning_rate
MAX_LEN=args.max_seq_length
n_best=3
input_ids=trainset['input_ids']
attention_mask=trainset['attention_mask']
token_type_ids=trainset['token_type_ids']
start_tokens=trainset['start_tokens']
end_tokens=trainset['end_tokens']
input_ids_test=testset['input_ids']
attention_mask_test=testset['attention_mask']
token_type_ids_test=testset['token_type_ids']
ct_train=input_ids.shape[0]
ct_test=input_ids_test.shape[0]
oof_start = np.zeros(( ct_train,MAX_LEN))
oof_end = np.zeros(( ct_train,MAX_LEN))
preds_start = np.zeros(( ct_test,MAX_LEN))
preds_end = np.zeros(( ct_test,MAX_LEN))
jac_valiation=np.zeros(( ct_train,))
out_of_range=[]
abnormal=[]
skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=args.seed)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train_df.sentiment.values)) :
print('
K.clear_session()
model, padded_model = build_model_cnn(model_class,model_path,config)
 inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = f'roberta-base-{fold+1}.h5'
for epoch in range(1, args.num_train_epochs + 1):
BATCH_SIZE=args.train_batch_size
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY,
callbacks=[],validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model for validation and pred...')
 padded_model.set_weights(model.get_weights())
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_test,attention_mask_test,token_type_ids_test],verbose=DISPLAY)
 preds_start+=preds[0]/n_splits
preds_end+=preds[1]/n_splits
fold_jac_val=[]
for k in idxV:
a= np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st=train_df.loc[k,'text']
else:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 jac_val=jaccard(st,train_df.loc[k,'selected_text'])
 fold_jac_val.append(jac_val)
 jac.append(np.mean(fold_jac_val))
print(f'>>>> FOLD {fold+1} Jaccard ={np.mean(fold_jac_val)}' )<compute_test_metric>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )
print(f'The CV jaccard is {np.mean(jac)}' )<define_variables>
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 training_stats = []
total_t0 = time.time()
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 40 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
logits = outputs[1]
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_time = format_time(time.time() - t0)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in validation_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
output = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = output[0]
logits = output[1]
total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
total_eval_accuracy += flat_accuracy(logits, label_ids)
 avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
 print(" Accuracy: {0:.2f}".format(avg_val_accuracy))
avg_val_loss = total_eval_loss / len(validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
training_stats.append(
 {
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid.Loss': avg_val_loss,
'Valid.Accur.': avg_val_accuracy,
'Training Time': training_time,
'Validation Time': validation_time
} 
)
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)) )
roberta_models=["roberta-base-1.h5",
"roberta-base-2.h5",
"roberta-base-3.h5",
"roberta-base-4.h5",
"roberta-base-5.h5"]<define_variables>
pd.set_option('precision', 2)
 df_stats = pd.DataFrame(data=training_stats)
 df_stats = df_stats.set_index('epoch')
 df_stats
MAX_LEN_WORD=args.max_seq_length
DISPLAY=1
input_ids_test=testset['input_ids']
attention_mask_test=testset['attention_mask']
token_type_ids_test=testset['token_type_ids']
preds_test_start = np.zeros(( ct_test,MAX_LEN_WORD))
preds_test_end = np.zeros(( ct_test,MAX_LEN_WORD))
input_ids_train=trainset['input_ids']
attention_mask_train=trainset['attention_mask']
token_type_ids_train=trainset['token_type_ids']
preds_oof_start = np.zeros(( ct_train,MAX_LEN_WORD))
preds_oof_end = np.zeros(( ct_train,MAX_LEN_WORD))<predict_on_test>
test_data = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 print('Number of test sentences: {:,}
'.format(test_data.shape[0]))
sentences = test_data.text.values
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 batch_size = 32
prediction_data = TensorDataset(input_ids, attention_masks,)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size )
model_number=len(roberta_models)
 for i in range(model_number):
print(f"predict roberta model----{i+1}")
 K.clear_session()
weight_fn=roberta_models[i]
_, padded_model = build_model_cnn(model_class,model_path,config)
 load_weights(padded_model, weight_fn)
 preds_test = padded_model.predict([input_ids_test,attention_mask_test,token_type_ids_test],verbose=DISPLAY)
 preds_test_start+=preds_test[0]/model_number
preds_test_end+=preds_test[1]/model_number
preds_oof = padded_model.predict([input_ids_train,attention_mask_train,token_type_ids_train],verbose=DISPLAY)
 preds_oof_start+=preds_oof[0]/model_number
preds_oof_end+=preds_oof[1]/model_number<feature_engineering>
print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))
 model.eval()
predictions = []
for batch in prediction_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
predictions.append(logits)
 print(' DONE.' )
def token_level_to_char_level(text, offsets, preds):
probas_char = np.zeros(len(text))
for i, offset in enumerate(offsets):
if offset[0] or offset[1]:
probas_char[offset[0]:offset[1]] = preds[i]
return probas_char<define_variables>
flat_predictions = np.concatenate(predictions, axis=0)
 flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
<feature_engineering><EOS> 
submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 submission.target = flat_predictions
submission.to_csv('submission.csv', index=False )