Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<train_model>
!pip install tensorflow_hub
!pip install bert-for-tf2
!pip install tensorflow
!pip install sentencepiece
!pip install transformers
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss<categorify>
BertTokenizer = bert.bert_tokenization.FullTokenizer
bert_layer = hub.KerasLayer("https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1", trainable=False)
 vocabulary_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
to_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = BertTokenizer(vocabulary_file, to_lower_case )
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.ReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.ReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<define_variables>
stopwrds = set(stopwords.words('english'))
TAG_RE = re.compile(r'<[^>]+>')
 def remove_tags(text):
return TAG_RE.sub('', text)
 def preprocess_text(sen):
sentence = emoji.demojize(sen)
 sentence = re.sub(r"http:\S+",'',sentence)
 sentence = ' '.join([x for x in nltk.word_tokenize(sentence)if x not in stopwrds])
 sentence = remove_tags(sentence)
 sentence = re.sub(r'['+string.punctuation+']','',sentence)
 sentence = re.sub(r"\s+[a-zA-Z]\s+", ' ', sentence)
 sentence = re.sub(r'\s+', ' ', sentence)
 return sentence.strip()
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<feature_engineering>
def tokenize_bert(data):
tokenized = data.apply(( lambda x: tokenizer.convert_tokens_to_ids(['[CLS]'])+ tokenizer.convert_tokens_to_ids(tokenizer.tokenize(x))))
return tokenized
def pad_mask(data_tokenized,max_len):
padded = tf.keras.preprocessing.sequence.pad_sequences(data_tokenized, maxlen=max_len, dtype='int32', padding='post',value=0.0)
 masked = np.where(padded!=0,1,0)
 return padded, masked
def get_max_len(data):
max_len = 0
for val in data:
tmp = len(tokenizer.tokenize(val))
if tmp > max_len:
max_len = tmp
return max_len
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<string_transform>
def encode(df):
tweet = tf.ragged.constant([tokenizer.convert_tokens_to_ids(tokenizer.tokenize(s)) for s in df])
 cls1 = [tokenizer.convert_tokens_to_ids(['[CLS]'])]*tweet.shape[0]
input_word_ids = tf.concat([cls1, tweet], axis=-1)
 input_mask = tf.ones_like(input_word_ids ).to_tensor()
type_cls = tf.zeros_like(cls1)
 type_tweets = tf.zeros_like(tweet)
 input_type_ids = tf.concat([type_cls, type_tweets], axis=-1 ).to_tensor()
inputs = {
'input_ids': input_word_ids.to_tensor() ,
'input_mask': input_mask,
'input_type_ids': input_type_ids}
return inputs
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )<save_to_csv>
finetune_train = pd.read_csv('/kaggle/input/twitter-sentiment-analysis-hatred-speech/train.csv',encoding="utf-8")
 finetune_test = pd.read_csv('/kaggle/input/twitter-sentiment-analysis-hatred-speech/test.csv',encoding="utf-8")
 finetune_train["hashtags"]=finetune_train["tweet"].apply(lambda x:re.findall(r"
finetune_test["hashtags"]=finetune_test["tweet"].apply(lambda x:re.findall(r"
finetune_train["hashtags"]=finetune_train["hashtags"].apply(lambda x: ' '.join(x))
finetune_test["hashtags"]=finetune_test["hashtags"].apply(lambda x: ' '.join(x))
finetune_train = finetune_train.rename(columns={'label':'target'})
 finetune_test = finetune_test.rename(columns={'label':'target'})
 finetune_train.info()
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<import_modules>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv",encoding="utf-8")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv",encoding="utf-8")
 train["hashtags"]=train["text"].apply(lambda x:re.findall(r"
test["hashtags"]=test["text"].apply(lambda x:re.findall(r"
train["hashtags"]=train["hashtags"].apply(lambda x: ' '.join(x))
test["hashtags"]=test["hashtags"].apply(lambda x: ' '.join(x))
train.info()
import os
import torch
import pandas as pd
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
from torch.optim import lr_scheduler
from sklearn import model_selection
from sklearn import metrics
import transformers
import tokenizers
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup
from tqdm import tqdm
from IPython.display import FileLink
from torch.nn.modules.loss import _WeightedLoss
from torch.nn import functional as F<define_variables>
finetune_train["clean"] = finetune_train["tweet"].apply(lambda x: preprocess_text(x.lower()))
finetune_test["clean"] = finetune_test["tweet"].apply(lambda x: preprocess_text(x.lower()))
print("length of finetune train set:",len(finetune_train))
print("length of finetune test set:",len(finetune_test))
MAX_LEN = 192
TRAIN_BATCH_SIZE = 8
VALID_BATCH_SIZE = 4
EPOCHS = 5
ROBERTA_PATH = ".. /input/roberta-base"
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True 
)
<load_pretrained>
train["clean"] = train["text"].apply(lambda x: preprocess_text(x.lower()))
test["clean"] = test["text"].apply(lambda x: preprocess_text(x.lower()))
print("length of train set:",len(train))
print("length of test set:",len(test))
class RoBertaModel(transformers.BertPreTrainedModel):
def __init__(self):
model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True
super(RoBertaModel, self ).__init__(model_config)
 self.roberta = transformers.RobertaModel.from_pretrained(ROBERTA_PATH, config=model_config)
 self.drop_out = nn.Dropout(0.1)
 self.Cov1S = nn.Conv1d(768 * 2, 128 , kernel_size = 2 ,stride = 1)
 self.Cov1E = nn.Conv1d(768 * 2, 128, kernel_size = 2 ,stride = 1)
 self.Cov2S = nn.Conv1d(128 , 64 , kernel_size = 2 ,stride = 1)
 self.Cov2E = nn.Conv1d(128 , 64 , kernel_size = 2 ,stride = 1)
 self.lS = nn.Linear(64 , 1)
 self.lE = nn.Linear(64 , 1)
 torch.nn.init.normal_(self.lS.weight, std=0.02)
 torch.nn.init.normal_(self.lE.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
_, _, out = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 out = out.permute(0,2,1)
 same_pad1 = torch.zeros(out.shape[0] , 768*2 , 1 ).cuda()
same_pad2 = torch.zeros(out.shape[0] , 128 , 1 ).cuda()
out1 = torch.cat(( same_pad1 , out), dim = 2)
 out1 = self.Cov1S(out1)
 out1 = torch.cat(( same_pad2 , out1), dim = 2)
 out1 = self.Cov2S(out1)
 out1 = F.leaky_relu(out1)
 out1 = out1.permute(0,2,1)
 start_logits = self.lS(out1 ).squeeze(-1)
 out2 = torch.cat(( same_pad1 , out), dim = 2)
 out2 = self.Cov1E(out2)
 out2 = torch.cat(( same_pad2 , out2), dim = 2)
 out2 = self.Cov2E(out2)
 out2 = F.leaky_relu(out2)
 out2 = out2.permute(0,2,1)
 end_logits = self.lE(out2 ).squeeze(-1)
 return start_logits, end_logits<string_transform>
def extract_features(df,test_df):
txt=' '.join(df[df["target"]==1]["clean"])
 disaster_unigram=nltk.FreqDist(nltk.word_tokenize(txt))
txt=' '.join(df[df["target"]==0]["clean"])
 nondisaster_unigram=nltk.FreqDist(nltk.word_tokenize(txt))
txt=' '.join(df[df["target"]==1]["clean"])
 disaster_bigram=nltk.FreqDist(nltk.bigrams(nltk.word_tokenize(txt)))
 txt=' '.join(df[df["target"]==0]["clean"])
 nondisaster_bigram=nltk.FreqDist(nltk.bigrams(nltk.word_tokenize(txt)))
 txt=' '.join(df[df["target"]==1]["hashtags"])
 disaster_unigram_hash=nltk.FreqDist(nltk.word_tokenize(txt))
txt=' '.join(df[df["target"]==0]["hashtags"])
 nondisaster_unigram_hash=nltk.FreqDist(nltk.word_tokenize(txt))
df["unigram_disas"]=df["clean"].apply(lambda x: sum([disaster_unigram.get(wrd)for wrd in nltk.word_tokenize(x)if disaster_unigram.get(wrd)!=None])/len(disaster_unigram))
test_df["unigram_disas"]=test_df["clean"].apply(lambda x: sum([disaster_unigram.get(wrd)for wrd in nltk.word_tokenize(x)if disaster_unigram.get(wrd)!=None])/len(disaster_unigram))
df["unigram_nondisas"]=df["clean"].apply(lambda x: sum([nondisaster_unigram.get(wrd)for wrd in nltk.word_tokenize(x)if nondisaster_unigram.get(wrd)!=None])/len(nondisaster_unigram))
test_df["unigram_nondisas"]=test_df["clean"].apply(lambda x: sum([nondisaster_unigram.get(wrd)for wrd in nltk.word_tokenize(x)if nondisaster_unigram.get(wrd)!=None])/len(nondisaster_unigram))
df["unigram_disas_hash"]=df["hashtags"].apply(lambda x: sum([disaster_unigram_hash.get(wrd)for wrd in nltk.word_tokenize(x)if disaster_unigram_hash.get(wrd)!=None])/len(disaster_unigram_hash))
test_df["unigram_disas_hash"]=test_df["hashtags"].apply(lambda x: sum([disaster_unigram_hash.get(wrd)for wrd in nltk.word_tokenize(x)if disaster_unigram_hash.get(wrd)!=None])/len(disaster_unigram_hash))
df["unigram_nondisas_hash"]=df["hashtags"].apply(lambda x: sum([nondisaster_unigram_hash.get(wrd)for wrd in nltk.word_tokenize(x)if nondisaster_unigram_hash.get(wrd)!=None])/len(nondisaster_unigram_hash))
test_df["unigram_nondisas_hash"]=test_df["hashtags"].apply(lambda x: sum([nondisaster_unigram_hash.get(wrd)for wrd in nltk.word_tokenize(x)if nondisaster_unigram_hash.get(wrd)!=None])/len(nondisaster_unigram_hash))
df["bigram_disas"]=df["clean"].apply(lambda x: sum([disaster_bigram.get(wrd)for wrd in nltk.bigrams(nltk.word_tokenize(x)) if disaster_bigram.get(wrd)!=None])/len(disaster_bigram)if x.strip() !='' else 0)
 test_df["bigram_disas"]=test_df["clean"].apply(lambda x: sum([disaster_bigram.get(wrd)for wrd in nltk.bigrams(nltk.word_tokenize(x)) if disaster_bigram.get(wrd)!=None])/len(disaster_bigram)if x.strip() !='' else 0)
 df["bigram_nondisas"]=df["clean"].apply(lambda x: sum([nondisaster_bigram.get(wrd)for wrd in nltk.bigrams(nltk.word_tokenize(x)) if nondisaster_bigram.get(wrd)!=None])/len(nondisaster_bigram)if x.strip() !='' else 0)
 test_df["bigram_nondisas"]=test_df["clean"].apply(lambda x: sum([nondisaster_bigram.get(wrd)for wrd in nltk.bigrams(nltk.word_tokenize(x)) if nondisaster_bigram.get(wrd)!=None])/len(nondisaster_bigram)if x.strip() !='' else 0)
 return df,test_df
def process_data(tweet, selected_text, sentiment, tokenizer, max_len):
tweet = " " + " ".join(str(tweet ).split())
 selected_text = ' ' + " ".join(str(selected_text ).split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st-1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids
tweet_offsets = tok_tweet.offsets
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 targets_start = target_idx[0]
targets_end = target_idx[-1]
sentiment_id = {
'positive': 1313,
'negative': 2430,
'neutral': 7974
}
input_ids = [0] + input_ids_orig + [2] + [2] + [sentiment_id[sentiment]] + [2]
token_type_ids = [0] *(len(input_ids_orig)+ 1)+ [0, 0, 0, 0]
mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] + tweet_offsets + [(0, 0)]*4
targets_start += 1
targets_end += 1
padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([1] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_length)
 return {
'ids': input_ids,
'mask': mask,
'token_type_ids': token_type_ids,
'targets_start': targets_start,
'targets_end': targets_end,
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'offsets': tweet_offsets
}<categorify>
finetune_train,finetune_test = extract_features(finetune_train,finetune_test)
 finetune_train.head(2 )
class TweetDataset:
def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = TOKENIZER
self.max_len = MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }<define_variables>
train,test = extract_features(train,test)
 train.head(2 )
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
best_idxs,
offsets,
verbose=False):
if best_idxs[1] < best_idxs[0]:
best_idxs[1] = best_idxs[0]
filtered_output = ""
for ix in range(best_idxs[0], best_idxs[1] + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
if sentiment_val != "neutral" and verbose == True:
if filtered_output.strip().lower() != target_string.strip().lower() :
print("********************************")
 print(f"Output= {filtered_output.strip() }")
 print(f"Target= {target_string.strip() }")
 print(f"Tweet= {original_tweet.strip() }")
 print("********************************")
 jac = 0
return jac, filtered_output<load_pretrained>
def build_bert(max_len):
input_ids = keras.layers.Input(shape=(max_len,), name="input_ids", dtype=tf.int32)
 input_typ = keras.layers.Input(shape=(max_len,), name="input_type_ids", dtype=tf.int32)
 input_mask = keras.layers.Input(shape=(max_len,), name="input_mask", dtype=tf.int32)
 input_features = keras.layers.Input(shape=(6,), name="input_features", dtype=tf.float32)
 bert_inputs = {"input_ids": input_ids, "input_mask": input_mask,'input_type_ids':input_typ}
bert_model = hub.KerasLayer("https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1", trainable=True,name='keraslayer')
 pooled_output, _ = bert_model([input_ids, input_mask,input_typ])
 merge_two = concatenate([pooled_output, input_features])
 out = keras.layers.Dense(1, activation='sigmoid' )(merge_two)
 model = keras.Model(inputs=[bert_inputs,input_features],outputs=out)
 return model
device = torch.device("cuda")
 model1 = RoBertaModel()
model1.to(device)
 model1.load_state_dict(torch.load("/kaggle/input/roberta-conv-8fold/trained_models/model_0.bin"))
model1.eval()
model2 = RoBertaModel()
model2.to(device)
 model2.load_state_dict(torch.load("/kaggle/input/roberta-conv-8fold/trained_models/model_1.bin"))
model2.eval()
model3 = RoBertaModel()
model3.to(device)
 model3.load_state_dict(torch.load("/kaggle/input/roberta-conv-8fold/trained_models/model_2.bin"))
model3.eval()
model4 = RoBertaModel()
model4.to(device)
 model4.load_state_dict(torch.load("/kaggle/input/roberta-conv-8fold/trained_models/model_3.bin"))
model4.eval()
model5 = RoBertaModel()
model5.to(device)
 model5.load_state_dict(torch.load("/kaggle/input/roberta-conv-8fold/trained_models/model_4.bin"))
model5.eval()
model6 = RoBertaModel()
model6.to(device)
 model6.load_state_dict(torch.load("/kaggle/input/roberta-conv-8fold/trained_models/model_5.bin"))
model6.eval()

 model8 = RoBertaModel()
model8.to(device)
 model8.load_state_dict(torch.load("/kaggle/input/roberta-conv-8fold/trained_models/model_7.bin"))
model8.eval() 
<load_from_csv>
all_df = pd.concat([finetune_train,finetune_test])
 max_len = get_max_len(all_df["clean"])+ 1
encode_ds_all = encode(all_df["clean"] )
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values
final_output = []
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=VALID_BATCH_SIZE,
num_workers=1 
)
<find_best_params>
encode_ds_tr = {'input_ids':encode_ds_all["input_ids"][0:31962,:],
'input_mask':encode_ds_all["input_mask"][0:31962,:],
'input_type_ids':encode_ds_all["input_type_ids"][0:31962,:]}
def get_best_start_end_idxs(_start_logits, _end_logits):
best_logit = -1000
best_idxs = None
for start_idx, start_logit in enumerate(_start_logits):
for end_idx, end_logit in enumerate(_end_logits[start_idx:]):
logit_sum =(start_logit + end_logit ).item()
if logit_sum > best_logit:
best_logit = logit_sum
best_idxs =(start_idx, start_idx+end_idx)
 return best_idxs<define_variables>
features = ['unigram_disas','unigram_nondisas','unigram_disas_hash','unigram_nondisas_hash','bigram_disas','bigram_nondisas']
encode_features_tr = all_df[features].iloc[0:31962,:]
fin_output_start = []
fin_output_end = []
fin_padding_lens = []
fin_tweet_tokens = []
fin_orig_sentiment = []
fin_orig_tweet = []
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 o11, o12 = modell1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)

o31, o32 = modell3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)

o51, o52 = modell5(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 o61, o62 = modell6(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 o71, o72 = modell7(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start1, outputs_end1 = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start2, outputs_end2 = model2(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start3, outputs_end3 = model3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start4, outputs_end4 = model4(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start5, outputs_end5 = model5(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start6, outputs_end6 = model6(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)

outputs_start8, outputs_end8 = model8(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start1 = torch.softmax(outputs_start1, dim=1 ).cpu().detach().numpy()
outputs_end1 = torch.softmax(outputs_end1, dim=1 ).cpu().detach().numpy()
outputs_start2 = torch.softmax(outputs_start2, dim=1 ).cpu().detach().numpy()
outputs_end2 = torch.softmax(outputs_end2, dim=1 ).cpu().detach().numpy()
outputs_start3 = torch.softmax(outputs_start3, dim=1 ).cpu().detach().numpy()
outputs_end3 = torch.softmax(outputs_end3, dim=1 ).cpu().detach().numpy()
outputs_start4 = torch.softmax(outputs_start4, dim=1 ).cpu().detach().numpy()
outputs_end4 = torch.softmax(outputs_end4, dim=1 ).cpu().detach().numpy()
outputs_start5 = torch.softmax(outputs_start5, dim=1 ).cpu().detach().numpy()
outputs_end5 = torch.softmax(outputs_end5, dim=1 ).cpu().detach().numpy()
outputs_start6 = torch.softmax(outputs_start6, dim=1 ).cpu().detach().numpy()
outputs_end6 = torch.softmax(outputs_end6, dim=1 ).cpu().detach().numpy()

 outputs_start8 = torch.softmax(outputs_start8, dim=1 ).cpu().detach().numpy()
outputs_end8 = torch.softmax(outputs_end8, dim=1 ).cpu().detach().numpy()
o31=torch.softmax(o31, dim=1 ).cpu().detach().numpy()
o32=torch.softmax(o32, dim=1 ).cpu().detach().numpy()

 o11=torch.softmax(o11, dim=1 ).cpu().detach().numpy()
o12=torch.softmax(o12, dim=1 ).cpu().detach().numpy()
o51=torch.softmax(o51, dim=1 ).cpu().detach().numpy()
o52=torch.softmax(o52, dim=1 ).cpu().detach().numpy()
o61=torch.softmax(o61, dim=1 ).cpu().detach().numpy()
o62=torch.softmax(o62, dim=1 ).cpu().detach().numpy()
o71=torch.softmax(o71, dim=1 ).cpu().detach().numpy()
o72=torch.softmax(o72, dim=1 ).cpu().detach().numpy()
outputs_start =(0.025*o31 + 0.1*o71 + 0.15*o51 + 0.12*o11 + 0.025*o61 +outputs_start1*0.15+ outputs_start2*0.08+ outputs_start3*0.15 + outputs_start4*0.1 + outputs_start5*0.15+ outputs_start6*0.05+ outputs_start8*0.025)
 outputs_end =(0.025*o32 + 0.1*o72 + 0.15*o52+ 0.12*o12 + 0.025*o62 +outputs_end1*0.15+ outputs_end2*0.08 + outputs_end3*0.15+ outputs_end4*0.1+ outputs_end5*0.05+ outputs_end6*0.15+ outputs_end8*0.025)
 for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
best_idxs=get_best_start_end_idxs(outputs_start[px, :],outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence )<feature_engineering>
y_enc = finetune_train["target"]
loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
 optimizer = keras.optimizers.Adam(lr=1e-3,decay=1e-3/64)
 model.compile(optimizer=optimizer, loss=[loss, loss],metrics=["accuracy"])
 checkpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True)
 earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy')
 fine_history = model.fit([encode_ds_tr,encode_features_tr], y_enc, validation_split=0.34,shuffle=True,epochs=2,batch_size=64,verbose=1 )
def get_indicies(text, find_text):
indicies = []
index = 0
while index < len(text):
index = text.find(find_text, index)
 if index == -1:
break
indicies.append(index)
 index += len(find_text)
 return indicies
def add_noise(tweet, pred, sentiment):
pred_idx = tweet.find(pred)
 len_pred = len(pred)
 ds_idxs = get_indicies(tweet, ' ')
 ts_idxs = get_indicies(tweet, ' ')
 new_sl = pred
if(( sentiment != 'neutral')&(len(ds_idxs)> 0)) :
if(pred_idx > ds_idxs[0])&(tweet[0] == ' ')&(len(pred)!= len(tweet)&(pred_idx > 1)) :
new_sl = tweet[pred_idx-2:pred_idx+len_pred]
if(( sentiment != 'neutral')&(len(ts_idxs)> 0)) :
if(pred_idx > ts_idxs[0])&(tweet[0] == ' ')&(len(pred)!= len(tweet)&(pred_idx > 1)) :
new_sl = tweet[pred_idx-2-1:pred_idx+len_pred-1]
if(( sentiment != 'neutral')&(len(ds_idxs)== 2)) :
if(pred_idx > ds_idxs[0])&(pred_idx > ds_idxs[1])&(len(pred)!= len(tweet)) :
new_sl = tweet[pred_idx-2-1:pred_idx+len_pred-1]
if(( sentiment == 'neutral')&(tweet[0] == '_')&(len(tweet.split())>1)) :
first_word_len = len(tweet.split() [0])
 new_sl = tweet[0+first_word_len+1:]
return new_sl 
<define_variables>
all_df = pd.concat([train,test])
 max_len = get_max_len(train["clean"])+ 1
encode_ds_all = encode(all_df["clean"] )
text_list = df_test.text.values.tolist()
pred_list = final_output
sentiment_list = df_test.sentiment.values.tolist() 
<categorify>
encode_ds_tr = {'input_ids':encode_ds_all["input_ids"][0:7613,:],
'input_mask':encode_ds_all["input_mask"][0:7613,:],
'input_type_ids':encode_ds_all["input_type_ids"][0:7613,:]}
encode_ds_tr
new_preds = []
for i in range(0,len(text_list)) :
new_pred = add_noise(text_list[i], pred_list[i].strip() , sentiment_list[i])
 new_preds.append(new_pred )<load_from_csv>
encode_features_tr = all_df[features].iloc[0:7613,:]
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = new_preds 
<save_to_csv>
y_enc = train["target"]
loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
 optimizer = keras.optimizers.Adam(lr=1e-5,decay=1e-5/64)
 model.compile(optimizer=optimizer, loss=[loss, loss],metrics=["accuracy"])
 checkpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True)
 earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy')
 fine_history = model.fit([encode_ds_tr,encode_features_tr], y_enc, validation_split=0.34,shuffle=True,epochs=3,batch_size=64,verbose=1 )
sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sample.to_csv("submission.csv", index=False )<import_modules>
y_pred=model.predict([encode_ds_tr,encode_features_tr])
 y_pred = y_pred.round()
print(classification_report(y_enc,y_pred))
import copy
from transformers import *
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset
from tqdm import tqdm
from functools import partial
from multiprocessing import Pool, cpu_count
import tokenizers<init_hyperparams>
encode_ds_ts = {'input_ids':encode_ds_all["input_ids"][7613:,:],
'input_mask':encode_ds_all["input_mask"][7613:,:],
'input_type_ids':encode_ds_all["input_type_ids"][7613:,:]}
encode_ds_ts
class Config:
def __init__(self):
self.max_seq_length = 192
self.val_batch_size = 256
self.num_workers = 8<set_options>
encode_features_ts = all_df[features].iloc[7613:,:]
Config = Config()<string_transform>
y_pred=model.predict([encode_ds_ts,encode_features_ts])
 y_pred= y_pred.round()
submission=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 submission['id']=test['id']
submission['target']=y_pred
submission['target']=submission['target'].astype(int)
 submission.head(10)
<import_modules><EOS> 
submission.to_csv('sample_submission.csv',index=False )