Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<filter>
!pip -q install pyspellchecker optuna
positive_train = train_data[train_data['sentiment']=='positive']
neutral_train = train_data[train_data['sentiment']=='neutral']
negative_train = train_data[train_data['sentiment']=='negative']<define_variables>
import os
import optuna
import pandas as pd
import numpy as np
import random
import re
from scipy import sparse
from spellchecker import SpellChecker
import string
import warnings
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.utils.validation import check_X_y, check_is_fitted
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn.metrics import accuracy_score, matthews_corrcoef, f1_score
max_len = 128
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file = '/kaggle/input/roberta-base/vocab.json',
merges_file = '/kaggle/input/roberta-base/merges.txt',
lowercase =True,
add_prefix_space=True 
)
 sentiment_id = {'positive':tokenizer.encode('positive' ).ids[0],
'negative':tokenizer.encode('negative' ).ids[0],
'neutral':tokenizer.encode('neutral' ).ids[0]}
train_data.reset_index(inplace=True)
 tot_tw = train_data.shape[0]
input_ids = np.ones(( tot_tw, max_len), dtype='int32')
 attention_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 token_type_ids = np.zeros(( tot_tw, max_len), dtype='int32')
 start_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 end_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 for i in range(tot_tw):
set1 = " "+" ".join(train_data.loc[i,'text'].split())
 set2 = " ".join(train_data.loc[i,'selected_text'].split())
 idx = set1.find(set2)
 set2_loc = np.zeros(( len(set1)))
 set2_loc[idx:idx+len(set2)]=1
if set1[idx-1]==" ":
set2_loc[idx-1]=1
enc_set1 = tokenizer.encode(set1)
 selected_text_token_idx=[]
for k,(a,b)in enumerate(enc_set1.offsets):
sm = np.sum(set2_loc[a:b])
 if sm > 0:
selected_text_token_idx.append(k)
 senti_token = sentiment_id[train_data.loc[i,'sentiment']]
input_ids[i,:len(enc_set1.ids)+5] = [0]+enc_set1.ids+[2,2]+[senti_token]+[2]
attention_mask[i,:len(enc_set1.ids)+5]=1
if len(selected_text_token_idx)> 0:
start_mask[i,selected_text_token_idx[0]+1]=1
end_mask[i, selected_text_token_idx[-1]+1]=1<compute_train_metric>
optuna.logging.set_verbosity(0)
 warnings.filterwarnings("ignore" )
def custom_loss(y_true, y_pred):
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, from_logits = False, label_smoothing = 0.20)
 loss = tf.reduce_mean(loss)
 return loss<load_pretrained>
SEED = 42
NUM_SPLITS = 10
NUM_TRIALS = 100
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 config_path = RobertaConfig.from_pretrained('/kaggle/input/tf-roberta/config-roberta-base.json')
 roberta_model = TFRobertaModel.from_pretrained('/kaggle/input/tf-roberta/pretrained-roberta-base.h5', config=config_path)
 x = roberta_model(ids, attention_mask = att, token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.05 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.05 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=custom_loss, optimizer=optimizer)
 return model<categorify>
os.environ['PYTHONHASHSEED']=str(SEED)
 random.seed(SEED)
 np.random.seed(SEED )
tot_test_tw = test_data.shape[0]
input_ids_t = np.ones(( tot_test_tw,max_len), dtype='int32')
 attention_mask_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 token_type_ids_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 for i in range(tot_test_tw):
set1 = " "+" ".join(test_data.loc[i,'text'].split())
 enc_set1 = tokenizer.encode(set1)
 s_token = sentiment_id[test_data.loc[i,'sentiment']]
input_ids_t[i,:len(enc_set1.ids)+5]=[0]+enc_set1.ids+[2,2]+[s_token]+[2]
attention_mask_t[i,:len(enc_set1.ids)+5]=1<load_pretrained>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv" )
pred_start= np.zeros(( input_ids_t.shape[0],max_len))
pred_end= np.zeros(( input_ids_t.shape[0],max_len))
for i in range(2):
print('--'*20)
 print('-- MODEL %i --'%(i+1))
print('--'*20)
 K.clear_session()
model = build_model()
model.load_weights('/kaggle/input/model4/v4-roberta-%i.h5'%(i+3))
pred = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=1)
 pred_start = pred_start +(pred[0]/2)
 pred_end = pred_end +(pred[1]/2 )<feature_engineering>
test["target"] = -1
df = pd.concat([train, test] )
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(pred_start[k,])
 b = np.argmax(pred_end[k,])
 if a>b:
st = test_data.loc[k,'text']
else:
text1 = " "+" ".join(test_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 test_data['selected_text']=all
test_data.head(20 )<save_to_csv>
print("NaN Distribution
")
 for col in df.columns:
print(f"{col}: {(( df[col].isna().sum() /df.shape[0])*100):.2f}" )
test_data[['textID','selected_text']].to_csv('submission.csv', index=False)
 print("Submission successful" )<load_from_csv>
df["text"] = df["text"].str.lower()
train_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 test_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 print('Train Dataset')
 print(train_data.head())
 print('Test Dataset')
 print(test_data.head() )<feature_engineering>
PUNCT_TO_REMOVE = string.punctuation
def remove_punctuation(text):
return text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))
df["text"] = df["text"].apply(lambda text: remove_punctuation(text))
train_data.dropna(axis = 0,inplace=True)
 def remove_punctuation(text):
no_punct = "".join([c for c in text if c not in string.punctuation])
 return no_punct
train_data['s_text_clean'] = train_data['selected_text'].apply(str ).apply(lambda x: remove_punctuation(x.lower()))
tokenizer = RegexpTokenizer(r'\w+')
 train_data['s_text_tokens'] = train_data['s_text_clean'].apply(str ).apply(lambda x: tokenizer.tokenize(x))
def remove_stopwords(text):
words = [w for w in text if(w not in stopwords.words('english')or w not in 'im')]
return words
train_data['s_text_tokens_NOTstop'] = train_data['s_text_tokens'].apply(lambda x: remove_stopwords(x))
lemmatizer = WordNetLemmatizer()
def word_lemmatizer(text):
lem_text = [lemmatizer.lemmatize(i)for i in text]
return lem_text
train_data['s_text_lemma'] = train_data['s_text_tokens_NOTstop'].apply(lambda x: word_lemmatizer(x))
stemmer = PorterStemmer()
def word_stemmer(text):
stem_text = " ".join([stemmer.stem(i)for i in text])
 return stem_text
train_data['s_text_stem'] = train_data['s_text_lemma'].apply(lambda x: word_stemmer(x))
train_data.head()<define_variables>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 df["text"] = df["text"].apply(lambda text: remove_emoji(text))
max_len = 150
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file = '/kaggle/input/roberta/vocab-roberta-base.json',
merges_file = '/kaggle/input/roberta/merges-roberta-base.txt',
lowercase =True,
add_prefix_space=True 
)
 sentiment_id = {'positive':tokenizer.encode('positive' ).ids[0],
'negative':tokenizer.encode('negative' ).ids[0],
'neutral':tokenizer.encode('neutral' ).ids[0]}
train_data.reset_index(inplace=True)
 train_shape = train_data.shape[0]
input_ids = np.ones(( train_shape, max_len), dtype='int32')
 attention_mask = np.zeros(( train_shape, max_len), dtype='int32')
 token_type_ids = np.zeros(( train_shape, max_len), dtype='int32')
 start_mask = np.zeros(( train_shape, max_len), dtype='int32')
 end_mask = np.zeros(( train_shape, max_len), dtype='int32')
 for i in range(train_shape):
set1 = " "+" ".join(train_data.loc[i,'text'].split())
 set2 = " ".join(train_data.loc[i,'selected_text'].split())
 idx = set1.find(set2)
 set2_loc = np.zeros(( len(set1)))
 set2_loc[idx:idx+len(set2)]=1
if set1[idx-1]==" ":
set2_loc[idx-1]=1
enc_set1 = tokenizer.encode(set1)
 selected_text_token_idx=[]
for k,(a,b)in enumerate(enc_set1.offsets):
sm = np.sum(set2_loc[a:b])
 if sm > 0:
selected_text_token_idx.append(k)
 senti_token = sentiment_id[train_data.loc[i,'sentiment']]
input_ids[i,:len(enc_set1.ids)+5] = [0]+enc_set1.ids+[2,2]+[senti_token]+[2]
attention_mask[i,:len(enc_set1.ids)+5]=1
if len(selected_text_token_idx)> 0:
start_mask[i,selected_text_token_idx[0]+1]=1
end_mask[i, selected_text_token_idx[-1]+1]=1<compute_test_metric>
def remove_urls(text):
url_pattern = re.compile(r'https?://\S+|www\.\S+')
 try:
return url_pattern.sub(r'', text)
 except:
print(text)
 df["text"] = df["text"].apply(lambda text: remove_urls(text))
def custom_loss(y_true, y_pred):
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, from_logits = False, label_smoothing = 0.2)
 loss = tf.reduce_mean(loss)
 return loss<load_pretrained>
def remove_html(text):
html_pattern = re.compile('<.*?>')
 return html_pattern.sub(r'', text)
 df["text"] = df["text"].apply(lambda text: remove_html(text))
os.environ['WANDB_MODE'] = 'dryrun'
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 config_path = RobertaConfig.from_pretrained('/kaggle/input/roberta/config-roberta-base.json')
 roberta_model = TFRobertaModel.from_pretrained('/kaggle/input/roberta/pretrained-roberta-base.h5', config=config_path)
 x = roberta_model(ids, attention_mask=att, token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=custom_loss, optimizer=optimizer)
 return model<categorify>
with open(".. /input/slangtext/slang.txt", "r")as file:
chat_words_str = file.read()
chat_words_map_dict = {}
chat_words_list = []
for line in chat_words_str.split("
"):
if line != "" and "=" in line:
cw = line.split("=")[0]
cw_expanded = line.split("=")[1]
chat_words_list.append(cw)
 chat_words_map_dict[cw] = cw_expanded
chat_words_list = set(chat_words_list )
test_shape = test_data.shape[0]
input_ids_t = np.ones(( test_shape,max_len), dtype='int32')
 attention_mask_t = np.zeros(( test_shape,max_len), dtype='int32')
 token_type_ids_t = np.zeros(( test_shape,max_len), dtype='int32')
 for i in range(test_shape):
set1 = " "+" ".join(test_data.loc[i,'text'].split())
 enc_set1 = tokenizer.encode(set1)
 senti_token_t = sentiment_id[test_data.loc[i,'sentiment']]
input_ids_t[i,:len(enc_set1.ids)+5]=[0]+enc_set1.ids+[2,2]+[senti_token_t]+[2]
attention_mask_t[i,:len(enc_set1.ids)+5]=1<predict_on_test>
def chat_words_conversion(text):
new_text = []
for w in text.split() :
if w.upper() in chat_words_list:
new_text.append(chat_words_map_dict[w.upper() ])
 else:
new_text.append(w)
 return " ".join(new_text)
 df["text"] = df["text"].apply(lambda text: chat_words_conversion(text))
preds_start= np.zeros(( input_ids_t.shape[0],max_len))
preds_end= np.zeros(( input_ids_t.shape[0],max_len))
model = build_model()
model.load_weights('/kaggle/input/roberta/v4-roberta-4.h5')
 pred = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=1)
 pred_start = pred[0]
pred_end = pred[1]
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(pred_start[k,])
 b = np.argmax(pred_end[k,])
 if a>b:
st = test_data.loc[k,'text']
else:
text1 = " "+" ".join(test_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 test_data['selected_text']=all
test_data.head()<save_to_csv>
spell = SpellChecker()
def correct_spellings(text):
corrected_text = []
misspelled_words = spell.unknown(text.split())
 for word in text.split() :
if word in misspelled_words:
corrected_text.append(spell.correction(word))
else:
corrected_text.append(word)
 return " ".join(corrected_text)
 df["text"] = df["text"].apply(lambda text: chat_words_conversion(text))
test_data[['textID','selected_text']].to_csv('submission.csv', index=False )<import_modules>
class NBSVMClassifier(BaseEstimator, ClassifierMixin):
def __init__(self, C=1.0, max_iter=100, dual=False, n_jobs=1):
self.C = C
self.dual = dual
self.n_jobs = n_jobs
self.max_iter = max_iter
def predict(self, x):
check_is_fitted(self, ['_r', '_clf'])
 return self._clf.predict(x.multiply(self._r))
def fit(self, x, y):
y = y.values
x, y = check_X_y(x, y, accept_sparse=True)
 def pr(x, y_i, y):
p = x[y==y_i].sum(0)
 return(p+1)/(( y==y_i ).sum() +1)
 self._r = sparse.csr_matrix(np.log(pr(x,1,y)/ pr(x,0,y)))
 x_nb = x.multiply(self._r)
 self._clf = LogisticRegression(C=self.C, dual=self.dual,
max_iter=self.max_iter,
n_jobs=self.n_jobs ).fit(x_nb, y)
 return self
from transformers import AutoModelForQuestionAnswering, AutoModel, AutoConfig, get_linear_schedule_with_warmup
from transformers.optimization import AdamW
import torch
import torch.nn as nn
import torch.nn.functional as F
import pandas as pd
import numpy as np
from pathlib import Path
import os
from itertools import compress<import_modules>
X_train, X_valid, y_train, y_valid = train_test_split(train["text"], train["target"],
test_size=0.2, random_state=SEED,
stratify=train["target"] )
from torch.utils.data import DataLoader
from functools import partial
from tokenizers import BertWordPieceTokenizer
from sklearn.model_selection import train_test_split
from tqdm import tqdm
from fastai.core import *
from fastai.text import *<load_from_csv>
vec = TfidfVectorizer(ngram_range=(1,2), min_df=3, max_df=0.9,
strip_accents='unicode', use_idf=1,
smooth_idf=1, sublinear_tf=1 )
file_dir, electra_dir = [Path(f'/kaggle/input/{i}')for i in ['tweet-sentiment-extraction', 'electrabase']]
train_df = pd.read_csv(file_dir/'train.csv')
 train_df['text'] = train_df['text'].apply(lambda x: str(x))
train_df['sentiment'] = train_df['sentiment'].apply(lambda x: str(x))
train_df['selected_text'] = train_df['selected_text'].apply(lambda x: str(x))
test_df = pd.read_csv(file_dir/'test.csv')
 test_df['text'] = test_df['text'].apply(lambda x: str(x))
test_df['sentiment'] = test_df['sentiment'].apply(lambda x: str(x))<choose_model_class>
def objective(trial):
C = trial.suggest_float(name="C", low=1e-3, high=1e3, log=True)
 max_iter = trial.suggest_discrete_uniform(name="max_iter", low=50, high=500, q=50)
 nbsvm = NBSVMClassifier(C=C, max_iter=max_iter)
 train_term_doc = vec.fit_transform(X_train)
 valid_term_doc = vec.transform(X_valid)
 nbsvm.fit(train_term_doc, y_train)
 preds = nbsvm.predict(valid_term_doc)
 preds[preds>=0.5] = 1
preds[preds<0.5] = 0
acc = accuracy_score(y_valid, preds)
 return acc
max_len = 128
bs = 64
tokenizer = BertWordPieceTokenizer(str(electra_dir/'vocab.txt'), lowercase=True )<categorify>
study = optuna.create_study(direction="maximize")
 study.optimize(objective, n_trials=NUM_TRIALS, show_progress_bar=True )
def preprocess(sentiment, tweet, selected, tokenizer, max_len):
_input = tokenizer.encode(sentiment, tweet)
 _span = tokenizer.encode(selected, add_special_tokens=False)
 len_span = len(_span.ids)
 start_idx = None
end_idx = None
for ind in(i for i, e in enumerate(_input.ids)if e == _span.ids[0]):
if _input.ids[ind: ind + len_span] == _span.ids:
start_idx = ind
end_idx = ind + len_span - 1
break
if not start_idx:
idx0 = tweet.find(selected)
 idx1 = idx0 + len(selected)
 char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1):
char_targets[ct] = 1
tweet_offsets = list(compress(_input.offsets, _input.type_ids)) [0:-1]
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx, end_idx = target_idx[0] +3 , target_idx[-1] + 3
_input.start_target = start_idx
_input.end_target = end_idx
_input.tweet = tweet
_input.sentiment = sentiment
_input.selected = selected
_input.pad(max_len)
 return _input<compute_train_metric>
print(f"Best Value: {study.best_trial.value}")
 print(f"Best Params: {study.best_params}" )
def reduce_loss(loss, reduction='mean'):
return loss.mean() if reduction=='mean' else loss.sum() if reduction=='sum' else loss
class LabelSmoothingCrossEntropy(nn.Module):
def __init__(self, ε:float=0.1, reduction='mean'):
super().__init__()
self.ε,self.reduction = ε,reduction
def forward(self, output, target):
c = output.size() [-1]
log_preds = F.log_softmax(output, dim=-1)
 loss = reduce_loss(-log_preds.sum(dim=-1), self.reduction)
 nll = F.nll_loss(log_preds, target, reduction=self.reduction)
 return torch.lerp(nll, loss/c, self.ε )<prepare_x_and_y>
kwargs = study.best_params
class TweetDataset(Dataset):
def __init__(self, dataset, test = None):
self.df = dataset
self.test = test
def __getitem__(self, idx):
if not self.test:
sentiment, tweet, selected =(self.df[col][idx] for col in ['sentiment', 'text', 'selected_text'])
 _input = preprocess(sentiment, tweet, selected, tokenizer, max_len)
 yb = [torch.tensor(_input.start_target), torch.tensor(_input.end_target)]
else:
_input = tokenizer.encode(self.df.sentiment[idx], self.df.text[idx])
 _input.pad(max_len)
 yb = 0
xb = [torch.LongTensor(_input.ids),
torch.LongTensor(_input.attention_mask),
torch.LongTensor(_input.type_ids),
np.array(_input.offsets)]
return xb, yb
def __len__(self):
return len(self.df )<load_pretrained>
train = df[df['target']!=-1]
test = df[df['target']==-1]
pt_model = AutoModel.from_pretrained(electra_dir )<init_hyperparams>
def print_metrics(y_true, y_pred):
print(f"Accuracy: {accuracy_score(y_true, y_pred)}")
 print(f"MCC: {matthews_corrcoef(y_true, y_pred)}")
 print(f"F1: {f1_score(y_true, y_pred)}
" )
class SpanModel(nn.Module):
def __init__(self,pt_model):
super().__init__()
self.model = pt_model
self.drop_out = nn.Dropout(0.5)
 self.qa_outputs1c = torch.nn.Conv1d(768*2, 128, 2)
 self.qa_outputs2c = torch.nn.Conv1d(768*2, 128, 2)
 self.qa_outputs1 = nn.Linear(128, 1)
 self.qa_outputs2 = nn.Linear(128, 1)
 def forward(self, input_ids, attention_mask, token_type_ids, offsets = None):
_, hidden_states = self.model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 out = torch.cat(( hidden_states[-1], hidden_states[-2]), dim=-1)
 out = self.drop_out(out)
 out = torch.nn.functional.pad(out.transpose(1,2),(1, 0))
out1 = self.qa_outputs1c(out ).transpose(1,2)
 out2 = self.qa_outputs2c(out ).transpose(1,2)
 start_logits = self.qa_outputs1(self.drop_out(out1)).squeeze(-1)
 end_logits = self.qa_outputs2(self.drop_out(out2)).squeeze(-1)
 return start_logits, end_logits<compute_train_metric>
final_preds = np.zeros(( len(test)))
 kfold = StratifiedKFold(n_splits=NUM_SPLITS, shuffle=True, random_state=SEED)
 for fold,(train_index, valid_index)in enumerate(kfold.split(train["text"], train["target"])) :
print("*"*60)
 print("*"+" "*26+f"FOLD {fold+1}"+" "*26+"*")
 print("*"*60, end="
")
 X_train = train.iloc[train_index, :].reset_index(drop=True)
 X_valid = train.iloc[valid_index, :].reset_index(drop=True)
 y_train = X_train['target']
y_valid = X_valid['target']
train_term_doc = vec.fit_transform(X_train["text"])
 valid_term_doc = vec.transform(X_valid["text"])
 test_term_doc = vec.transform(test["text"])
 nbsvm = NBSVMClassifier(**kwargs)
 nbsvm.fit(train_term_doc, y_train)
 valid_preds = nbsvm.predict(valid_term_doc)
 print_metrics(y_valid, valid_preds)
 test_preds = nbsvm.predict(test_term_doc)
 final_preds += test_preds
class CELoss(Module):
def __init__(self, loss_fn = nn.CrossEntropyLoss()):
self.loss_fn = loss_fn
def forward(self, inputs, start_targets, end_targets):
start_logits, end_logits = inputs
logits = torch.cat([start_logits, end_logits] ).contiguous()
targets = torch.cat([start_targets, end_targets] ).contiguous()
return self.loss_fn(logits, targets )<string_transform>
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<init_hyperparams>
submission["target"] = final_preds/NUM_SPLITS
submission["target"] = submission["target"].apply(lambda x: 1 if x>=0.5 else 0 )
class JaccardScore(Callback):
"Stores predictions and targets to perform calculations on epoch end."
def __init__(self, valid_ds):
self.valid_ds = valid_ds
self.context_text = valid_ds.df.text
self.answer_text = valid_ds.df.selected_text
def on_epoch_begin(self, **kwargs):
self.jaccard_scores = []
self.valid_ds_idx = 0
def on_batch_end(self, last_input:Tensor, last_output:Tensor, last_target:Tensor, **kwargs):
input_ids = last_input[0]
attention_masks = last_input[1].bool()
token_type_ids = last_input[2].bool()
offsets = last_input[3]
start_logits, end_logits = last_output
for i in range(len(input_ids)) :
_offsets = offsets[i]
start_idx, end_idx = torch.argmax(start_logits[i]), torch.argmax(end_logits[i])
 _answer_text = self.answer_text[self.valid_ds_idx]
original_start, original_end = _offsets[start_idx][0], _offsets[end_idx][1]
pred_span = self.context_text[self.valid_ds_idx][original_start : original_end]
score = jaccard(pred_span, _answer_text)
 self.jaccard_scores.append(score)
 self.valid_ds_idx += 1
def on_epoch_end(self, last_metrics, **kwargs):
res = np.mean(self.jaccard_scores)
 return add_metrics(last_metrics, res )<split>
submission.to_csv("submission.csv", index=False )
<categorify><EOS> 
submission.to_csv("submission.csv", index=False )