Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<choose_model_class>
import numpy as np
import pandas as pd
from fastai.text.all import *
import re
def get_model() :
input1_pre = Input(shape=(maxlen,))
embed_layer1_pre = Embedding(max_features,
embed_size,
input_length=maxlen,
weights=[embedding_matrix],
trainable=False )(input1_pre)
 embed_layer1_pre = SpatialDropout1D(0.4 )(embed_layer1_pre)
 x_pre = Bidirectional(CuDNNGRU(128, return_sequences=True))(embed_layer1_pre)
 capsule_pre = Capsule(num_capsule=10, dim_capsule=16, routings=5,share_weights=True )(x_pre)
 capsule_pre = GlobalMaxPooling1D()(capsule_pre)
 capsule_pre = Dropout(0.25 )(capsule_pre)
 input1_post = Input(shape=(maxlen,))
embed_layer1_post = Embedding(max_features,
embed_size,
input_length=maxlen,
weights=[embedding_matrix],
trainable=False )(input1_post)
 embed_layer1_post = SpatialDropout1D(0.4 )(embed_layer1_post)
 x_post = Bidirectional(CuDNNGRU(128, return_sequences=True))(embed_layer1_post)
 capsule_post = Capsule(num_capsule=10, dim_capsule=16, routings=5,share_weights=True )(x_post)
 capsule_post = GlobalMaxPooling1D()(capsule_post)
 capsule_post = Dropout(0.25 )(capsule_post)
 concat = concatenate([capsule_pre,capsule_post])
 output = Dense(6, activation='sigmoid' )(concat)
 model = Model(inputs=[input1_pre,input1_post], outputs=output)
 model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=1e-3,decay=0),
metrics=['accuracy'])
 return model<train_model>
dir_path = "/kaggle/input/nlp-getting-started/"
train_df = pd.read_csv(dir_path + "train.csv")
 test_df = pd.read_csv(dir_path + "test.csv" )
file_path = "capsule_val0.05.h5"
model = get_model()
checkpoint = ModelCheckpoint(file_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
 early = EarlyStopping(monitor="val_loss", mode="min", patience=3)
 callbacks_list = [checkpoint, early]
hist = model.fit([data, data_post], y, epochs=10, batch_size=128, shuffle=True, validation_split=0.05, callbacks = callbacks_list,
verbose=1)
 best_score = min(hist.history['val_loss'])
 print(best_score )<train_model>
train_df = train_df.drop(columns=["id", "keyword", "location"] )
test_predicts_list = []
def train_folds(data,data_post, y,fold_count=10):
print("Starting to train models...")
 fold_size = len(data)// fold_count
models = []
for fold_id in range(0, fold_count):
fold_start = fold_size * fold_id
fold_end = fold_start + fold_size
if fold_id == fold_size - 1:
fold_end = len(data)
 print("Fold {0}".format(fold_id))
train_x = np.concatenate([data[:fold_start], data[fold_end:]])
 train_xp = np.concatenate([data_post[:fold_start], data_post[fold_end:]])
 train_y = np.concatenate([y[:fold_start], y[fold_end:]])
 val_x = data[fold_start:fold_end]
val_xp = data_post[fold_start:fold_end]
val_y = y[fold_start:fold_end]
file_path="capsule_fold{0}.h5".format(fold_id)
 model = get_model()
checkpoint = ModelCheckpoint(file_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
 early = EarlyStopping(monitor="val_loss", mode="min", patience=3)
 RocAuc = RocAucEvaluation(validation_data=([val_x, val_xp], val_y), interval=1)
 callbacks_list = [checkpoint, early,RocAuc]
hist = model.fit([train_x, train_xp], train_y, epochs=15, batch_size=128, shuffle=True,
validation_data=([val_x, val_xp], val_y), callbacks = callbacks_list, verbose=1)
 model.load_weights(file_path)
 best_score = min(hist.history['val_loss'])
 print("Fold {0} loss {1}".format(fold_id, best_score))
print("Predicting validation...")
 val_predicts_path = "capsule_val_predicts{0}.npy".format(fold_id)
 val_predicts = model.predict([val_x, val_xp], batch_size=1024, verbose=1)
 np.save(val_predicts_path, val_predicts)
 print("Predicting results...")
 test_predicts_path = "capsule_test_predicts{0}.npy".format(fold_id)
 test_predicts = model.predict([test_data, test_data_post], batch_size=1024, verbose=1)
 test_predicts_list.append(test_predicts)
 np.save(test_predicts_path, test_predicts )<concatenate>
train_df["target"].value_counts()
train_folds(data, data_post, y )<save_to_csv>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 train_df["text"] = train_df["text"].apply(remove_URL)
 test_df["text"] = test_df["text"].apply(remove_URL )
CLASSES = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
test_predicts_am = np.zeros(test_predicts_list[0].shape)
 for fold_predict in test_predicts_list:
test_predicts_am += fold_predict
test_predicts_am =(test_predicts_am / len(test_predicts_list))
test_ids = test_df["id"].values
test_ids = test_ids.reshape(( len(test_ids), 1))
test_predicts_am = pd.DataFrame(data=test_predicts_am, columns=CLASSES)
 test_predicts_am["id"] = test_ids
test_predicts_am = test_predicts_am[["id"] + CLASSES]
test_predicts_am.to_csv("10fold_capsule_am.csv", index=False )<import_modules>
def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 train_df["text"] = train_df["text"].apply(remove_html)
 test_df["text"] = test_df["text"].apply(remove_html )
import os
import warnings
import logging
from typing import Mapping, List, Union, Optional, Tuple
from pprint import pprint
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from transformers import AutoConfig, AutoModel, AutoTokenizer
import catalyst
from catalyst.dl import SupervisedRunner
from catalyst.dl.callbacks import OptimizerCallback, CheckpointCallback, InferCallback
from catalyst.dl.utils import plot_metrics
from catalyst.utils import set_global_seed, prepare_cudnn<import_modules>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train_df["text"] = train_df["text"].apply(remove_emoji)
 test_df["text"] = test_df["text"].apply(remove_emoji )
catalyst.__version__<define_variables>
train_df["text"].apply(lambda x:len(x.split())).plot(kind="hist");
MODEL_NAME = 'distilbert-base-uncased'
LOG_DIR = "./logdir"
NUM_EPOCHS = 3
BATCH_SIZE = 96
MAX_SEQ_LENGTH = 256
LEARN_RATE = 3e-5
ACCUM_STEPS = 4
SEED = 17<define_variables>
from transformers import AutoTokenizer, AutoModelForSequenceClassification
PATH_TO_DATA = '.. /input/jigsaw-toxic-comment-classification-challenge/'
TEXT_FIELD = 'comment_text'
TARGET_FIELDS = ['toxic','severe_toxic','obscene','threat','insult', 'identity_hate']
NUM_CLASSES = len(TARGET_FIELDS)
 PRED_THRES = 0.4<load_from_csv>
tokenizer = AutoTokenizer.from_pretrained("roberta-large" )
train_df = pd.read_csv(PATH_TO_DATA + 'train.csv.zip', index_col='id')
 test_df = pd.read_csv(PATH_TO_DATA + 'test.csv.zip', index_col='id' )<split>
train_tensor = tokenizer(list(train_df["text"]), padding="max_length",
truncation=True, max_length=30,
return_tensors="pt")["input_ids"]
X_train, X_valid, y_train, y_valid = train_test_split(train_df[TEXT_FIELD],
train_df[TARGET_FIELDS],
test_size=0.1,
random_state=17)
 X_test = test_df[TEXT_FIELD]<import_modules>
class TweetDataset:
def __init__(self, tensors, targ, ids):
self.text = tensors[ids, :]
self.targ = targ[ids].reset_index(drop=True)
 def __len__(self):
return len(self.text)
 def __getitem__(self, idx):
t = self.text[idx]
y = self.targ[idx]
return t, tensor(y )
class TextClassificationDataset(Dataset):

 def __init__(self,
texts: List[str],
labels: np.ndarray = None,
max_seq_length: int = 512,
model_name: str = 'distilbert-base-uncased'):

 self.texts = texts
self.labels = labels
self.max_seq_length = max_seq_length
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
 logging.getLogger(
 "transformers.tokenization_utils" ).setLevel(logging.FATAL)
 def __len__(self):

 return len(self.texts)
 def __getitem__(self, index)-> Mapping[str, torch.Tensor]:

 x = self.texts[index]
output_dict = self.tokenizer.encode_plus(
 x,
add_special_tokens=True,
pad_to_max_length=True,
max_length=self.max_seq_length,
return_tensors="pt",
return_attention_mask=True 
)
 output_dict['features'] = output_dict['input_ids'].squeeze(0)
 del output_dict['input_ids']
if self.labels is not None:
output_dict["targets"] = torch.from_numpy(self.labels[index] ).float()
return output_dict<create_dataframe>
train_ids, valid_ids = RandomSplitter()(train_df)
 target = train_df["target"]
train_ds = TweetDataset(train_tensor, target, train_ids)
 valid_ds = TweetDataset(train_tensor, target, valid_ids)
 train_dl = DataLoader(train_ds, bs=64)
 valid_dl = DataLoader(valid_ds, bs=512)
 dls = DataLoaders(train_dl, valid_dl ).to("cuda" )
train_dataset = TextClassificationDataset(
 texts=X_train.values.tolist() ,
labels=y_train.values,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME 
)
 valid_dataset = TextClassificationDataset(
 texts=X_valid.values.tolist() ,
labels=y_valid.values,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME 
)
 test_dataset = TextClassificationDataset(
 texts=X_test.values.tolist() ,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME
)<load_pretrained>
bert = AutoModelForSequenceClassification.from_pretrained("roberta-large", num_labels=2 ).train().to("cuda")
 class BertClassifier(Module):
def __init__(self, bert):
self.bert = bert
def forward(self, x):
return self.bert(x ).logits
model = BertClassifier(bert )
train_val_loaders = {
"train": DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
shuffle=True),
"valid": DataLoader(dataset=valid_dataset,
batch_size=BATCH_SIZE,
shuffle=False)
 }<load_pretrained>
learn = Learner(dls, model, metrics=[accuracy, F1Score() ] ).to_fp16()
learn.lr_find()
class BertForSequenceClassification(nn.Module):

 def __init__(self, pretrained_model_name: str, num_classes: int = None, dropout: float = 0.3):

 super().__init__()
config = AutoConfig.from_pretrained(
 pretrained_model_name, num_labels=num_classes)
 self.model = AutoModel.from_pretrained(pretrained_model_name,
config=config)
 self.classifier = nn.Linear(config.hidden_size, num_classes)
 self.dropout = nn.Dropout(dropout)
 def forward(self, features, attention_mask=None, head_mask=None):

 assert attention_mask is not None, "attention mask is none"
bert_output = self.model(input_ids=features,
attention_mask=attention_mask,
head_mask=head_mask)
 seq_output = bert_output[0]
pooled_output = seq_output.mean(axis=1)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 return logits<choose_model_class>
learn.fit_one_cycle(3, lr_max=1e-5 )
model = BertForSequenceClassification(pretrained_model_name=MODEL_NAME,
num_classes=NUM_CLASSES )<choose_model_class>
preds, targs = learn.get_preds()
min_threshold = None
max_f1 = -float("inf")
 thresholds = np.linspace(0.3, 0.7, 50)
 for threshold in thresholds:
f1 = f1_score(targs, F.softmax(preds, dim=1)[:, 1]>threshold)
 if f1 > max_f1:
min_threshold = threshold
min_f1 = f1
print(f"threshold:{threshold:.4f} - f1:{f1:.4f}" )
criterion = torch.nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters() , lr=LEARN_RATE)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer )<categorify>
test_tensor = tokenizer(list(test_df["text"]),
padding="max_length",
truncation=True,
max_length=30,
return_tensors="pt")["input_ids"]
def preprocess_multi_label_metrics(
 outputs: torch.Tensor,
targets: torch.Tensor,
weights: Optional[torch.Tensor] = None, 
)-> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:

 if not torch.is_tensor(outputs):
outputs = torch.from_numpy(outputs)
 if not torch.is_tensor(targets):
targets = torch.from_numpy(targets)
 if weights is not None:
if not torch.is_tensor(weights):
weights = torch.from_numpy(weights)
 weights = weights.squeeze()
if outputs.dim() == 1:
outputs = outputs.view(-1, 1)
 else:
assert outputs.dim() == 2,(
 "wrong `outputs` size "
"(should be 1D or 2D with one column per class)" 
)
 if targets.dim() == 1:
targets = targets.view(-1, 1)
 else:
assert targets.dim() == 2,(
 "wrong `targets` size "
"(should be 1D or 2D with one column per class)" 
)
 if weights is not None:
assert weights.dim() == 1, "Weights dimension should be 1"
assert weights.numel() == targets.size(
 0 
), "Weights dimension 1 should be the same as that of target"
assert torch.min(weights)>= 0, "Weight should be non-negative only"
assert torch.equal(
 targets ** 2, targets 
), "targets should be binary(0 or 1)"
return outputs, targets, weights
def multi_label_accuracy(
 outputs: torch.Tensor,
targets: torch.Tensor,
threshold: Union[float, torch.Tensor],
activation: Optional[str] = None, 
)-> torch.Tensor:

 outputs, targets, _ = preprocess_multi_label_metrics(
 outputs=outputs, targets=targets 
)
 activation_fn = get_activation_fn(activation)
 outputs = activation_fn(outputs)
 outputs =(outputs > threshold ).long()
output =(targets.long() == outputs.long() ).sum().float() / np.prod(
 targets.shape 
)
 return output
class MultiLabelAccuracyCallback(MetricCallback):

 def __init__(
 self,
input_key: str = "targets",
output_key: str = "logits",
prefix: str = "multi_label_accuracy",
threshold: float = None,
activation: str = "Sigmoid", 
):

 super().__init__(
 prefix=prefix,
metric_fn=multi_label_accuracy,
input_key=input_key,
output_key=output_key,
threshold=threshold,
activation=activation,
)<set_options>
class TestDS:
def __init__(self, tensors):
self.tensors = tensors
def __len__(self):
return len(self.tensors)
 def __getitem__(self, idx):
t = self.tensors[idx]
return t, tensor(0)
 test_dl = DataLoader(TestDS(test_tensor), bs=128 )
os.environ['CUDA_VISIBLE_DEVICES'] = "0"
set_global_seed(SEED)
 prepare_cudnn(deterministic=True )<train_model>
test_preds = learn.get_preds(dl=test_dl )
<set_options><EOS> 
sub = pd.read_csv(dir_path + "sample_submission.csv")
 prediction =(F.softmax(test_preds[0], dim=1)[:, 1]>min_threshold ).int()
sub = pd.read_csv(dir_path + "sample_submission.csv")
 sub["target"] = prediction
sub.to_csv("submission.csv", index=False )