Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<import_modules>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
import os
import warnings
import logging
from typing import Mapping, List
from pprint import pprint
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from transformers import AutoConfig, AutoModel, AutoTokenizer
from catalyst.dl import SupervisedRunner
from catalyst.dl.callbacks import AccuracyCallback, F1ScoreCallback, OptimizerCallback
from catalyst.dl.callbacks import CheckpointCallback, InferCallback
from catalyst.utils import set_global_seed, prepare_cudnn<define_variables>
sns.set_style("whitegrid")
 notebookstart = time.time()
pd.options.display.max_colwidth = 500
print("Tensorflow Version: ", tf.__version__)
 print("TF-Hub version: ", hub.__version__)
 print("Eager mode enabled: ", tf.executing_eagerly())
 print("GPU available: ", tf.test.is_gpu_available() )
MODEL_NAME = 'distilbert-base-uncased'
LOG_DIR = "./logdir"
NUM_EPOCHS = 5
BATCH_SIZE = 128
MAX_SEQ_LENGTH = 64
LEARN_RATE = 5e-5
ACCUM_STEPS = 1
SEED = 42<load_from_csv>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )<feature_engineering>
def build_model(bert_layer, out_channels, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(out_channels, activation='softmax' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='categorical_crossentropy',
metrics=['accuracy'])
 return model
train['text'] = train['text'].apply(lambda x: x.lower())
 train['len'] = train['text'].apply(lambda x: len(x.split()))<split>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
X_train, X_test, y_train, y_test = train_test_split(train, train.target.values, random_state = 42, test_size = 0.2, stratify=train.target.values )<categorify>
MAX_LEN = 64*3
BATCH_SIZE = 8
EPOCHS = 15
SEED = 42
NROWS = None
TEXTCOL = "text"
TARGETCOL = "author"
NCLASS = 3
class TextClassificationDataset(Dataset):

 def __init__(self,
texts: List[str],
labels: List[str] = None,
label_dict: Mapping[str, int] = None,
max_seq_length: int = 512,
model_name: str = 'distilbert-base-uncased'):

 self.texts = texts
self.labels = labels
self.label_dict = label_dict
self.max_seq_length = max_seq_length
if self.label_dict is None and labels is not None:
self.label_dict = dict(zip(sorted(set(labels)) ,
range(len(set(labels)))))
 self.tokenizer = AutoTokenizer.from_pretrained(model_name)
 logging.getLogger(
 "transformers.tokenization_utils" ).setLevel(logging.FATAL)
 self.sep_vid = self.tokenizer.vocab["[SEP]"]
self.cls_vid = self.tokenizer.vocab["[CLS]"]
self.pad_vid = self.tokenizer.vocab["[PAD]"]
def __len__(self):

 return len(self.texts)
 def __getitem__(self, index)-> Mapping[str, torch.Tensor]:

 x = self.texts[index]
x_encoded = self.tokenizer.encode(
 x,
add_special_tokens=True,
max_length=self.max_seq_length,
return_tensors="pt",
).squeeze(0)
 true_seq_length = x_encoded.size(0)
 pad_size = self.max_seq_length - true_seq_length
pad_ids = torch.Tensor([self.pad_vid] * pad_size ).long()
x_tensor = torch.cat(( x_encoded, pad_ids))
mask = torch.ones_like(x_encoded, dtype=torch.int8)
 mask_pad = torch.zeros_like(pad_ids, dtype=torch.int8)
 mask = torch.cat(( mask, mask_pad))
output_dict = {
"features": x_tensor,
'attention_mask': mask
}
if self.labels is not None:
y = self.labels[index]
y_encoded = torch.Tensor(
 [self.label_dict.get(y, -1)]
).long().squeeze(0)
 output_dict["targets"] = y_encoded
return output_dict<create_dataframe>
train = pd.read_csv(".. /input/spooky-author-identification/train.zip")
 test = pd.read_csv(".. /input/spooky-author-identification/test.zip")
 testdex = test.id
submission = pd.read_csv(".. /input/spooky-author-identification/sample_submission.zip")
 sub_cols = submission.columns
print("Train Shape: {} Rows, {} Columns".format(*train.shape))
print("Test Shape: {} Rows, {} Columns".format(*test.shape))
length_info = [len(x)for x in np.concatenate([train[TEXTCOL].values, test[TEXTCOL].values])]
print("Train Sequence Length - Mean {:.1f} +/- {:.1f}, Max {:.1f}, Min {:.1f}".format(
 np.mean(length_info), np.std(length_info), np.max(length_info), np.min(length_info)) )
train_dataset = TextClassificationDataset(
 texts=X_train['text'].values.tolist() ,
labels=y_train.tolist() ,
label_dict=None,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME 
)
 valid_dataset = TextClassificationDataset(
 texts=X_test['text'].values.tolist() ,
labels=y_test.tolist() ,
label_dict=train_dataset.label_dict,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME 
)
 test_dataset = TextClassificationDataset(
 texts=test['text'].values.tolist() ,
labels=None,
label_dict=None,
max_seq_length=MAX_SEQ_LENGTH,
model_name=MODEL_NAME
)<define_variables>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
NUM_CLASSES = len(train_dataset.label_dict )<load_pretrained>
train_input = bert_encode(train[TEXTCOL].values, tokenizer, max_len=MAX_LEN)
 test_input = bert_encode(test[TEXTCOL].values, tokenizer, max_len=MAX_LEN)
 label_mapper = {name: i for i,name in enumerate(set(train[TARGETCOL].values)) }
num_label = np.vectorize(label_mapper.get )(train[TARGETCOL].values)
 train_labels = to_categorical(num_label )
train_val_loaders = {
"train": DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
shuffle=True),
"valid": DataLoader(dataset=valid_dataset,
batch_size=BATCH_SIZE,
shuffle=False)
 }<load_pretrained>
model = build_model(bert_layer, NCLASS, max_len=MAX_LEN)
 model.summary()
class DistilBertForSequenceClassification(nn.Module):

 def __init__(self, pretrained_model_name: str, num_classes: int = None):

 super().__init__()
config = AutoConfig.from_pretrained(
 pretrained_model_name, num_labels=num_classes)
 self.distilbert = AutoModel.from_pretrained(pretrained_model_name,
config=config)
 self.pre_classifier = nn.Linear(config.dim, config.dim)
 self.classifier = nn.Linear(config.dim, num_classes)
 self.dropout = nn.Dropout(config.seq_classif_dropout)
 def forward(self, features, attention_mask=None, head_mask=None):

 assert attention_mask is not None, "attention mask is none"
distilbert_output = self.distilbert(input_ids=features,
attention_mask=attention_mask,
head_mask=head_mask)
 hidden_state = distilbert_output[0]
pooled_output = hidden_state[:, 0]
pooled_output = self.pre_classifier(pooled_output)
 pooled_output = nn.ReLU()(pooled_output)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 return logits<choose_model_class>
checkpoint = callbacks.ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001,
patience=4, verbose=1, mode='min', baseline=None,
restore_best_weights=False)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=EPOCHS,
callbacks=[checkpoint, es],
batch_size=BATCH_SIZE
)
model = DistilBertForSequenceClassification(pretrained_model_name=MODEL_NAME,
num_classes=NUM_CLASSES )<choose_model_class>
import matplotlib.pyplot as plt
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters() , lr=LEARN_RATE)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer )<set_options>
model.load_weights('model.h5')
 test_pred = model.predict(test_input)
 test_pred.shape
os.environ['CUDA_VISIBLE_DEVICES'] = "0"
set_global_seed(SEED)
 prepare_cudnn(deterministic=True )<choose_model_class>
submission = pd.DataFrame(test_pred, columns=label_mapper.keys())
 submission['id'] = testdex
submission = submission[sub_cols]
submission.to_csv('submission_bert.csv', index=False)
 print(submission.shape )
<set_options><EOS> 
print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))