Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<import_modules>
!pip install transformers
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer, SentencePieceBPETokenizer<categorify>
print('fastai version :', fastai.__version__)
 print('transformers version :', transformers.__version__ )
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<set_options>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
MODEL_CLASSES = {
'bert':(BertForSequenceClassification, BertTokenizer, BertConfig),
'xlnet':(XLNetForSequenceClassification, XLNetTokenizer, XLNetConfig),
'xlm':(XLMForSequenceClassification, XLMTokenizer, XLMConfig),
'roberta':(RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig),
'distilbert':(DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertConfig)}
seed = 40
use_fp16 = False
bs = 64
model_type = 'roberta'
pretrained_model_name = 'roberta-base'
model_class, tokenizer_class, config_class = MODEL_CLASSES[model_type]
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 128<load_pretrained>
def seed_all(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_all(seed )
tokenizer = transformers.AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large')
 tokenizer.save_pretrained('.')
<load_from_csv>
class TransformersBaseTokenizer(BaseTokenizer):

 def __init__(self, pretrained_tokenizer: PreTrainedTokenizer, model_type = 'bert', **kwargs):
self._pretrained_tokenizer = pretrained_tokenizer
self.max_seq_len = pretrained_tokenizer.max_len
self.model_type = model_type
def __call__(self, *args, **kwargs):
return self
def tokenizer(self, t:str)-> List[str]:

 CLS = self._pretrained_tokenizer.cls_token
SEP = self._pretrained_tokenizer.sep_token
if self.model_type in ['roberta']:
tokens = self._pretrained_tokenizer.tokenize(t, add_prefix_space=True)[:self.max_seq_len - 2]
tokens = [CLS] + tokens + [SEP]
else:
tokens = self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2]
if self.model_type in ['xlnet']:
tokens = tokens + [SEP] + [CLS]
else:
tokens = [CLS] + tokens + [SEP]
return tokens
transformer_tokenizer = tokenizer_class.from_pretrained(pretrained_model_name)
 transformer_base_tokenizer = TransformersBaseTokenizer(pretrained_tokenizer = transformer_tokenizer, model_type = model_type)
 fastai_tokenizer = Tokenizer(tok_func = transformer_base_tokenizer, pre_rules=[], post_rules=[] )
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
<concatenate>
class TransformersVocab(Vocab):
def __init__(self, tokenizer: PreTrainedTokenizer):
super(TransformersVocab, self ).__init__(itos = [])
 self.tokenizer = tokenizer
def numericalize(self, t:Collection[str])-> List[int]:
"Convert a list of tokens `t` to their ids."
return self.tokenizer.convert_tokens_to_ids(t)
 def textify(self, nums:Collection[int], sep=' ')-> List[str]:
"Convert a list of `nums` to their tokens."
nums = np.array(nums ).tolist()
return sep.join(self.tokenizer.convert_ids_to_tokens(nums)) if sep is not None else self.tokenizer.convert_ids_to_tokens(nums)
 def __getstate__(self):
return {'itos':self.itos, 'tokenizer':self.tokenizer}
def __setstate__(self, state:dict):
self.itos = state['itos']
self.tokenizer = state['tokenizer']
self.stoi = collections.defaultdict(int,{v:k for k,v in enumerate(self.itos)} )
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=150000, random_state=0)
 ] )<count_values>
transformer_vocab = TransformersVocab(tokenizer = transformer_tokenizer)
 numericalize_processor = NumericalizeProcessor(vocab=transformer_vocab)
 tokenize_processor = TokenizeProcessor(tokenizer=fastai_tokenizer,
include_bos=False,
include_eos=False)
 transformer_processor = [tokenize_processor, numericalize_processor]
train.toxic.value_counts()<categorify>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
def fast_encode_xlm(texts, tokenizer, chunk_size=256, maxlen=512):

 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.batch_encode_plus(text_chunk, pad_to_max_length = True, max_length = maxlen)
 all_ids.extend(np.array(encs.input_ids))
return np.array(all_ids )<prepare_x_and_y>
twt = TweetTokenizer(strip_handles=True)
 def tweets(r):
s = ' '.join(twt.tokenize(r['text']))
s = re.sub(r'http\S+', '', s)
 s = re.sub(r'https\S+', '', s)
 return s
x_train = fast_encode_xlm(train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode_xlm(valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode_xlm(test.content.astype(str), tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<define_variables>
train['ptext'] = train.apply(tweets, axis=1)
 test['ptext'] = test.apply(tweets, axis=1 )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<choose_model_class>
pad_first = bool(model_type in ['xlnet'])
 pad_idx = transformer_tokenizer.pad_token_id
databunch =(TextList.from_df(train, cols='ptext', processor=transformer_processor)
.split_by_rand_pct(0.1, seed=seed)
.label_from_df(cols='target')
.add_test(test)
.databunch(bs=bs, pad_first=pad_first, pad_idx=pad_idx))
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_pretrained>
class CustomTransformerModel(nn.Module):
def __init__(self, transformer_model: PreTrainedModel):
super(CustomTransformerModel,self ).__init__()
self.transformer = transformer_model
def forward(self, input_ids, attention_mask=None, token_type_ids=None):
attention_mask =(input_ids!=pad_idx ).type(input_ids.type())
 token_type_ids =(attention_mask==-2 ).type(input_ids.type())
 logits = self.transformer(input_ids,
attention_mask = attention_mask, token_type_ids=token_type_ids)[0]
return logits
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFAutoModelWithLMHead 
.from_pretrained('jplu/tf-xlm-roberta-large')
)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary() 
<train_model>
config = config_class.from_pretrained(pretrained_model_name)
 config.num_labels = 2
transformer_model = model_class.from_pretrained(pretrained_model_name, config = config)
 custom_transformer_model = CustomTransformerModel(transformer_model = transformer_model )
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
CustomAdamW = partial(AdamW, correct_bias=False)
 learner = Learner(databunch,
custom_transformer_model,
opt_func = CustomAdamW,
metrics=[accuracy, FBeta(beta=1)])
 learner.callbacks.append(ShowGraph(learner))
if use_fp16: learner = learner.to_fp16()
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
list_layers = [learner.model.transformer.roberta.embeddings,
learner.model.transformer.roberta.encoder.layer[0],
learner.model.transformer.roberta.encoder.layer[1],
learner.model.transformer.roberta.encoder.layer[2],
learner.model.transformer.roberta.encoder.layer[3],
learner.model.transformer.roberta.encoder.layer[4],
learner.model.transformer.roberta.encoder.layer[5],
learner.model.transformer.roberta.encoder.layer[6],
learner.model.transformer.roberta.encoder.layer[7],
learner.model.transformer.roberta.encoder.layer[8],
learner.model.transformer.roberta.encoder.layer[9],
learner.model.transformer.roberta.encoder.layer[10],
learner.model.transformer.roberta.encoder.layer[11],
learner.model.transformer.roberta.pooler]
learner.split(list_layers)
 num_groups = len(learner.layer_groups)
 print('Learner split in',num_groups,'groups')
 print(learner.layer_groups )
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False)
<load_from_csv>
learner.save('untrain' )
%matplotlib inline
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 macro = pd.read_csv('.. /input/macro.csv')
 id_test = test.id
train.sample(3)
<feature_engineering>
seed_all(seed)
 learner.load('untrain');
<categorify>
learner.freeze_to(-1)
y_train = train["price_doc"]
x_train = train.drop(["id", "timestamp", "price_doc"], axis=1)
 x_test = test.drop(["id", "timestamp"], axis=1)
 for c in x_train.columns:
if x_train[c].dtype == 'object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(x_train[c].values))
x_train[c] = lbl.transform(list(x_train[c].values))
for c in x_test.columns:
if x_test[c].dtype == 'object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(x_test[c].values))
x_test[c] = lbl.transform(list(x_test[c].values)) 
<train_model>
learner.fit_one_cycle(1,max_lr=5e-04,moms=(0.8,0.7))
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
dtrain = xgb.DMatrix(x_train, y_train)
 dtest = xgb.DMatrix(x_test )<compute_train_metric>
learner.save('first_cycle' )
cv_output = xgb.cv(xgb_params, dtrain, num_boost_round=1000, early_stopping_rounds=20,
verbose_eval=50, show_stdv=False)
 cv_output[['train-rmse-mean', 'test-rmse-mean']].plot()<train_model>
seed_all(seed)
 learner.load('first_cycle');
num_boost_rounds = len(cv_output)
 model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round= num_boost_rounds )<predict_on_test>
learner.unfreeze()
lr = 1e-5
<save_to_csv><EOS> 
learner.fit_one_cycle(2, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))