Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<set_options>
import pandas as pd
import numpy as np
import tensorflow as tf
import tokenizers
import transformers
from tqdm import tqdm
from sklearn import metrics
from sklearn.model_selection import KFold
import re
def is_interactive() :
return 'runtime' in get_ipython().config.IPKernelApp.connection_file
print('Interactive?', is_interactive() )<import_modules>
MAX_LEN = 64
TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE = 8
LEARNING_RATE = 3e-5
EPOCHS = 3
TRAINING_FILE = ".. /input/nlp-getting-started/train.csv"
TEST_FILE = ".. /input/nlp-getting-started/test.csv"
ROBERTA_PATH = ".. /input/tf-roberta"
roberta_tokenizer = transformers.RobertaTokenizer.from_pretrained('roberta-base', lower_case=True )
<define_variables>
train = pd.read_csv(TRAINING_FILE ).fillna('')
 test = pd.read_csv(TEST_FILE ).fillna('')
 print("Training samples: {}".format(train.shape[0]))
print("Test samples: {}".format(test.shape[0]))
train.head(5 )
tf_env="tensorflow"<define_variables>
url = "Great paper by Kalchbrenner https://arxiv.org/pdf/1404.2188.pdf?utm_medium=App.net&utm_source=PourOver"
def remove_urls(text):
re_url = re.compile(r'https?://\S+|www\.\S+')
 return re_url.sub('', text ).strip()
print(remove_urls(url))
train['text'] = train['text'].apply(lambda x : remove_urls(x))
test['text'] = test['text'].apply(lambda x : remove_urls(x))
fit_verbocity=1
model=tf_env
nrows=None
interativeEnvironment=is_interactive()
interativeEnvironment=False
if localEnvironment:
warnings.warn("Nrows limited")
 pass
elif interativeEnvironment:
pass
else:
fit_verbocity=2
pooling_mode_cls_token="CLS_TOKEN"
pooling_mode_fc="FC"
pooling_mode=pooling_mode_fc
fc_dims=[]
trainable_transformer=False
use_dann=False
dann_lambda=-1
use_parcor=False
parcor_lambda=1
use_augmented_data=False
use_lowercase_data=False
use_finetuning=True
use_validation_during_pretraining=True
use_translated_data=True
use_pretraining=True
load_model=None<feature_engineering>
html =
def remove_html(text):
re_html = re.compile(r'<.*?>')
 return re_html.sub('', text)
 print(remove_html(html))
train['text'] = train['text'].apply(lambda x : remove_html(x))
test['text'] = test['text'].apply(lambda x : remove_html(x))
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True 
<normalization>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text ).strip()
print(remove_emoji("Difficult kernel 😔😔"))
train['text'] = train['text'].apply(lambda x: remove_emoji(x))
test['text'] = test['text'].apply(lambda x: remove_emoji(x))
@tf.custom_gradient
def grad_reverse(x):
y = tf.identity(x)
 def custom_grad(dy):
return tf.math.scalar_mul(-1,dy)
 return y, custom_grad
class GradReverse(tf.keras.layers.Layer):
def __init__(self):
super().__init__()
def call(self, x):
return grad_reverse(x )<set_options>
punct = 'Cristiano.is
def remove_puncts(text):
table = str.maketrans('','',string.punctuation)
 return text.translate(table ).strip()
print(remove_puncts(punct))
train['text'] = train['text'].apply(lambda x: remove_puncts(x))
test['text'] = test['text'].apply(lambda x: remove_puncts(x))
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
num_classes = train.target.nunique()
num_classes
AUTO = tf.data.experimental.AUTOTUNE
if not localEnvironment and not IN_COLAB:
GCS_DS_PATH = KaggleDatasets().get_gcs_path("jigsaw-multilingual-toxic-comment-classification")
 NEXAMPLESPEREPOCH=240000
EPOCHS = 10
if strategy.num_replicas_in_sync==1:
BATCH_SIZE = 4 * strategy.num_replicas_in_sync
else:
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
if localEnvironment:
MODEL='jplu/tf-xlm-roberta-base'
if IN_COLAB:
MODEL='jplu/tf-xlm-roberta-base'
print("BATCH_SIZE: ", BATCH_SIZE )<define_variables>
n_train = train.shape[0]
input_ids = np.ones(( n_train, MAX_LEN), dtype='int32')
 mask = np.zeros(( n_train, MAX_LEN), dtype='int32')
 for k in range(train.shape[0]):
text = train.loc[k, 'text']
output = roberta_tokenizer.encode_plus(text, max_length=MAX_LEN, pad_to_max_length=True)
 input_ids[k] = output['input_ids']
mask[k] = output["attention_mask"]
if IN_COLAB:
project_id="global-sun-279412"<set_options>
n_test = test.shape[0]
input_ids_t = np.ones(( n_test, MAX_LEN), dtype='int32')
 mask_t = np.zeros(( n_test, MAX_LEN), dtype='int32')
 for k in range(test.shape[0]):
text = test.loc[k, 'text']
output = roberta_tokenizer.encode_plus(text, max_length=MAX_LEN, pad_to_max_length=True)
 input_ids_t[k] = output['input_ids']
mask_t[k] = output["attention_mask"]
!gcloud config set project {project_id}<define_variables>
def create_dataset() :
xtrain = [input_ids, mask]
xtest = [input_ids_t, mask_t]
ytrain = tf.keras.utils.to_categorical(train['target'].values.reshape(-1, 1))
return xtrain, ytrain, xtest
bucket_name="jigsaw-tfrecords"<set_options>
xtrain, ytrain, xtest = create_dataset()
print("X train : {0}".format(len(xtrain[0])))
 print("Y train : {0}".format(len(ytrain)))
 print("X test : {0}".format(len(xtest[0])) )
if IN_COLAB:
auth.authenticate_user()<define_variables>
def build_model() :
roberta = transformers.TFRobertaForSequenceClassification.from_pretrained('roberta-base')
 optim = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE, clipnorm=2.0)
 loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)
 roberta.compile(loss='binary_crossentropy', optimizer=optim, metrics=['accuracy'])
 return roberta
if localEnvironment:
root =".. "
elif IN_COLAB:
root='gs://jigsaw-tfrecords/'
else:
root="/kaggle"
root<define_variables>
Folds = 5
kfold = KFold(n_splits=Folds)
 predictions = list()
for i,(train_idx, test_idx)in enumerate(kfold.split(xtrain[0])) :
xtrain_fold = [xtrain[i][train_idx] for i in range(len(xtrain)) ]
xvalid_fold = [xtrain[i][test_idx] for i in range(len(xtrain)) ]
ytrain_fold = ytrain[train_idx]
yvalid_fold = ytrain[test_idx]
positive = train.iloc[train_idx, :].target.value_counts() [0]
negative = train.iloc[train_idx, :].target.value_counts() [1]
pos_weight = positive /(positive + negative)
 neg_weight = negative /(positive + negative)
 class_weight = [{0:pos_weight, 1:neg_weight}, {0:neg_weight, 1:pos_weight}]
tf.keras.backend.clear_session()
roberta = build_model()
roberta.fit(xtrain_fold, ytrain_fold,
batch_size=TRAIN_BATCH_SIZE,
epochs=EPOCHS,
class_weight=class_weight,
validation_data=(xvalid_fold, yvalid_fold))
val_preds = roberta.predict(xvalid_fold, batch_size=VALID_BATCH_SIZE, verbose=1)
 val_preds = np.argmax(val_preds, axis=1 ).flatten()
print(metrics.accuracy_score(train.iloc[test_idx, :].target.values, val_preds))
preds = roberta.predict(xtest, batch_size=TRAIN_BATCH_SIZE, verbose=1)
 predictions.append(preds)
<feature_engineering><EOS> 
sample_submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 predictions = np.average(predictions, axis=0)
 predictions = np.argmax(predictions, axis=1 ).flatten()
sample_submission['target'] = predictions
sample_submission['target'].value_counts()
sample_submission.to_csv('submission.csv', index=False )