Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
print('Transformers version: ', transformers.__version__)
 print('Tensorflow version: ', tf.__version__ )
def validationFoldOnlyFilter(validation_fold_index):
def _filter(features):
return tf.math.equal(features["ShardIndex"],validation_fold_index)
 return _filter<define_variables>
randomseed=42
bertmodel='bert-large-uncased'
data_dir = '/kaggle/input/nlp-getting-started/'
train_df = pd.read_csv(data_dir+'train.csv')
 test_df = pd.read_csv(data_dir+'test.csv')
 train_df = train_df.sample(n=len(train_df), random_state=randomseed)
 sample_submission = pd.read_csv(data_dir+'sample_submission.csv')
if IN_COLAB:
TF_DS_PATH=root+"jigsaw-official"
elif not localEnvironment:
TF_DS_PATH = KaggleDatasets().get_gcs_path("jigsawaugmented")+"/tfrecords"
else:
TF_DS_PATH = ".. /tfrecords"
TF_DS_PATH<define_variables>
x_train = train_df['text']
y_train = train_df['target']
x_test = test_df['text']
file="en1_train_non_toxic_PropercaseSentences_0_4.tfrecord" 
("Sentences" not in file)or("LowercaseSentences" in file )<define_variables>
tokenizer = transformers.BertTokenizer.from_pretrained(bertmodel, do_lower_case=True )
data_files=tf.io.gfile.glob(TF_DS_PATH + "/*.tfrecord")
 if use_augmented_data:
pass
elif use_lowercase_data:
data_files=list(filter(lambda file:("Sentences" not in file)or("LowercaseSentences" in file),data_files))
else:
data_files=list(filter(lambda file:"Sentences" not in file,data_files))
data_files<define_variables>
def encode_tweet(df):
input_ids = []
for x in df:
encoded_x = tokenizer.encode(x,
add_special_tokens = True)
 input_ids.append(encoded_x)
 return input_ids
toxic_en_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non_toxic" not in file and "valid" not in file and "sharded" not in file, data_files))
toxic_en_files<define_variables>
test_input = encode_tweet(x_test)
non_toxic_en_files=list(filter(lambda file:"nen" not in file and "non_toxic" in file and "valid" not in file and "sharded" not in file, data_files))
non_toxic_en_files<define_variables>
train_input = encode_tweet(x_train )
non_toxic_nen_files=list(filter(lambda file:"non_toxic" in file and "nen1" in file, data_files))
non_toxic_nen_files<define_variables>
MAX_LEN = 84
def pad_tweets(df):
df = pad_sequences(df, maxlen=MAX_LEN, dtype="long",
value=0, truncating="post", padding="post")
 return df
toxic_nen_files=list(filter(lambda file:"non_toxic" not in file and "nen1" in file, data_files))
toxic_nen_files<define_variables>
train_input = pad_tweets(train_input)
 test_input = pad_tweets(test_input )
unknown_nen_files=list(filter(lambda file: "test" not in file and "nen2" in file, data_files))
unknown_nen_files<define_variables>
def get_att_mask(df):
attention_masks = []
for tweet in df:
att_mask = [int(token_id > 0)for token_id in tweet]
attention_masks.append(att_mask)
 return attention_masks
valid_files=list(filter(lambda file:"test" not in file and "nen1" in file and "Sentences" not in file, data_files))
valid_files<define_variables>
train_att = get_att_mask(train_input)
 test_att = get_att_mask(test_input )
if use_translated_data:
if IN_COLAB:
TRANLSATED_DATA_PATH_ONE=root+"/translated-train-bias-all-langs"
TRANLSATED_DATA_PATH_TWO=root+"/jigsaw-train-multilingual-coments-google-api"
elif not localEnvironment:
TRANSLATED_DATA_PATH = KaggleDatasets().get_gcs_path("jigsawtranslated")+"/tfrecords-translated"
else:
pass
TRANSLATED_DATA_PATH<define_variables>
tr_input, val_input, tr_label, val_label = train_test_split(train_input, y_train,
random_state=2020, test_size=0.15)
 tr_mask, val_mask, _, _ = train_test_split(train_att, y_train,
random_state=2020, test_size=0.15 )
translated_files=[]
if use_translated_data:
translated_files=tf.io.gfile.glob(TRANSLATED_DATA_PATH + "/*tfrecord")
 translated_files[:3],TRANSLATED_DATA_PATH<define_variables>
tr_input = torch.tensor(tr_input)
 val_input = torch.tensor(val_input)
 tr_label = torch.tensor(np.array(tr_label))
val_label = torch.tensor(np.array(val_label))
tr_mask = torch.tensor(tr_mask)
 val_mask = torch.tensor(val_mask)
 te_input = torch.tensor(test_input)
 te_mask = torch.tensor(test_att )
toxic_translated_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non" not in file and "valid" not in file and "sharded" not in file, translated_files))
random.choice(toxic_translated_files )<define_variables>
BATCH_SIZE = 32
train_data = TensorDataset(tr_input, tr_mask, tr_label)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler = train_sampler, batch_size = BATCH_SIZE)
 val_data = TensorDataset(val_input, val_mask, val_label)
 val_sampler = RandomSampler(val_data)
 val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size = BATCH_SIZE )
non_toxic_translated_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non" in file and "valid" not in file and "sharded" not in file, translated_files))
random.choice(non_toxic_translated_files )<define_variables>
model = BertForSequenceClassification.from_pretrained(bertmodel,
num_labels = 2,
output_attentions = False,
output_hidden_states = False )
<define_variables>
params = list(model.named_parameters())
 print('The BERT model has {:} different named parameters.
 '.format(len(params)))
 print('==== Embedding Layer ====
')
 for p in params[0:5]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== First Transformer ====
')
 for p in params[5:21]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== Output Layer ====
')
 for p in params[-4:]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
opus_langs=[['en', 'it'],
['en', 'ru'],
['en', 'tr'],
['en', 'pt_br'],
['en', 'fr'],
['en', 'pt'],
['en', 'es'],
['en', 'hi']]
if IN_COLAB:
OPUS_DS_PATH=root+"opus"
elif not localEnvironment:
OPUS_DS_PATH = KaggleDatasets().get_gcs_path("opus-for-jigsaw-sharded")
 else:
OPUS_DS_PATH = ".. /opus-processed"
OPUS_DS_PATH<define_variables>
optimizer = AdamW(model.parameters() ,
lr = 1e-5,
eps = 1e-8 
)
 epochs = 1
total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )
if IN_COLAB:
extension=".gz"
else:
extension=""
opus_files=tf.io.gfile.glob(OPUS_DS_PATH + "/*.tfrecord"+extension)
 if interativeEnvironment:
print(opus_files)
<data_type_conversions>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )
feature_description = {
'tokens': tf.io.FixedLenFeature([192], tf.int64),
'mask': tf.io.FixedLenFeature([192], tf.int64),
'toxic': tf.io.FixedLenFeature([], tf.float32),
's_toxic': tf.io.FixedLenFeature([], tf.float32),
'source':tf.io.FixedLenFeature([], tf.int64),
'toxicity_annotator_count':tf.io.FixedLenFeature([], tf.int64),
'lang':tf.io.FixedLenFeature([], tf.string),
'ShardIndex':tf.io.FixedLenFeature([], tf.int64),
}
def _parse_proto(example_proto):
example= tf.io.parse_single_example(example_proto, feature_description)
 return example
def _form_tuple(example):
y={"toxic":tf.cast(example['s_toxic'],tf.float32)}
if use_dann:
y["lang"]=tf.cast(tf.math.equal(example['lang'],'en'),tf.float32)
 if use_parcor:
y["is_same"]=example["is_same"]
return(example['tokens'],example['mask']),y
def smoother(smoothing=0.01):
def smooth(example):
label=tf.cast(example['s_toxic'],tf.float32)
 label=tf.cond(label>=1.0-smoothing,lambda :tf.ones_like(label)-smoothing,lambda :tf.cond(label<0.0+smoothing,lambda :tf.zeros_like(label)+smoothing,lambda :label))
example['s_toxic']=label
return example
return smooth
def isValidLabel(example):
example['isLabelValid']=tf.cond(example['toxic']==-1,lambda :tf.constant(0.0),lambda :tf.constant(1.0))
return example<categorify>
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))
def perFileDataset(filename):
tf.print("Retrieving from filename",filename)
 return tf.data.TFRecordDataset(filename ).map(_parse_proto)
 def perFileDatasetShuffled(filename):
return perFileDataset(filename ).shuffle(200000)
 def unsqueezeTargetDimensions(features,labels):
labels=tf.cast(labels,tf.float32)
 return features,tf.stack([tf.math.subtract(tf.constant([1.0]),labels),labels],axis=-1),<split>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )
def toxic_train_en_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_en_files ).shuffle(2048 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother())
 def toxic_train_translated_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_translated_files ).shuffle(2048*4 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother(0.1))<prepare_x_and_y>
seed = 50
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 loss_arr = []
for i in range(0, epochs):
print('====== Epoch {:} of {:}'.format(i+1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 30 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print('Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids = None,
attention_mask = b_input_mask,
labels = b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_arr.append(avg_train_loss)
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in val_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids = None,
attention_mask = b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print(" Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print(" Validation took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )
def non_toxic_train_en_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(non_toxic_en_files ).shuffle(2048 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother())
 def non_toxic_train_translated_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(non_toxic_translated_files ).shuffle(2048*4 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother(0.1))<create_dataframe>
pred_labels = np.array(sample_submission['target'] )
def toxic_train_nen_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_nen_files ).shuffle(2048 ).interleave(perFileDataset ).filter(validationFoldRemoveFilter(validation_fold_index)).map(smoother() ).shuffle(8000)
<create_dataframe>
te_labels = torch.tensor(pred_labels )
def non_toxic_train_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(non_toxic_nen_files ).shuffle(2048 ).interleave(perFileDataset ).filter(validationFoldRemoveFilter(validation_fold_index)).map(smoother() ).shuffle(8000)
<split>
prediction_data = TensorDataset(te_input, te_mask, te_labels)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler = prediction_sampler, batch_size = BATCH_SIZE )
def unknown_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(unknown_nen_files ).shuffle(2048 ).interleave(perFileDataset,num_parallel_calls=AUTO)
<create_dataframe>
print('Predicting labels for {:,} test sentences...'.format(len(te_input)))
 model.eval()
predictions , true_labels = [], []
for batch in prediction_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
predictions.append(logits)
 true_labels.append(label_ids)
 print(' DONE.' )
def valid_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(valid_files ).interleave(perFileDataset ).filter(validationFoldOnlyFilter(validation_fold_index))<create_dataframe>
flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
<groupby><EOS> 
sample_submission['target'] = flat_predictions
sample_submission.to_csv('submission.csv', index = False )