Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<create_dataframe>
confusion_matrix)
train_dataset = TweetDataset(train_df)
 test_dataset = TweetDataset(test_df)
 train_metadata = train_dataset.get_metadata()
test_metadata = test_dataset.get_metadata()<define_variables>
PRETRAINED_MODEL_NAME = 'bert-base-uncased'
LABELS_NUMBER = 2
MAX_LENGHT = 512
BATCH_SIZE = 6
LEARNING_RATE = 2e-5
EPOCHS_NUMBER = 1
N_PREDICTIONS_TO_SHOW = 10
def train_loop() :
kf = StratifiedKFold(n_splits=n_splits, random_state=random_seed, shuffle=True)
 val_start_logits = [0 for tm in train_metadata]
val_end_logits = [0 for tm in train_metadata]
test_start_logits = [torch.zeros(len(tm["input_ids"]), dtype=torch.float)for tm in test_metadata]
test_end_logits = [torch.zeros(len(tm["input_ids"]), dtype=torch.float)for tm in test_metadata]
test_dataloader = test_dataset.get_test_dataloader()
for fold_idx,(train_idxs, val_idxs)in enumerate(kf.split(X=train_dataset.df,
y=train_dataset.df.sentiment.values)) :
train_dataloader = train_dataset.get_train_dataloader(train_idxs)
 val_dataloader = train_dataset.get_train_dataloader(val_idxs)
 model_config = RobertaConfig.from_pretrained(roberta_path, lowercase=True)
 model_config.hidden_dropout_prob = hidden_dropout_prob
model_config.attention_probs_dropout_prob = attention_probs_dropout_prob
model = TweetModel(conf=model_config)
 model.cuda()
optimizer = AdamW(model.parameters() , lr=lr)
 num_train_steps = epochs * len(train_dataloader)
 scheduler = get_cosine_schedule_with_warmup(optimizer, warmup_steps,
num_train_steps)
 for epoch in range(epochs):
model.train()
for batch in train_dataloader:
batch = tuple(t.to(device)for t in batch)
 input_ids, attention_mask, loss_mask, start_labels, end_labels = batch
start_logits, end_logits = model(input_ids=input_ids, attention_mask=attention_mask)
 loss = tweet_loss(start_logits, start_labels, end_logits, end_labels, loss_mask)
 loss.backward()
optimizer.step()
scheduler.step()
model.zero_grad()
model.eval()
val_fold_start_logits = []
val_fold_end_logits = []
for batch in val_dataloader:
batch = tuple(t.to(device)for t in batch)
 input_ids, attention_mask, loss_mask, start_labels, end_labels = batch
with torch.no_grad() :
start_logits, end_logits = model(input_ids=input_ids, attention_mask=attention_mask)
 batch_start_logits = [start_logits[s].cpu() for s in range(input_ids.shape[0])]
batch_end_logits = [end_logits[s].cpu() for s in range(input_ids.shape[0])]
val_fold_start_logits += batch_start_logits
val_fold_end_logits += batch_end_logits
for e, idx in enumerate(val_idxs):
val_start_logits[idx] = val_fold_start_logits[e]
val_end_logits[idx] = val_fold_end_logits[e]
test_fold_start_logits = []
test_fold_end_logits = []
for batch in test_dataloader:
batch = tuple(t.to(device)for t in batch)
 input_ids, attention_mask, loss_mask = batch
with torch.no_grad() :
start_logits, end_logits = model(input_ids=input_ids, attention_mask=attention_mask)
 batch_start_logits = [start_logits[s].cpu() for s in range(input_ids.shape[0])]
batch_end_logits = [end_logits[s].cpu() for s in range(input_ids.shape[0])]
test_fold_start_logits += batch_start_logits
test_fold_end_logits += batch_end_logits
for e in range(len(test_dataset)) :
test_start_logits[e] += test_fold_start_logits[e].squeeze(1)
 test_end_logits[e] += test_fold_end_logits[e].squeeze(1)
 print("one fold over!")
 return val_start_logits, val_end_logits, test_start_logits, test_end_logits 
<train_model>
train_data = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 print(train_data.shape)
 train_data.head(3 )
%%time
device = torch.device("cuda")
 val_start_logits, val_end_logits, test_start_logits, test_end_logits = train_loop()<define_variables>
test_data = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 print(test_data.shape)
 test_data.head(3 )
jaccs = []
predictions = []
br = 0
for e, meta in enumerate(train_metadata):
text = meta["text"]
preprocessed_text = meta["preprocessed_text"]
selected_text = meta["selected_text"]
loss_mask = meta["loss_mask"]
tokens = meta["tokens"]
offsets = meta["offsets"]
actives = np.asarray(meta["loss_mask"] ).reshape(-1)== 1
start_probs = val_start_logits[e][actives].sigmoid()
end_probs = val_end_logits[e][actives].sigmoid()
start_idx = start_probs.argmax().item()
end_idx = end_probs.argmax().item()
if start_idx > end_idx:
br += 1
start_idx , end_idx = 0, start_probs.shape[0]-1
start_max = start_probs.max().item()
end_max = end_probs.max().item()
pp = preprocessed_text[offsets[start_idx][0]:offsets[end_idx][1]]
if pp not in text or pp not in " " + text:
pp = prepare_predictions(pp)
 predictions.append(pp)
 jaccs.append(jaccard(selected_text, pp))
print(len(jaccs))
print(round(statistics.mean(jaccs),4), round(statistics.mean(jaccs)*0.5955+0.9767*0.4045,4))
print(br )<feature_engineering>
tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME,
do_lower_case=True )
test_raw["selected_text"] = ""
test_raw.loc[test_raw.sentiment == "neutral", "selected_text"] = test_raw.loc[test_raw.sentiment == "neutral", "text"] 
<define_variables>
vocabulary = tokenizer.get_vocab()
print(f'Size of the vocabulary: {len(vocabulary)}')
 print(f'Some tokens of the vocabulary: {list(vocabulary.keys())[5000:5010]}' )
predictions = []
br = 0
for e, meta in enumerate(test_metadata):
text = meta["text"]
preprocessed_text = meta["preprocessed_text"]
loss_mask = meta["loss_mask"]
tokens = meta["tokens"]
offsets = meta["offsets"]
actives = np.asarray(meta["loss_mask"] ).reshape(-1)== 1
start_probs = test_start_logits[e][actives].sigmoid()
end_probs = test_end_logits[e][actives].sigmoid()
start_idx = start_probs.argmax().item()
end_idx = end_probs.argmax().item()
if start_idx > end_idx:
br += 1
start_idx , end_idx = 0, start_probs.shape[0]-1
pp = preprocessed_text[offsets[start_idx][0]:offsets[end_idx][1]]
if pp not in text or pp not in " " + text:
pp = prepare_predictions(pp)
 predictions.append(pp)
 print(br )<feature_engineering>
def prepare_sequence(text):

 prepared_sequence = tokenizer.encode_plus(
 text,
add_special_tokens = True,
max_length = MAX_LENGHT,
padding = 'max_length',
return_attention_mask = True 
)
 return prepared_sequence
test_raw.loc[test_raw.sentiment != "neutral", "selected_text"] = predictions 
<save_to_csv>
test_sentence = 'Is this jacksonville?'
test_sentence_encoded = prepare_sequence(test_sentence)
 token_ids = test_sentence_encoded["input_ids"]
print(f'Test sentence: {test_sentence}')
 print(f'Keys: {test_sentence_encoded.keys() }')
 print(f'Tokens: {tokenizer.convert_ids_to_tokens(token_ids)[:12]}')
 print(f'Token IDs: {token_ids[:12]}')
 print(f'Segment IDs: {test_sentence_encoded["token_type_ids"][:12]}')
 print(f'Mask IDs {test_sentence_encoded["attention_mask"][:12]}')
 print(f'Input dimension: {len(token_ids)}' )
submission = test_raw.drop(columns = ["text", "sentiment"])
 submission.to_csv("submission.csv", index=False )<install_modules>
def map_example_to_dict(input_ids, attention_masks, token_type_ids, label):

 mapped_example = {
"input_ids": input_ids,
"token_type_ids": token_type_ids,
"attention_mask": attention_masks,
}
return mapped_example, label
def encode_examples(texts_and_labels):

 input_ids_list = []
token_type_ids_list = []
attention_mask_list = []
label_list = []
for text, label in texts_and_labels:
bert_input = prepare_sequence(text)
 input_ids_list.append(bert_input['input_ids'])
 token_type_ids_list.append(bert_input['token_type_ids'])
 attention_mask_list.append(bert_input['attention_mask'])
 label_list.append([label])
 dataset = tf.data.Dataset.from_tensor_slices(
(input_ids_list, attention_mask_list, token_type_ids_list,
label_list)
)
 dataset_mapped = dataset.map(map_example_to_dict)
 return dataset_mapped
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q<load_from_csv>
X = train_data["text"]
y = train_data["target"]
train_data = list()
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 for id, row in train.iterrows() :
template = {
'context': "",
'qas': [
{
'id': "",
'is_impossible': False,
'question': "",
'answers': [
{
'text': "",
'answer_start':''
}
]
}
]
}
template['qas'][0]['id'] = row['textID']
context = str(row['text'])
 question = row['sentiment']
answer = str(row['selected_text'])
 template['context'] = context.lower()
template['qas'][0]['question'] = question.lower()
template['qas'][0]['answers'][0]['text'] = answer.lower()
try:
template['qas'][0]['answers'][0]['answer_start'] = context.index(answer)
 except AttributeError:
print(id, row['text'], row['selected_text'])
 train_data.append(template)
 with open('train_processed.json', 'w')as f:
json.dump(train_data, f )<load_from_csv>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.10,
random_state=1 )
test_data = list()
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 for id, row in test.iterrows() :
template = {
'context': "",
'qas': [
{
'id': "",
'is_impossible': False,
'question': "",
'answers': [
{
'text': "",
'answer_start': ''
}
]
}
]
}
template['context'] = str(row['text'] ).lower()
template['qas'][0]['id'] = row['textID']
template['qas'][0]['question'] = str(row['sentiment'] ).lower()
test_data.append(template)
 with open('test_processed.json', 'w')as f:
json.dump(test_data, f )<load_pretrained>
n_training_examples = X_train.shape[0]
n_positive_training_examples = y_train.value_counts() [1]
n_negative_training_examples = y_train.value_counts() [0]
print(f'Number examples in training dataset: {n_training_examples}')
 print(f'Number of positive examples in training dataset: {n_positive_training_examples}')
 print(f'Number of negative examples in training dataset: {n_negative_training_examples}' )
logging.basicConfig(level=logging.INFO)
 transformers_logger = logging.getLogger("transformers")
 transformers_logger.setLevel(logging.WARNING)
 with open('train_processed.json', 'r')as f:
train_data = json.load(f)
 train_args = {
'reprocess_input_data': True,
'use_multiprocessing': False,
'do_lower_case': True,
"wandb_project": False,
'learning_rate': 5e-5,
'num_train_epochs': 3,
'max_seq_length': 192,
'doc_stride': 64,
'overwrite_output_dir': True,
'reprocess_input_data': False,
'save_steps': 0,
'fp16': False,
'save_eval_checkpoints': False,
'save_model_every_epoch': False
}
arch = 'albert'
m = '/kaggle/input/pretrained-albert-pytorch/albert-large-v1'
model = QuestionAnsweringModel(arch, m,
args=train_args,
use_cuda=True 
)
 model.train_model(train_data)
 with open('test_processed.json', 'r')as f:
test_data = json.load(f)
 pred = model.predict(test_data)
 final_output = list()
for p in pred:
idText = p['id']
answer = p['answer']
out = {'textID': idText, 'selected_text': answer}
final_output.append(out)
 out_df = pd.DataFrame(final_output)
 out_df.to_csv('submission.csv', index=False )<import_modules>
train_dataset = list(zip(X_train, y_train))
val_dataset = list(zip(X_val, y_val))
print('TF version',tf.__version__ )<load_from_csv>
ds_train_encoded = encode_examples(train_dataset ).shuffle(10000 ).batch(BATCH_SIZE)
 ds_val_encoded = encode_examples(val_dataset ).batch(BATCH_SIZE )
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()<string_transform>
def get_model() :
config = AutoConfig.from_pretrained(PRETRAINED_MODEL_NAME,
hidden_dropout_prob=0.2,
num_labels=LABELS_NUMBER)
 model = TFBertForSequenceClassification.from_pretrained(PRETRAINED_MODEL_NAME,
config=config)
 return model
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
model = get_model()
optimizer = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)
 loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
 metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')
 model.compile(optimizer=optimizer, loss=loss, metrics=[metric] )
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<define_variables>
weight_for_0 =(1 / n_negative_training_examples)*(n_training_examples)/2.0
weight_for_1 =(1 / n_positive_training_examples)*(n_training_examples)/2.0
class_weight = {0: weight_for_0, 1: weight_for_1}
print('Weight for class 0: {:.2f}'.format(weight_for_0))
print('Weight for class 1: {:.2f}'.format(weight_for_1))
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<categorify>
model.fit(ds_train_encoded, epochs=EPOCHS_NUMBER, validation_data=ds_val_encoded,
class_weight = class_weight )
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
val_predictions = model.predict(ds_val_encoded)
 val_probabilities = softmax(val_predictions[0], axis=1)
 y_val_predictions = np.argmax(val_probabilities, axis=1 ).flatten()
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.BatchNormalization()(x2)
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model<define_variables>
def encode_test_examples(texts):

 input_ids_list = []
token_type_ids_list = []
attention_mask_list = []
for text in texts:
bert_input = prepare_sequence(text)
 input_ids_list.append(bert_input['input_ids'])
 token_type_ids_list.append(bert_input['token_type_ids'])
 attention_mask_list.append(bert_input['attention_mask'])
 dataset = tf.data.Dataset.from_tensor_slices(
(input_ids_list, attention_mask_list, token_type_ids_list)
)
 dataset_mapped = dataset.map(map_test_example_to_dict)
 return dataset_mapped
def map_test_example_to_dict(input_ids, attention_masks, token_type_ids):

 mapped_example = {
"input_ids": input_ids,
"token_type_ids": token_type_ids,
"attention_mask": attention_masks,
}
return mapped_example
n_splits = 2<init_hyperparams>
X_test = test_data["text"]
test_dataset = list(X_test)
 ds_test_encoded = encode_test_examples(test_dataset ).batch(BATCH_SIZE )
<predict_on_test>
test_predictions = model.predict(ds_test_encoded)
 test_probabilities = softmax(test_predictions[0], axis=1)
 y_test_predictions = np.argmax(test_probabilities, axis=1 ).flatten()
<string_transform><EOS> 
final_submission = pd.DataFrame(data={"id":test_data["id"], "target":y_test_predictions})
 final_submission.to_csv("submissionTweets.csv", index=False )