Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<filter>
!pip install transformers
test_ids = list(final.loc[final['Target'].isnull() , 'idhogar'] )<import_modules>
from tqdm.auto import tqdm
from torch import nn
from torch.nn import functional as F
from torch import optim
from torch.utils import data as torchdata
from sklearn import metrics as ms
from sklearn import model_selection as md
from matplotlib import pyplot as plt
import torch
import random
import os
import argparse as ap
import pandas as pd
import numpy as np
import transformers as tr
import lightgbm
from lightgbm import LGBMClassifier<compute_train_metric>
def seed_everything(seed = 42):
random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything()
lgb_model = LGBMClassifier(objective = 'multiclass', random_state = 42)
 lgb_model.fit(X_train, y_train)
 y_pred = lgb_model.predict(X_test)
 print(y_pred)
 f1_score(y_test, y_pred, average = 'macro' )<save_to_csv>
def process_tweet(s, save_text_formatting=False, keep_emoji=False, keep_usernames=False):
EMOJI_DESCRIPTION_SCRUB = re.compile(r':(\S+?):')
 HASHTAG_BEFORE = re.compile(r'
BAD_HASHTAG_LOGIC = re.compile(r'(\S+)!!')
 FIND_MENTIONS = re.compile(r'@(\S+)')
 LEADING_NAMES = re.compile(r'^\s*(( ?:@\S+\s*)+)')
 TAIL_NAMES = re.compile(r'\s*(( ?:@\S+\s*)+)$')
 def remove_accents(input_str):
nfkd_form = unicodedata.normalize('NFKD', input_str)
 return u"".join([c for c in nfkd_form if not unicodedata.combining(c)])
 if save_text_formatting:
s = re.sub(r'https\S+', r'xxxx', str(s))
else:
s = re.sub(r'https\S+', r' ', str(s))
s = re.sub(r'x{3,5}', r' ', str(s))
s = re.sub(r'\
', ' ', s)
 s = re.sub(r'\s', ' ', s)
 s = re.sub(r'
', ' ', s)
 s = re.sub(r'&amp;', '&', s)
 s = re.sub(r'&
s = re.sub(r'&gt;', '>', s)
 s = re.sub(r'&lt;', '<', s)
 s = re.sub(r
xgb_submission.to_csv('xgb.csv', index = False)
 lgb_submission.to_csv('lgb.csv', index = False )<train_model>
cfg = ap.Namespace()
cfg.train_file = ".. /input/nlp-getting-started/train.csv"
cfg.test_file = ".. /input/nlp-getting-started/test.csv"
cfg.x_column = "text"
cfg.y_column = "target"
cfg.device = "cuda:0"
cfg.model_name = 'roberta-base'
cfg.num_classes = 1
cfg.max_length = 128
cfg.batch_size = 32
cfg.lr = 2e-5
cfg.weight_decay = 5e-5
cfg.n_epochs = 3
cfg.warmup_ratio = 0.1
cfg.max_grad_norm = 1.0
device = torch.device(cfg.device )
print('Train start')
 params = {"objective" : "multiclass",
"num_class" : 4,
"num_leaves" : 60,
"max_depth": -1,
"learning_rate" : 0.01,
"bagging_fraction" : 0.9,
"feature_fraction" : 0.9,
"bagging_freq" : 5,
"bagging_seed" : 42,
"verbosity" : -1 }<define_variables>
class DataframeDataset(torchdata.Dataset):
def __init__(self, dataframe: pd.DataFrame, x_column: str, y_column: str, tokenizer: tr.PreTrainedTokenizer, max_length: int):
super(DataframeDataset, self ).__init__()
self.X = dataframe.loc[:, [x_column]].values
if y_column is not None:
self.Y = dataframe.loc[:, [y_column]].values
else:
self.Y = None
self.tokenizer = tokenizer
self.max_length = max_length
def __len__(self):
return len(self.X)
 def __getitem__(self, idx):
x = process_tweet(self.X[idx])
 x = self.tokenizer.encode_plus(x)['input_ids']
x_tensor = np.full(( self.max_length), fill_value=self.tokenizer.pad_token_id)
 attn_mask = np.zeros_like(x_tensor)
 for i, tok in enumerate(x):
x_tensor[i] = tok
attn_mask[i] = 1
if self.Y is None:
return x_tensor, attn_mask
else:
y = self.Y[idx]
return x_tensor, attn_mask, y.astype(np.float32 )
y_tra = y_train - 1
y_tes = y_test -1<train_model>
tokenizer = tr.RobertaTokenizer.from_pretrained(cfg.model_name)
 model = tr.RobertaForSequenceClassification.from_pretrained(
 cfg.model_name,
num_labels=cfg.num_classes,
)
lgtrain, lgval = lgb.Dataset(data = X_train, label = y_tra), lgb.Dataset(data = X_test, label = y_tes)
 print(lgtrain)
 lgb_ad_model = lgb.train(params, lgtrain, num_boost_round = 2000, valid_sets=[lgtrain, lgval],
early_stopping_rounds=100, verbose_eval=100 )<predict_on_test>
train_df = pd.read_csv(cfg.train_file)
 train, valid = md.train_test_split(train_df, test_size=0.2, random_state=42)
 test = pd.read_csv(cfg.test_file )
predictions = lgb_ad_model.predict(test_set )<define_variables>
train_dataset = DataframeDataset(train, cfg.x_column, cfg.y_column, tokenizer, cfg.max_length)
 valid_dataset = DataframeDataset(valid, cfg.x_column, cfg.y_column, tokenizer, cfg.max_length)
 test_dataset = DataframeDataset(test, cfg.x_column, None, tokenizer, cfg.max_length )
predict_df_1 = []
predict_df_2 = []
predict_df_3 = []
predict_df_4 = []<predict_on_test>
train_dl = torchdata.DataLoader(train_dataset, batch_size=cfg.batch_size, num_workers=8, sampler=torchdata.RandomSampler(train_dataset))
valid_dl = torchdata.DataLoader(valid_dataset, batch_size=cfg.batch_size, num_workers=8, sampler=torchdata.SequentialSampler(valid_dataset))
for i in range(len(predictions)) :
predict_df_1.append(predictions[i][0])
 predict_df_2.append(predictions[i][1])
 predict_df_3.append(predictions[i][2])
 predict_df_4.append(predictions[i][3] )<create_dataframe>
total_steps = len(train_dl)* cfg.n_epochs
n_warmup_steps = int(total_steps * cfg.warmup_ratio )
df_pred = pd.DataFrame(predict_df_1, columns = ['1'] )<prepare_output>
def model_step(model, x, attn_mask, y, compute_metrics=False, return_true_pred=False):
x = x.to(device)
 attn_mask = attn_mask.to(device)
 y = y.to(device)
 logits, = model(x, attention_mask=attn_mask)
 loss = F.binary_cross_entropy_with_logits(logits, y)
 result_dict = {
'loss': loss
}
if compute_metrics:
y_pred = torch.sigmoid(logits ).detach().cpu().numpy() > 0.5
y_true = y.detach().cpu().numpy()
f1 = ms.f1_score(y_true, y_pred)
 accuracy = ms.accuracy_score(y_true, y_pred)
 result_dict['f1'] = f1
result_dict['accuracy'] = accuracy
if return_true_pred:
y_pred = np.squeeze(torch.sigmoid(logits ).detach().cpu().numpy() > 0.5)
 y_true = np.squeeze(y.detach().cpu().numpy())
 result_dict['data'] =(y_true, y_pred)
 return result_dict
df_pred['2'] = predict_df_2
df_pred['3'] = predict_df_3
df_pred['4'] = predict_df_4
df_pred.head()<feature_engineering>
def evaluate(model, test_dataset):
test_dl = torchdata.DataLoader(test_dataset, batch_size=1, num_workers=1, sampler=torchdata.SequentialSampler(test_dataset))
predicts = []
model.eval()
with torch.no_grad() :
for x, attn in tqdm(test_dl):
x = x.to(device)
 attn = attn.to(device)
 logits, = model(x, attention_mask=attn)
 pred = torch.sigmoid(logits)> 0.5
pred = pred.cpu().squeeze().item()
predicts.append(pred)
 return np.array(predicts )
df_pred['Target'] = df_pred[['1', '2', '3', '4']].idxmax(axis = 1 )<save_to_csv>
model = model.to(device)
 optimizer = tr.optimization.AdamW(params=model.parameters() , lr=cfg.lr)
 scheduler = tr.get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=n_warmup_steps, num_training_steps=total_steps )
df_pred.to_csv('submission.csv', index = False )<set_options>
global_train_loss = []
global_val_acc = []
global_val_f1 = []
global_val_loss = []
for i in range(cfg.n_epochs):
model.train()
train_loss = []
for x, attn, y in tqdm(train_dl, unit='batch', desc=f"Train {i}"):
result = model_step(model, x, attn, y)
 loss = result['loss']
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , cfg.max_grad_norm)
 optimizer.step()
scheduler.step()
train_loss.append(loss.item())
 optimizer.zero_grad()
global_train_loss += train_loss
model.eval()
valid_loss = []
valid_f1 = []
valid_acc = []
with torch.no_grad() :
for x, attn, y in tqdm(valid_dl, unit='batch', desc=f"Valid {i}"):
result = model_step(model, x, attn, y, compute_metrics=True)
 valid_loss.append(result['loss'].item())
 valid_f1.append(result['f1'])
 valid_acc.append(result['accuracy'])
 global_val_acc += valid_acc
global_val_f1 += valid_f1
global_val_loss += valid_loss
print(f"Epoch {i}: Train Loss = {np.mean(train_loss):.6f}, Val Loss = {np.mean(valid_loss):.6f}, F1 = {np.mean(valid_f1):.4f}, Accuracy = {np.mean(valid_acc):.4f}" )
%matplotlib inline
plt.style.use('fivethirtyeight')
 plt.rcParams['font.size'] = 15
plt.rcParams['patch.edgecolor'] = 'k'
warnings.filterwarnings('ignore')
<load_from_csv>
true, pred = [], []
for x, attn, y in tqdm(valid_dl):
with torch.no_grad() :
y_true, y_pred = model_step(model, x, attn, y, return_true_pred=True)['data']
true += list(y_true)
 pred += list(y_pred)
 true = np.vstack(true)
 pred = np.vstack(pred)
 f1 = ms.f1_score(y_true=true, y_pred=pred)
 acc = ms.accuracy_score(y_true=true, y_pred=pred )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print("Train Dataset: Rows, Columns: ", train_df.shape)
 print("Test Dataset: Rows, Columns: ", test_df.shape )<prepare_output>
print(f"F1 - {f1:.4f}")
 print(f"Acc - {acc:.4f}" )
submit = test_df[['Id','idhogar']] 
<count_values>
predicts = evaluate(model, test_dataset )
train_df['dependency'].value_counts(ascending=False )<count_values>
submission = pd.DataFrame({
'id': test['id'],
'target': predicts.astype(np.int32)
 } )
<data_type_conversions><EOS> 
submission.to_csv("./submission.csv", index=False, encoding='utf-8' )