Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<prepare_x_and_y>
import torch
import pandas as pd
import random
import time
import datetime
from sklearn.model_selection import train_test_split, StratifiedKFold, KFold
from sklearn.metrics import f1_score
import numpy as np
from torch.utils.data import TensorDataset,Subset
from transformers import BertTokenizer
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
from torch.nn import functional as F
X = train.drop(['id', 'target'], axis=1 ).values
y = train.target.values
test_id = test.id.values
test = test.drop('id', axis=1 ).values<define_variables>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )
X = X[:,[142,135,150,14,138,143,139,141,136,61,151,11,179,146,140,86,175,147, 152,145,180,148,177,133,59,60,183,14,51,34,67,137,16,161,88,85,159,33,178,184,65,13,12,63,18,156,83,87]]
test = test[:,[142,135,150,14,138,143,139,141,136,61,151,11,179,146,140,86,175,147, 152,145,180,148,177,133,59,60,183,14,51,34,67,137,16,161,88,85,159,33,178,184,65,13,12,63,18,156,83,87]]<prepare_output>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )
sub = pd.DataFrame()
sub['id'] = test_id
sub['target'] = np.zeros_like(test_id )<train_model>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 sentences = train.text.values
labels = train.target.values
for i,(train_index, test_index)in enumerate(skf.split(X, y)) :
print('[Fold %d/%d]' %(i + 1, kfold))
X_train, X_valid = X[train_index], X[test_index]
y_train, y_valid = y[train_index], y[test_index]
d_train = xgb.DMatrix(X_train, y_train)
 d_valid = xgb.DMatrix(X_valid, y_valid)
 d_test = xgb.DMatrix(test)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
mdl = xgb.train(params, d_train, 1600, watchlist, early_stopping_rounds=70, feval=gini_xgb, maximize=True, verbose_eval=100)
 print('[Fold %d/%d Prediciton:]' %(i + 1, kfold))
p_test = mdl.predict(d_test, ntree_limit=mdl.best_ntree_limit)
 sub['target'] += p_test/kfold<save_to_csv>
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 labels = torch.tensor(labels)
 print('Original: ', sentences[0])
 print('Token IDs:', input_ids[0] )
sub.to_csv('StratifiedKFold5.csv', index=False )<set_options>
batch_size = 16
def get_data_loaders(dataset,train_indexes,val_indexes):
train_tensor = Subset(dataset,train_indexes)
 val_tensor = Subset(dataset,val_indexes)
 train_dataloader = DataLoader(
 train_tensor,
sampler = RandomSampler(train_tensor),
batch_size = batch_size 
)
 val_dataloader = DataLoader(
 val_tensor,
sampler = SequentialSampler(val_tensor),
batch_size = batch_size 
)
 return train_dataloader,val_dataloader
seed =45
% matplotlib inline<load_from_csv>
dataset = TensorDataset(input_ids, attention_masks, labels )
path = '.. /input/'
train = pd.read_csv(path+'train.csv',na_values=-1)
 test = pd.read_csv(path+'test.csv',na_values=-1)
 print('Number rows and columns:',train.shape)
 print('Number rows and columns:',test.shape )<drop_column>
df = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 sentences = df.text.values
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 64,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 batch_size = 16
prediction_data = TensorDataset(input_ids, attention_masks)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size )
ps_cal = train.columns[train.columns.str.startswith('ps_calc')]
train = train.drop(ps_cal,axis =1)
 test = test.drop(ps_cal,axis=1)
 train.shape<prepare_output>
def get_bert_model() :
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()
return model
k= pd.DataFrame()
k['train']= train.isnull().sum()
k['test'] = test.isnull().sum()
k<count_missing_values>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )
def missing_value(df):
col = df.columns
for i in col:
if df[i].isnull().sum() >0:
df[i].fillna(df[i].mode() [0],inplace=True )<count_missing_values>
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))
missing_value(train)
 missing_value(test )<feature_engineering>
seed_val = 1000
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val )
def descrictive_stat_feat(df):
df = pd.DataFrame(df)
 dcol= [c for c in train.columns if train[c].nunique() >5]
dcol.remove('id')
 d_median = df[dcol].median(axis=0)
 d_mean = df[dcol].mean(axis=0)
 q1 = df[dcol].apply(np.float32 ).quantile(0.25)
 q3 = df[dcol].apply(np.float32 ).quantile(0.75)
 for c in dcol:
df[c+str('_median_range')] =(df[c].astype(np.float32 ).values > d_median[c] ).astype(np.int8)
 df[c+str('_mean_range')] =(df[c].astype(np.float32 ).values > d_mean[c] ).astype(np.int8)
 df[c+str('_q1')] =(df[c].astype(np.float32 ).values < q1[c] ).astype(np.int8)
 df[c+str('_q3')] =(df[c].astype(np.float32 ).values > q3[c] ).astype(np.int8)
 return df<feature_engineering>
total_folds = 12
current_fold = 0
all_folds_preds = []
epochs = 1
fold=StratifiedKFold(n_splits=total_folds, shuffle=True, random_state=1000)
 training_stats = []
train = descrictive_stat_feat(train)
 test = descrictive_stat_feat(test )<count_unique_values>
total_t0 = time.time()
for train_index, test_index in fold.split(train,train['target']):
model = get_bert_model()
optimizer = AdamW(model.parameters() ,lr = 5e-5,eps = 1e-8)
 current_fold = current_fold+1
train_dataloader,validation_dataloader = get_data_loaders(dataset,train_index,test_index)
 print("")
 print('================= Fold {:} / {:} ================='.format(current_fold,total_folds))
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_time = format_time(time.time() - t0)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
total_f1_score = 0
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in validation_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() : 
(loss, logits)= model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
total_eval_accuracy += flat_accuracy(logits, label_ids)
 total_f1_score += f1_score(np.argmax(logits,axis=1),label_ids)
 avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
 print(" Accuracy: {0:.2f}".format(avg_val_accuracy))
avg_f1_score = total_f1_score / len(validation_dataloader)
 print(" F1_score: {0:.2f}".format(avg_f1_score))
avg_val_loss = total_eval_loss / len(validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
training_stats.append(
 {
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid.Loss': avg_val_loss,
'Valid.Accur.': avg_val_accuracy,
'f1_score' : avg_f1_score,
'Training Time': training_time,
'Validation Time': validation_time,
'fold' : current_fold
} 
)
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)))
 print("")
 print("now predicting for this fold")
 model.eval()
predictions = []
for batch in prediction_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 stack = np.vstack(predictions)
 final_preds = F.softmax(torch.from_numpy(stack)) [:,1].numpy()
all_folds_preds.append(final_preds)
 print("Completed")
category_col = [c for c in train.columns if train[c].nunique() <= 104]
len(category_col )<categorify>
pd.set_option('precision', 2)
 df_stats = pd.DataFrame(data=training_stats)
 df_stats = df_stats.set_index('fold')
 df_stats
def OHE(df1,df2,column):
cat_col = column
len_df1 = df1.shape[0]
df = pd.concat([df1,df2],ignore_index=True)
 c2,c3 = [],{}
print('Categorical feature',len(column))
for c in cat_col:
if df[c].nunique() >2 :
c2.append(c)
 c3[c] = 'ohe_'+c
df = pd.get_dummies(df, prefix=c3, columns=c2,drop_first=True)
 df1 = df.loc[:len_df1]
df2 = df.loc[len_df1:]
print('Train',df1.shape)
 print('Test',df2.shape)
 return df1,df2<compute_test_metric>
to_submit =np.mean(all_folds_preds,0 )
train1,test1 = OHE(train,test,category_col )<prepare_x_and_y>
sub=pd.DataFrame()
sub['id'] = df['id']
sub['target'] = to_submit
sub['target'] = sub['target'].apply(lambda x: 1 if x>0.5 else 0)
 sub.head()
<drop_column><EOS> 
sub.to_csv('bert_base_12_2e-5_64.csv',index=False )