Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<features_selection>
import numpy as np
import pandas as pd
import random
import plotly.express as px
import seaborn as sns
import matplotlib.pyplot as plt
import torch
from torch.utils.data import TensorDataset, DataLoader, random_split
from transformers import BertTokenizer
from transformers import BertForSequenceClassification, AdamW
from transformers import get_linear_schedule_with_warmup
importances = rnd.feature_importances_
indices = np.argsort(importances)[::-1] 
<feature_engineering>
seed = 0
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
device = torch.device("cuda" )
train["Elevation_roadways"] = train["Elevation"] -.02*train["Horizontal_Distance_To_Roadways"]
train["Elevation_vd"] = train["Elevation"] - train["Vertical_Distance_To_Hydrology"]
train["Elevation_hd"] = train["Elevation"] -.2*train["Horizontal_Distance_To_Hydrology"]
test["Elevation_roadways"] = test["Elevation"] -.02*test["Horizontal_Distance_To_Roadways"]
test["Elevation_vd"] = test["Elevation"] - test["Vertical_Distance_To_Hydrology"]
test["Elevation_hd"] = test["Elevation"] -.2*test["Horizontal_Distance_To_Hydrology"]<prepare_x_and_y>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 print("Shape of train data : ",train.shape)
 print("Shape of test data : ",test.shape )
predictor = train.drop(["Id","Cover_Type"],axis = 1)
 target = train.Cover_Type<import_modules>
incorrect_labels_df = train.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 incorrect_labels_df = incorrect_labels_df[incorrect_labels_df['target'] > 1]
incorrect_texts = incorrect_labels_df.index.tolist()
train = train[~train.text.isin(incorrect_texts)]
train['keyword'].fillna('', inplace=True)
 train['final_text'] = train['keyword'] + ' ' + train['text']
test['keyword'].fillna('', inplace=True)
 test['final_text'] = test['keyword'] + ' ' + test['text']
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict 
<choose_model_class>
text_values = train.final_text.values
labels = train.target.values
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )
rnd1 = RandomForestClassifier(random_state = 123,n_estimators = 175 )<compute_train_metric>
print('Original Text : ', text_values[1])
 print('Tokenized Text: ', tokenizer.tokenize(text_values[1]))
print('Token IDs : ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text_values[1])) )
print(cross_val_score(rnd1,predictor,target,cv = 3 ).mean() )<choose_model_class>
cls, sep, pad = '[CLS]', '[SEP]', '[PAD]'
print('Originally Format:', cls, sep, pad)
 print('Tokenized Text: ', tokenizer.tokenize(cls), tokenizer.tokenize(sep), tokenizer.tokenize(pad))
print('Token IDs : ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(cls)) , tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sep)) , tokenizer.convert_tokens_to_ids(tokenizer.tokenize(pad)) )
ex = ExtraTreesClassifier(random_state =123,n_estimators = 1000)
 ex1 = AdaBoostClassifier(base_estimator = ex,n_estimators = 5)
<compute_test_metric>
print('Original Text : ', text_values[1])
 print('Tokenized Text : ', tokenizer.tokenize(text_values[1]))
print('Token IDs : ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text_values[1])))
 print('Adding Special Tokens :', tokenizer.build_inputs_with_special_tokens(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text_values[1]))))
print('Adding Special Tokens Using Encode Func:', tokenizer.encode(text_values[1]))
cross_val_score(ex1,predictor,target,cv = 3 ).mean()<train_model>
def encode_fn(text_list):

 all_input_ids = []
for text in text_values:
input_ids = tokenizer.encode(
 text,
add_special_tokens = True,
max_length = 160,
pad_to_max_length = True,
return_tensors = 'pt' 
)
 all_input_ids.append(input_ids)
 all_input_ids = torch.cat(all_input_ids, dim=0)
 return all_input_ids
ex.fit(predictor,target )<predict_on_test>
epochs = 10
batch_size = 32
all_input_ids = encode_fn(text_values)
 labels = torch.tensor(labels)
 dataset = TensorDataset(all_input_ids, labels)
 train_size = int(0.95 * len(dataset))
val_size = len(dataset)- train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
 train_dataloader = DataLoader(train_dataset, batch_size = batch_size, shuffle = True)
 val_dataloader = DataLoader(val_dataset, batch_size = batch_size, shuffle = False )
prediction = ex.predict(test.drop("Id",axis = 1))<feature_engineering>
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()
optimizer = AdamW(model.parameters() , lr = 2e-5)
 total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = 0,num_training_steps = total_steps )
test["Cover_Type"] = prediction<define_variables>
for epoch in range(epochs):
model.train()
total_loss, total_val_loss = 0, 0
for step, batch in enumerate(train_dataloader):
model.zero_grad()
loss, logits = model(batch[0].to(device),
token_type_ids = None,
attention_mask =(batch[0] > 0 ).to(device),
labels = batch[1].to(device))
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
model.eval()
for i, batch in enumerate(val_dataloader):
with torch.no_grad() :
loss, logits = model(batch[0].to(device),
token_type_ids = None,
attention_mask =(batch[0] > 0 ).to(device),
labels = batch[1].to(device))
total_val_loss += loss.item()
avg_train_loss = total_loss / len(train_dataloader)
 avg_val_loss = total_val_loss / len(val_dataloader)
 print('Train Loss : ', avg_train_loss)
 print('Validation Loss: ', avg_val_loss)
 print('
' )
my_submission = test[["Id","Cover_Type"]]<count_values>
text_values = test.final_text.values
all_input_ids = encode_fn(text_values)
 pred_data = TensorDataset(all_input_ids)
 pred_dataloader = DataLoader(pred_data, batch_size = batch_size, shuffle = False )
test["Cover_Type"].value_counts(normalize = True)*100<predict_on_test>
model.eval()
preds = []
for i,(batch,)in enumerate(pred_dataloader):
with torch.no_grad() :
outputs = model(batch.to(device),
token_type_ids = None,
attention_mask =(batch > 0 ).to(device))
logits = outputs[0]
logits = logits.detach().cpu().numpy()
preds.append(logits)
 final_preds = np.concatenate(preds, axis=0)
 final_preds = np.argmax(final_preds, axis=1 )
prediction = cross_val_predict(ex1,predictor,target,cv = 3 )<import_modules>
submission = pd.DataFrame()
submission['id'] = test['id']
submission['target'] = final_preds
submission.to_csv('submission.csv', index=False )