Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<train_on_grid>
!pip install transformers
grid = GridSearchCV(estimator=MLPerseptron, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_<train_model>
from keras.preprocessing import sequence
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
import re
import string
from nltk.stem.snowball import SnowballStemmer
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from nltk import word_tokenize
from transformers import BertForSequenceClassification, AdamW, BertConfig
from transformers import BertTokenizer
from transformers import get_linear_schedule_with_warmup
import random
from sklearn.model_selection import train_test_split
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
import time
import datetime
from sklearn.metrics import accuracy_score
from sklearn.metrics import matthews_corrcoef
MLPerseptron = MLPClassifier(**best_param)
 MLPerseptron.fit(X_train,y_train )<compute_test_metric>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )
y_hat= MLPerseptron.predict_proba(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['MLPerseptron']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % MLPerseptron.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))<train_on_grid>
no_classes = 1
MAX_LEN = 64
BATCH_SIZE = 16
no_epochs = 3
random_state=42
seed_val = 2020
steps = 50
grid = GridSearchCV(estimator=MLPerseptron, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train_ol,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_<train_model>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 df_submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 print(train.head())
 print(test.head())
 print(train.shape, test.shape )
MLPerseptron_ol = MLPClassifier(**best_param)
 MLPerseptron_ol.fit(X_train_ol,y_train )<compute_train_metric>
print(train.isnull().sum() ,'
')
 print(test.isnull().sum() )
y_hat= MLPerseptron_ol.predict_proba(X_test_ol)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['MLPerseptron_ol']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % MLPerseptron_ol.score(X_test_ol, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))<train_on_grid>
train = train.drop(["keyword", "location"], axis=1)
 test = test.drop(["keyword","location"], axis=1)
 print(train.head())
 print(test.head() )
grid = GridSearchCV(estimator=g_NB, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_<train_model>
train['text'] = train['text'].apply(lambda x: x.lower())
 test['text'] = test['text'].apply(lambda x: x.lower())
 train['text'].head()
g_NB = GaussianNB(**best_param)
 g_NB.fit(X_train, y_train )<compute_train_metric>
def remove_noise(text):
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 text = re.sub(r'[^\x00-\x7f]',r' ', text)
 return text
train['text'] = train['text'].apply(lambda x: remove_noise(x))
test['text'] = test['text'].apply(lambda x: remove_noise(x))
train['text'].head(20 )
y_hat= g_NB.predict_proba(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['g_NB']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % g_NB.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))<train_model>
!pip install nlppreprocess
nlp = NLP()
train['text'] = train['text'].apply(nlp.process)
 test['text'] = test['text'].apply(nlp.process)
 train['text'].head(20 )
grid.fit(X_train_ol,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_<train_model>
stemmer = SnowballStemmer("english")
 def stemming(text):
text = [stemmer.stem(word)for word in text.split() ]
return ' '.join(text)
 train['text'] = train['text'].apply(stemming)
 test['text'] = test['text'].apply(stemming)
 train['text'].tail(20 )
grid.fit(X_train_ol,y_train)
 best_param=grid.best_params_
g_NB_ol = GaussianNB(**best_param)
 g_NB_ol.fit(X_train_ol, y_train )<compute_train_metric>
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}
y_hat= g_NB.predict_proba(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['g_NB_ol']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % g_NB.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))<choose_model_class>
def convert_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word
def convert_abbrev_in_text(text):
tokens = word_tokenize(text)
 tokens = [convert_abbrev(word)for word in tokens]
text = ' '.join(tokens)
 return text
param_grid = { 'n_neighbors': list(range(4,100)) ,
'weights': ['uniform','distance'],
'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'],
'p' : [1,2]}
knn = KNeighborsClassifier()<train_on_grid>
train["text"] = train["text"].apply(lambda x: convert_abbrev_in_text(x))
test["text"] = test["text"].apply(lambda x: convert_abbrev_in_text(x))
grid = GridSearchCV(estimator=knn, param_grid=param_grid, scoring= 'roc_auc', verbose=1, n_jobs=-1)
 grid.fit(X_train,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_<train_model>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text )
knn = KNeighborsClassifier(**best_param)
 knn.fit(X_train, y_train )<compute_test_metric>
train["text"] = train["text"].apply(lambda x: remove_emoji(x))
test["text"] = test["text"].apply(lambda x: remove_emoji(x))
y_hat= knn.predict_proba(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['knn']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % knn.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))<train_model>
tweets = train["text"].values
target = train["target"].values
sentences_test = test["text"].values
grid.fit(X_train_ol,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_<train_model>
print('Loading BERT tokenizer...')
 tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )
grid.fit(X_train_ol,y_train)
 best_param=grid.best_params_
knn_ol = KNeighborsClassifier(**best_param)
 knn_ol.fit(X_train_ol, y_train )<compute_test_metric>
print(' Original: ', tweets[100])
 print('Tokenized: ', tokenizer.tokenize(tweets[100]))
print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(tweets[100])) )
y_hat= knn_ol.predict_proba(X_test_ol)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['knn_ol']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % knn_ol.score(X_test_ol, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))<create_dataframe>
def tocknize(text_input):
input_ids = []
for sent in text_input:
encoded_sent = tokenizer.encode(
 sent,
add_special_tokens = True, 
)
 input_ids.append(encoded_sent)
 return input_ids
temp=pd.DataFrame()
temp['mean']=result.mean(axis=1)
 result2=pd.concat([result, temp], axis=1 )<compute_test_metric>
X = tocknize(tweets)
 test_token = tocknize(sentences_test )
FinalResult=pd.DataFrame()
for results in result2.columns:
y_hat=result2[results]
AUC_ROC=roc_auc_score(y_test, y_hat)
 FinalResult[results]=[AUC_ROC]
FinalResult.transpose()<import_modules>
def padding_seq(input_token):
out_seq = pad_sequences(input_token,
maxlen=MAX_LEN,
dtype="long",
truncating="post",
padding="post")
 return out_seq
X = padding_seq(X)
 print('Shape of X tensor:', X.shape)
 y = target
print('Shape of y set:', y.shape)
 print(X[221])
 print(y[221])
 test_token = padding_seq(test_token)
 print(test_token[221])
 max_length = X.shape[1]
print(max_length )
import sklearn.metrics as metrics
import matplotlib.pyplot as plt<train_model>
def mask_atten(input_seq):
attention_masks = []
for seq in input_seq:
out_mask = [float(i>0)for i in seq]
attention_masks.append(out_mask)
 return attention_masks
train_mask = mask_atten(X)
 test_mask = mask_atten(test_token )
X_Test = test_df.drop(columns=["id"])
 X_Test_ol=X_Test[d]
LR_ol.fit(X_ol,y)
<predict_on_test>
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(X, y,
random_state=random_state, test_size=0.2)
 train_masks, validation_masks, _, _ = train_test_split(train_mask, y,
random_state=random_state, test_size=0.2 )
y_hat= knn_ol.predict_proba(X_Test_ol)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[round(( y_hat[:,1].size)*(100-tax)/100)])
 submission['target']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)) )<save_to_csv>
train_inputs = torch.tensor(train_inputs ).to(torch.int64)
 validation_inputs = torch.tensor(validation_inputs ).to(torch.int64)
 train_labels = torch.tensor(train_labels ).to(torch.int64)
 validation_labels = torch.tensor(validation_labels ).to(torch.int64)
 train_masks = torch.tensor(train_masks ).to(torch.int64)
 validation_masks = torch.tensor(validation_masks ).to(torch.int64)
 tensor_input = torch.tensor(test_token ).to(torch.int64)
 tensor_mask = torch.tensor(test_mask ).to(torch.int64 )
submission.to_csv('submission.csv', index=False)
 submission.head(20 )<set_options>
given_labels = np.array(df_submission['target'])
 tensor_labels = torch.tensor(given_labels ).to(torch.int64)
 train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=BATCH_SIZE, shuffle=False)
 prediction_data = TensorDataset(tensor_input, tensor_mask, tensor_labels)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler = prediction_sampler, batch_size = BATCH_SIZE )
%matplotlib inline 
<load_from_csv>
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
<statistical_test>
params = list(model.named_parameters())
 print('The BERT model has {:} different named parameters.
 '.format(len(params)))
 print('==== Embedding Layer ====
')
 for p in params[0:5]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== First Transformer ====
')
 for p in params[5:21]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== Output Layer ====
')
 for p in params[-4:]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
train_df=reduce_mem_usage(train_df)
 test_df=reduce_mem_usage(test_df )<drop_column>
optimizer = AdamW(model.parameters() ,
lr = 2e-5,
eps = 1e-8
)
target = train_df["target"]
train_df = train_df.drop(["target","id"],axis=1)
 test_id = test_df["id"]
test_df = test_df.drop(["id"],axis=1)
<split>
total_steps = len(train_dataloader)* no_epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps)
shuffle_split = StratifiedShuffleSplit(test_size=0.25,train_size=0.75,n_splits=25 )<define_search_space>
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, no_epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, no_epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % steps == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(X)
 loss_values.append(avg_train_loss)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
eval_loss, eval_accuracy,eval_mcc_accuracy = 0, 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
pred_flat = np.argmax(logits, axis=1 ).flatten()
labels_flat = label_ids.flatten()
tmp_eval_accuracy = accuracy_score(pred_flat, labels_flat)
 tmp_eval_mcc_accuracy = matthews_corrcoef(labels_flat, pred_flat)
 eval_accuracy += tmp_eval_accuracy
eval_mcc_accuracy += tmp_eval_mcc_accuracy
nb_eval_steps += 1
print(F'
\tValidation Accuracy: {eval_accuracy/nb_eval_steps}')
 print(F'
\tValidation MCC Accuracy: {eval_mcc_accuracy/nb_eval_steps}')
 print(" Validation took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )
param_grid = {'C' : [1,0.01,0.1,10,100],
'penalty' : ["l1", "l2"],
'class_weight' : [{1:0.5, 0:0.5}, {1:0.4, 0:0.6}, {1:0.6, 0:0.4}, {1:0.7, 0:0.3}]
}
grid_search = GridSearchCV(
 estimator = LogisticRegression(random_state=12,solver="liblinear"),
param_grid = param_grid,
scoring='roc_auc',
cv = shuffle_split
)<train_on_grid>
print('Predicting labels for {:,} test sentences...'.format(len(tensor_input)))
 model.eval()
predictions , true_labels = [], []
for batch in prediction_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
predictions.append(logits)
 true_labels.append(label_ids)
 print('DONE.' )
<find_best_params><EOS> 
flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
df_submission['target'] = flat_predictions
df_submission.to_csv('submission.csv', index = False)
 df_submission.head()