Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<count_unique_values>
import pandas as pd
import numpy as np
import re
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torch.utils.data import TensorDataset, DataLoader
import spacy
from gensim.models import KeyedVectors
from nltk.stem import PorterStemmer, SnowballStemmer
from nltk.stem.lancaster import LancasterStemmer
import time
import gc
from tqdm import tqdm
assetName_code = market_train_df.groupby('assetName')['assetCode'].nunique().reset_index().sort_values(by='assetCode',ascending=False )<groupby>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 labels = np.array(train_df.target, dtype=int )
assetCodeCount = assetName_code.groupby('assetCode')['assetName'].count().reset_index()
assetCodeCount.columns = ['assetCodeNo', 'counts']
assetCodeCount.head()
del assetCodeCount<merge>
def n_upper(sentence):
return len(re.findall(r'[A-Z]',sentence))
def n_unique_words(sentence):
return len(set(sentence.split()))
def n_question_mark(sentence):
return len(re.findall(r'[?]',sentence))
def n_exclamation_mark(sentence):
return len(re.findall(r'[!]',sentence))
def n_asterisk(sentence):
return len(re.findall(r'[*]',sentence))
def n_parentheses(sentence):
return len(re.findall(r'[() ]',sentence))
def n_brackets(sentence):
return len(re.findall(r'[\[\]]',sentence))
def n_braces(sentence):
return len(re.findall(r'[{}]',sentence))
def n_quotes(sentence):
return len(re.findall(r'["]',sentence))
def n_ampersand(sentence):
return len(re.findall(r'[&]',sentence))
def n_dash(sentence):
return len(re.findall(r'[-]',sentence))
n_stats = 11
def get_stat(questions_list):

 stat_feat = np.zeros(( len(questions_list), n_stats), dtype=int)
 for i,question in tqdm(enumerate(questions_list)) :
stat_feat[i,0] = n_upper(question)
 stat_feat[i,1] = n_unique_words(question)
 stat_feat[i,2] = n_question_mark(question)
 stat_feat[i,3] = n_exclamation_mark(question)
 stat_feat[i,4] = n_asterisk(question)
 stat_feat[i,5] = n_parentheses(question)
 stat_feat[i,6] = n_brackets(question)
 stat_feat[i,7] = n_braces(question)
 stat_feat[i,8] = n_quotes(question)
 stat_feat[i,9] = n_ampersand(question)
 stat_feat[i,10] = n_dash(question)
 return stat_feat
thisAssetMark_number = thisAssetMark_df[columns_corr_market+['date']]
thisAssetMark_number = thisAssetMark_number.groupby('date' ).mean().reset_index()
thisAssetNews_number = thisAssetNews_df[columns_corr+['date']]
thisAssetNews_number = thisAssetNews_number.groupby('date' ).mean().reset_index()
thisAssetNews_number['news_volume'] = thisAssetNews_df.groupby('date')['sourceId'].count().reset_index() ['sourceId']
thisAssetMerge = pd.merge(thisAssetMark_number, thisAssetNews_number, how='left', on = 'date' )<drop_column>
train_stat = get_stat(train_df.question_text)
 test_stat = get_stat(test_df.question_text )
del thisAssetMark_df
del news_relevance
del market_train_fill
del news_train_df
del news_rmv_outlier<sort_values>
train_list = list(train_df.question_text.apply(lambda s: s.lower()))
test_list = list(test_df.question_text.apply(lambda s: s.lower()))
train_text = ' '.join(train_list)
 test_text = ' '.join(test_list )
market_train_orig = market_train_orig.sort_values('time')
 news_train_orig = news_train_orig.sort_values('time')
 market_train_df = market_train_orig.copy()
news_train_df = news_train_orig.copy()
del market_train_orig
del news_train_orig<feature_engineering>
nlp = spacy.load("en", disable=['tagger','parser','ner','textcat'] )
market_train_df = market_train_df.loc[market_train_df['time'].dt.date>=datetime.date(2009,1,1)]
news_train_df = news_train_df.loc[news_train_df['time'].dt.date>=datetime.date(2009,1,1)]<feature_engineering>
vocab = {}
lemma_vocab = {}
word_idx = 1
train_tokens = []
for doc in tqdm(nlp.pipe(train_list)) :
curr_tokens = []
for token in doc:
if token.text not in vocab:
vocab[token.text] = word_idx
lemma_vocab[token.text] = token.lemma_
word_idx += 1
curr_tokens.append(vocab[token.text])
 train_tokens.append(np.array(curr_tokens, dtype=int))
test_tokens = []
for doc in tqdm(nlp.pipe(test_list)) :
curr_tokens = []
for token in doc:
if token.text not in vocab:
vocab[token.text] = word_idx
lemma_vocab[token.text] = token.lemma_
word_idx += 1
curr_tokens.append(vocab[token.text])
 test_tokens.append(np.array(curr_tokens, dtype=int))
market_train_df['close_open_ratio'] = np.abs(market_train_df['close']/market_train_df['open'])
 threshold = 0.5
print('In %i lines price increases by 50%% or more in a day' %(market_train_df['close_open_ratio']>=1.5 ).sum())
 print('In %i lines price decreases by 50%% or more in a day' %(market_train_df['close_open_ratio']<=0.5 ).sum() )<drop_column>
def pad(questions, seq_length):

 features = np.zeros(( len(questions), seq_length+1), dtype=int)
 for i, sentence in enumerate(questions):
if len(sentence)==0:
continue
features[i, 0] = len(sentence)
 features[i, -len(sentence):] = sentence
return features
market_train_df = market_train_df.loc[market_train_df['close_open_ratio'] < 1.5]
market_train_df = market_train_df.loc[market_train_df['close_open_ratio'] > 0.5]
market_train_df = market_train_df.drop(columns=['close_open_ratio'] )<data_type_conversions>
seq_length = max(max(map(len, train_tokens)) , max(map(len, test_tokens)) )
column_market = ['returnsClosePrevMktres1','returnsOpenPrevMktres1','returnsClosePrevMktres10', 'returnsOpenPrevMktres10']
column_raw = ['returnsClosePrevRaw1', 'returnsOpenPrevRaw1','returnsClosePrevRaw10', 'returnsOpenPrevRaw10']
for i in range(len(column_raw)) :
market_train_df[column_market[i]] = market_train_df[column_market[i]].fillna(market_train_df[column_raw[i]] )<define_variables>
train_tokens = pad(train_tokens, seq_length)
 test_tokens = pad(test_tokens, seq_length )
print('Removing outliers...')
 column_return = column_market + column_raw + ['returnsOpenNextMktres10']
orig_len = market_train_df.shape[0]
for column in column_return:
market_train_df = market_train_df.loc[market_train_df[column]>=-2]
market_train_df = market_train_df.loc[market_train_df[column]<=2]
new_len = market_train_df.shape[0]
rmv_len = np.abs(orig_len-new_len)
 print('There were %i lines removed' %rmv_len )<define_variables>
def get_embeddings(file):
embeddings = {}
with open(file, encoding="utf8", errors='ignore')as f:
for line in tqdm(f):
line_list = line.split(" ")
 if len(line_list)> 100:
embeddings[line_list[0]] = np.array(line_list[1:], dtype='float32')
 return embeddings
def get_embeddings_matrix(vocab, lemma_vocab, embeddings, keyedVector=False):
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
 n_words = len(vocab)
 if keyedVector:
emb_size = embeddings.vector_size
else:
emb_size = next(iter(embeddings.values())).shape[0]
if keyedVector:
emb_dict = {}
for word in vocab:
try:
emb_dict[word] = embeddings.get_vector(word)
 except:
continue
embeddings = emb_dict
embedding_matrix = np.zeros(( n_words+1, emb_size), dtype=np.float32)
 unknown_vec = np.zeros(( emb_size,), dtype=np.float32)- 1
unknown_words = 0
for word in tqdm(vocab):
emb_vec = embeddings.get(word)
 if emb_vec is not None:
embedding_matrix[vocab[word]] = emb_vec
continue
emb_vec = embeddings.get(lemma_vocab[word])
 if emb_vec is not None:
embedding_matrix[vocab[word]] = emb_vec
continue
emb_vec = embeddings.get(ps.stem(word))
if emb_vec is not None:
embedding_matrix[vocab[word]] = emb_vec
continue
emb_vec = embeddings.get(lc.stem(word))
if emb_vec is not None:
embedding_matrix[vocab[word]] = emb_vec
continue
emb_vec = embeddings.get(sb.stem(word))
if emb_vec is not None:
embedding_matrix[vocab[word]] = emb_vec
continue
embedding_matrix[vocab[word]] = unknown_vec
unknown_words += 1
print('% known words: {:.2%}'.format(1 - unknown_words/n_words))
return embedding_matrix
print('Removing strange data...')
 orig_len = market_train_df.shape[0]
market_train_df = market_train_df[~market_train_df['assetCode'].isin(['PGN.N','EBRYY.OB'])]
new_len = market_train_df.shape[0]
rmv_len = np.abs(orig_len-new_len)
 print('There were %i lines removed' %rmv_len )<create_dataframe>
glove_file = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
glove_emb = get_embeddings(glove_file)
 glove_emb_matrix = get_embeddings_matrix(vocab, lemma_vocab, glove_emb)
 del glove_emb
gc.collect()
def remove_outliers(data_frame, column_list, low=0.02, high=0.98):
for column in column_list:
this_column = data_frame[column]
quant_df = this_column.quantile([low,high])
 low_limit = quant_df[low]
high_limit = quant_df[high]
data_frame[column] = data_frame[column].clip(lower=low_limit, upper=high_limit)
 return data_frame<drop_column>
fasttext_file = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
fasttext_emb = get_embeddings(fasttext_file)
 fasttext_emb_matrix = get_embeddings_matrix(vocab, lemma_vocab, fasttext_emb)
 del fasttext_emb
gc.collect()
columns_outlier = ['takeSequence', 'bodySize', 'sentenceCount', 'wordCount', 'sentimentWordCount', 'firstMentionSentence','noveltyCount12H',\
'noveltyCount24H', 'noveltyCount3D', 'noveltyCount5D', 'noveltyCount7D', 'volumeCounts12H', 'volumeCounts24H',\
'volumeCounts3D','volumeCounts5D','volumeCounts7D']
print('Clipping news outliers...')
 news_train_df = remove_outliers(news_train_df, columns_outlier )<define_variables>
word2vec_file = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
word2vec_emb = KeyedVectors.load_word2vec_format(word2vec_file, binary=True)
 word2vec_emb_matrix = get_embeddings_matrix(vocab, lemma_vocab, word2vec_emb, keyedVector=True)
 del word2vec_emb
gc.collect()
asset_code_dict = {k: v for v, k in enumerate(market_train_df['assetCode'].unique())}
drop_columns = [col for col in news_train_df.columns if col not in ['sourceTimestamp', 'urgency', 'takeSequence', 'bodySize', 'companyCount',
'sentenceCount', 'firstMentionSentence', 'relevance','firstCreated', 'assetCodes']]
columns_news = ['firstCreated','relevance','sentimentClass','sentimentNegative','sentimentNeutral',
'sentimentPositive','noveltyCount24H','noveltyCount7D','volumeCounts24H','volumeCounts7D','assetCodes','sourceTimestamp',
'assetName','audiences', 'urgency', 'takeSequence', 'bodySize', 'companyCount',
'sentenceCount', 'firstMentionSentence','time']<feature_engineering>
paragram_file = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
paragram_emb = get_embeddings(paragram_file)
 paragram_emb_matrix = get_embeddings_matrix(vocab, lemma_vocab, paragram_emb)
 del paragram_emb
gc.collect()
def data_prep(market_df,news_df):
market_df['date'] = market_df.time.dt.date
market_df['close_to_open'] = market_df['close'] / market_df['open']
market_df.drop(['time'], axis=1, inplace=True)
 news_df = news_df[columns_news]
news_df['sourceTimestamp']= news_df.sourceTimestamp.dt.hour
news_df['firstCreated'] = news_df.firstCreated.dt.date
news_df['assetCodesLen'] = news_df['assetCodes'].map(lambda x: len(eval(x)))
 news_df['assetCodes'] = news_df['assetCodes'].map(lambda x: list(eval(x)) [0])
 news_df['asset_sentiment_count'] = news_df.groupby(['assetName', 'sentimentClass'])['time'].transform('count')
 news_df['len_audiences'] = news_train_df['audiences'].map(lambda x: len(eval(x)))
 kcol = ['firstCreated', 'assetCodes']
news_df = news_df.groupby(kcol, as_index=False ).mean()
market_df = pd.merge(market_df, news_df, how='left', left_on=['date', 'assetCode'],
right_on=['firstCreated', 'assetCodes'])
 del news_df
market_df['assetCodeT'] = market_df['assetCode'].map(asset_code_dict)
 market_df = market_df.drop(columns = ['firstCreated','assetCodes','assetName'] ).fillna(0)
 return market_df<categorify>
emb_matrix = np.concatenate(( glove_emb_matrix,
paragram_emb_matrix), axis=1)
 del glove_emb_matrix, fasttext_emb_matrix, word2vec_emb_matrix, paragram_emb_matrix
gc.collect()
print('Merging data...')
 market_train_df = data_prep(market_train_df, news_train_df)
 market_train_df.head()<filter>
train_feat = np.concatenate(( train_stat, train_tokens), axis=1)
 test_feat = np.concatenate(( test_stat, test_tokens), axis=1 )
market_train_df = market_train_df.loc[market_train_df['date']>=datetime.date(2009,1,1)]<define_variables>
x_train, x_val, label_train, label_val = train_test_split(train_feat, labels, test_size=0.1, random_state=0)
 train_data = TensorDataset(torch.from_numpy(x_train), torch.from_numpy(label_train))
valid_data = TensorDataset(torch.from_numpy(x_val), torch.from_numpy(label_val))
test_data = TensorDataset(torch.from_numpy(test_feat))
batch_size = 64
train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)
 valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size)
 test_loader = DataLoader(test_data, shuffle=False, batch_size=batch_size )
num_columns = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1', 'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10', 'close_to_open', 'sourceTimestamp', 'urgency', 'companyCount', 'takeSequence', 'bodySize', 'sentenceCount',
'relevance', 'sentimentClass', 'sentimentNegative', 'sentimentNeutral', 'sentimentPositive',
'noveltyCount24H','noveltyCount7D','volumeCounts24H','volumeCounts7D','assetCodesLen', 'asset_sentiment_count', 'len_audiences']
cat_columns = ['assetCodeT']
feature_columns = num_columns+cat_columns<normalization>
train_on_gpu=torch.cuda.is_available()
if train_on_gpu:
print('Training on GPU.')
 else:
print('No GPU available, training on CPU.' )
data_scaler = StandardScaler()
market_train_df[num_columns] = data_scaler.fit_transform(market_train_df[num_columns] )<split>
def init_emb_layer(self, embedding_matrix):

 embedding_matrix = torch.tensor(embedding_matrix, dtype=torch.float32)
 num_emb, emb_size = embedding_matrix.size()
emb_layer = nn.Embedding.from_pretrained(embedding_matrix)
 return emb_layer
class SelfAttention(nn.Module):

 def __init__(self, attention_size, batch_first=False, non_linearity="tanh"):
super(SelfAttention, self ).__init__()
self.batch_first = batch_first
self.attention_weights = nn.Parameter(torch.FloatTensor(attention_size))
self.softmax = nn.Softmax(dim=-1)
 if non_linearity == "relu":
self.non_linearity = nn.ReLU()
else:
self.non_linearity = nn.Tanh()
nn.init.uniform(self.attention_weights.data, -0.005, 0.005)
 def get_mask(self, attentions, lengths):

 max_len = max(lengths.data)
 mask = torch.autograd.Variable(torch.ones(attentions.size())).detach()
if attentions.data.is_cuda:
mask = mask.cuda()
for i, l in enumerate(lengths.data):
if l < max_len:
mask[i, :-l] = 0
return mask
def forward(self, inputs, lengths):
scores = self.non_linearity(inputs.matmul(self.attention_weights))
scores = self.softmax(scores)
 mask = self.get_mask(scores, lengths)
 masked_scores = scores * mask
_sums = masked_scores.sum(-1, keepdim=True)
 scores = masked_scores.div(_sums)
 weighted = torch.mul(inputs, scores.unsqueeze(-1 ).expand_as(inputs))
representations = weighted.sum(1 ).squeeze()
return representations
class Quora_model(nn.Module):
def __init__(self, hidden_layer_dim, embedding_matrix, hidden_dim, gru_layers, stat_layers, drop_prob=0.5):

 super(Quora_model, self ).__init__()
self.hidden_layer_dim = hidden_layer_dim
self.gru_layers = gru_layers
self.emb_dim = embedding_matrix.shape[1]
self.hidden_dim = hidden_dim
self.stat_layers = stat_layers
stat_in_dim = n_stats + 1
modules = []
for out_dim in self.stat_layers:
modules.append(nn.Linear(stat_in_dim, out_dim))
modules.append(nn.ReLU())
 stat_in_dim = out_dim
self.stat_dense = nn.Sequential(*modules)
 self.embedding = init_emb_layer(self, embedding_matrix)
 self.gru = nn.GRU(self.emb_dim, self.hidden_dim, self.gru_layers,
batch_first=True, bidirectional=True, dropout = drop_prob)
 self.attention = SelfAttention(self.hidden_dim*2, batch_first=True)
 self.final_dense = nn.Sequential(
 nn.Dropout(p=drop_prob),
nn.Linear(self.hidden_dim*2 + out_dim, self.hidden_layer_dim),
nn.ReLU() ,
nn.Dropout(p=drop_prob),
nn.Linear(self.hidden_layer_dim, 1),
nn.Sigmoid() 
)
 def forward(self, x, hidden):
batch_size, _ = x.size()
if hidden.size(1)!= batch_size:
hidden = hidden[:, :batch_size, :].contiguous()
lengths = x[:,n_stats].cpu().numpy().astype(int)
 seq_len = max(lengths)
 x_text = x[:, -seq_len:]
x_stat = x[:, :n_stats+1].type(torch.FloatTensor)
 if train_on_gpu:
x_stat = x_stat.cuda()
x_text = self.embedding(x_text)
 out_gru, _ = self.gru(x_text, hidden)
 out_att = self.attention(out_gru, lengths)
 out_stat = self.stat_dense(x_stat)
 out = torch.cat(( out_att, out_stat), dim=1)
 out = self.final_dense(out)
 return out
def init_hidden(self, batch_size):

 weight = next(self.parameters() ).data
if train_on_gpu:
hidden = weight.new(self.gru_layers*2, batch_size, self.hidden_dim ).zero_().cuda()
else:
hidden = weight.new(self.gru_layers*2, batch_size, self.hidden_dim ).zero_()
return hidden
market_train_df = market_train_df.reset_index()
market_train_df = market_train_df.drop(columns='index')
 train_indices, val_indices = train_test_split(market_train_df.index.values,test_size=0.1, random_state=92 )<prepare_x_and_y>
hidden_dim = 256
gru_layers = 1
dropout = 0.1
stat_layers_dim = [16, 8]
hidden_layer_dim = 64
model = Quora_model(hidden_layer_dim, emb_matrix, hidden_dim, gru_layers, stat_layers_dim, dropout)
 model
def get_input(market_train, indices):
X = market_train.loc[indices, feature_columns].values
y = market_train.loc[indices,'returnsOpenNextMktres10'].map(lambda x: 0 if x<0 else 1 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'date']
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train_df, train_indices)
 X_val,y_val,r_val,u_val,d_val = get_input(market_train_df, val_indices )<normalization>
epochs = 4
print_every = 1000
early_stop = 20
clip = 5
lr=0.001
criterion = nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters() , lr=lr )
def learning_rate_power(current_round):
base_learning_rate = 0.19000424246380565
min_learning_rate = 0.01
lr = base_learning_rate * np.power(0.995,current_round)
 return max(lr, min_learning_rate )<define_search_space>
def train_model(model, train_loader, valid_loader, batch_size, epochs,
optimizer, criterion, clip, print_every, early_stop):
if(train_on_gpu):
model.cuda()
counter = 0
model.train()
breaker = False
for e in range(epochs):
for inputs, labels in train_loader:
counter += 1
if(train_on_gpu):
inputs, labels = inputs.cuda() , labels.cuda()
h = model.init_hidden(batch_size)
 model.zero_grad()
output = model(inputs, h)
 loss = criterion(output.squeeze() , labels.float())
 loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , clip)
 optimizer.step()
if counter % print_every == 0:
with torch.no_grad() :
val_losses = []
all_val_labels = []
all_val_preds = []
all_val_probs = []
model.eval()
for inputs, labels in valid_loader:
all_val_labels += list(labels)
 if(train_on_gpu):
inputs, labels = inputs.cuda() , labels.cuda()
val_h = model.init_hidden(batch_size)
 output = model(inputs, val_h)
 val_loss = criterion(output.squeeze() , labels.float())
 val_losses.append(val_loss.item())
 preds = torch.round(output.squeeze())
 preds = np.squeeze(preds.cpu().numpy())
 all_val_preds += list(preds)
 output = np.squeeze(output.cpu().detach().numpy())
 all_val_probs += list(output)
 current_loss = np.mean(val_losses)
 print("Epoch: {}/{}...".format(e+1, epochs),
"Step: {}...".format(counter),
"Loss: {:.6f}...".format(loss.item()),
"Val Loss: {:.6f}...".format(current_loss),
"F1-score(threshold=0.5): {:.3%}".format(f1_score(all_val_labels, all_val_preds)))
 if counter == print_every or current_loss < best_loss:
best_loss = current_loss
best_val_labels = all_val_labels
best_probs = all_val_probs
torch.save(model.state_dict() , 'checkpoint.pth')
 counter_eval = 0
counter_eval += 1
if counter_eval == early_stop:
breaker = True
break
model.train()
if breaker:
break
state_dict = torch.load('checkpoint.pth')
 model.load_state_dict(state_dict)
 return best_probs, best_val_labels
tune_params = {'n_estimators': [200,500,1000,2500,5000],
'max_depth': sp_randint(4,12),
'colsample_bytree':sp_uniform(loc=0.8, scale=0.15),
'min_child_samples':sp_randint(60,120),
'subsample': sp_uniform(loc=0.75, scale=0.25),
'reg_lambda':[1e-3, 1e-2, 1e-1, 1]}
fit_params = {'early_stopping_rounds':40,
'eval_metric': 'accuracy',
'eval_set': [(X_train, y_train),(X_val, y_val)],
'verbose': 20,
'callbacks': [lgb.reset_parameter(learning_rate=learning_rate_power)]}<choose_model_class>
t0 = time.time()
all_val_probs, all_val_labels = train_model(model, train_loader, valid_loader, batch_size, epochs,
optimizer, criterion, clip, print_every, early_stop)
 tf = time.time()
print("
Execution time: {:.2f}min".format(( tf-t0)/60))
lgb_clf = lgb.LGBMClassifier(n_jobs=4, objective='binary',random_state=1)
 gs = RandomizedSearchCV(estimator=lgb_clf,
param_distributions=tune_params,
n_iter=40,
scoring='f1',
cv=5,
refit=True,
random_state=1,
verbose=True )<train_model>
best_score = 0
for thr in np.arange(0.0, 0.5, 0.005):
pred = np.array(all_val_probs > thr, dtype=int)
 score = f1_score(all_val_labels, pred)
 print("Threshold: {:.3f}...F1-score {:.3%}".format(thr, score))
if score > best_score:
best_score = score
best_thr = thr
print("
Best threshold: {:.3f}...F1-score {:.3%}".format(best_thr, best_score))
lgb_clf = lgb.LGBMClassifier(n_jobs=4,
objective='multiclass',
random_state=100)
 opt_params = {'n_estimators':500,
'boosting_type': 'dart',
'objective': 'binary',
'num_leaves':2452,
'min_child_samples':212,
'reg_lambda':0.01}
lgb_clf.set_params(**opt_params)
 lgb_clf.fit(X_train, y_train,**fit_params )<compute_test_metric>
model.eval()
with torch.no_grad() :
all_test_preds = []
for inputs in test_loader:
inputs = inputs[0]
if(train_on_gpu):
inputs = inputs.cuda()
test_h = model.init_hidden(batch_size)
 output = model(inputs, test_h)
 preds =(output.squeeze() > best_thr ).type(torch.IntTensor)
 preds = np.squeeze(preds.cpu().numpy())
 all_test_preds += list(preds.astype(int))
<normalization><EOS> 
sub = pd.DataFrame({
'qid': test_df.qid,
'prediction': all_test_preds
})
 sub = sub[['qid', 'prediction']]
sub.to_csv('submission.csv', index=False, sep=',' )