Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<count_values>
import re
import time
import gc
import random
import os
import nltk
import multiprocess as mp
import numpy as np
import pandas as pd
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import f1_score, roc_curve, precision_recall_curve
from sklearn.preprocessing import StandardScaler
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import torch
import torch.nn as nn
import torch.utils.data
from torch.nn import Parameter
train_df.Type.value_counts()<feature_engineering>
embed_size = 300
max_features = 120000
maxlen = 50
batch_size = 512
train_epochs = 6
SEED = 1029
train_df.loc[124,'Type'] = 'IL'<count_values>
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(SEED )
test_df.Type.value_counts()<count_values>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )
train_df.P1.value_counts()<concatenate>
def preprocess(df):
df.question_text = df["question_text"].apply(lambda x: x.lower())
 df.question_text = df["question_text"].apply(lambda x: clean_text(x))
df.question_text = df["question_text"].apply(lambda x: clean_numbers(x))
df.question_text = df["question_text"].apply(lambda x: replace_typical_misspell(x))
df['total_length'] = df['question_text'].apply(len)
 df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
texts = df.question_text.tolist()
tagged_texts = nltk.pos_tag_sents(map(nltk.word_tokenize, texts))
df["tagged_texts"] = tagged_texts
df.tagged_texts = df["tagged_texts"].apply(lambda x: " ".join([i[1] for i in x]))
return df
chunksize = 10000
comp_df = pd.concat([train_df, test_df])
 comp_df.reset_index(drop=True, inplace=True )<count_values>
train_reader = pd.read_csv(".. /input/train.csv", chunksize=chunksize)
 start_time = time.time()
pool = mp.Pool(4)
 funclist = []
for df in train_reader:
f = pool.apply_async(preprocess, [df])
 funclist.append(f)
 result = []
for f in funclist:
result.append(f.get(timeout=120))
train_df = pd.concat(result)
 elapsed_time =(time.time() - start_time)/ 60
print(elapsed_time )
comp_df.Type.value_counts() ,comp_df['City Group'].value_counts()<categorify>
test_reader = pd.read_csv(".. /input/test.csv", chunksize=chunksize)
 start_time = time.time()
pool = mp.Pool(4)
 funclist = []
for df in test_reader:
f = pool.apply_async(preprocess, [df])
 funclist.append(f)
 result = []
for f in funclist:
result.append(f.get(timeout=120))
test_df = pd.concat(result)
 elapsed_time =(time.time() - start_time)/ 60
print(elapsed_time )
comp_df.Type = comp_df.Type.map({'MB':0,'DT':1, 'IL':2,'FC':3})
 comp_df['City Group'] = comp_df['City Group'].map({'Big Cities':1, 'Other':0} )<normalization>
start_time = time.time()
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
train_tag_X = train_df["tagged_texts"].fillna("_
test_tag_X = test_df["tagged_texts"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 tag_tokenizer = Tokenizer(num_words=max_features)
 tag_tokenizer.fit_on_texts(list(train_tag_X))
train_tag_X = tag_tokenizer.texts_to_sequences(train_tag_X)
 test_tag_X = tag_tokenizer.texts_to_sequences(test_tag_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_tag_X = pad_sequences(train_tag_X, maxlen=maxlen)
 test_tag_X = pad_sequences(test_tag_X, maxlen=maxlen)
 train_features = train_df[['caps_vs_length', 'words_vs_unique']].fillna(0)
 test_features = test_df[['caps_vs_length', 'words_vs_unique']].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( train_features, test_features)))
 train_features = ss.transform(train_features)
 test_features = ss.transform(test_features)
 elapsed_time =(time.time() - start_time)/ 60
print(elapsed_time )
p_name = ['P'+str(i)for i in range(1,38)]
comp_df[p_name] = MinMaxScaler().fit_transform(comp_df[p_name] )<categorify>
train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_tag_X = train_tag_X[trn_idx]
train_features = train_features[trn_idx]
train_y = train_y[trn_idx]
pca_list = ['pca'+str(i)for i in range(1,30,1)]
comp_df[pca_list] = PCA(n_components=29 ).fit_transform(comp_df[p_name])
 comp_df.drop(p_name,axis=1,inplace=True )<feature_engineering>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
comp_df['launch_days'] =(datetime.datetime.now() - comp_df[['Open Date']])
 comp_df['launch_days'] = comp_df['launch_days'].dt.days<drop_column>
start_time = time.time()
word_index = tokenizer.word_index
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_para(word_index)
 embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2], axis=0)
 print(np.shape(embedding_matrix))
total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))
del embedding_matrix_1, embedding_matrix_2
gc.collect()
comp_df.drop('Open Date',axis=1,inplace=True )<normalization>
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )
comp_df['launch_days'] = MinMaxScaler().fit_transform(comp_df[['launch_days']])
 comp_df['open_year'] = MinMaxScaler().fit_transform(comp_df[['open_year']] )<drop_column>
class BackHook(torch.nn.Module):
def __init__(self, hook):
super(BackHook, self ).__init__()
self._hook = hook
self.register_backward_hook(self._backward)
 def forward(self, *inp):
return inp
@staticmethod
def _backward(self, grad_in, grad_out):
self._hook()
return None
class WeightDrop(torch.nn.Module):

 def __init__(self, module, weights, dropout=0, variational=False):
super(WeightDrop, self ).__init__()
self.module = module
self.weights = weights
self.dropout = dropout
self.variational = variational
self._setup()
self.hooker = BackHook(lambda: self._backward())
 def _setup(self):
for name_w in self.weights:
w = getattr(self.module, name_w)
 self.register_parameter(name_w + '_raw', Parameter(w.data))
def _setweights(self):
for name_w in self.weights:
raw_w = getattr(self, name_w + '_raw')
 mask = raw_w.new_ones(( raw_w.size(0), 1))
mask = torch.nn.functional.dropout(mask, p=self.dropout, training=True)
 w = mask.expand_as(raw_w)* raw_w
rnn_w = getattr(self.module, name_w)
 rnn_w.data.copy_(w)
 setattr(self, name_w + "_mask", mask)
 def _backward(self):
for name_w in self.weights:
raw_w = getattr(self, name_w + '_raw')
 rnn_w = getattr(self.module, name_w)
 raw_w.grad = rnn_w.grad * getattr(self, name_w + "_mask")
 def forward(self, *args):
self._setweights()
return self.module(*self.hooker(*args))
test_df = comp_df[comp_df['revenue'].isnull() ]
train_df = comp_df[comp_df['revenue'].notnull() ]
test_df.drop('revenue',axis=1, inplace=True )<prepare_x_and_y>
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 60
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.lstm = WeightDrop(nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.1)
 self.gru = WeightDrop(nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.1)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.tag_lstm = nn.LSTM(maxlen, hidden_size, bidirectional=True, batch_first=True)
 self.tag_gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.linear = WeightDrop(nn.Linear(hidden_size*8+2, 32), ['weight'], dropout=0.1)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(32, 1)
 def forward(self, x, y, z):
x_embedding = self.embedding(x)
 x_lstm, _ = self.lstm(x_embedding)
 x_gru, _ = self.gru(x_lstm)
 x_lstm_atten = self.lstm_attention(x_lstm)
 x_gru_atten = self.gru_attention(x_gru)
 x_max_pool, _ = torch.max(x_gru, 1)
 y_lstm, _ = self.tag_lstm(y.view(len(y), -1, maxlen))
y_gru, _ = self.tag_gru(y_lstm)
 y_max_pool_1, _ = torch.max(y_lstm, 1)
 y_max_pool_2, _ = torch.max(y_gru, 1)
 y_max_pool = torch.add(y_max_pool_1, y_max_pool_2)
 conc = torch.cat(( x_lstm_atten, x_gru_atten, x_max_pool, y_max_pool, z), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out
x_train = train_df.drop('revenue',axis=1)
 y_train = train_df['revenue']<import_modules>
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED ).split(train_X, train_y))
from lightgbm import LGBMRegressor
from sklearn.model_selection import GridSearchCV, cross_validate, RepeatedKFold<compute_train_metric>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
cv = RepeatedKFold(n_splits=10, n_repeats=3)
 scores = cross_validate(LGBMRegressor() , x_train,y_train, scoring=['r2','neg_root_mean_squared_error'],cv=cv )<compute_test_metric>
def threshold_search(y_true, y_proba):
precision, recall, thresholds = precision_recall_curve(y_true, y_proba)
 thresholds = np.append(thresholds, 1.001)
 F = 2 /(1/precision + 1/recall)
 best_score = np.max(F)
 best_th = thresholds[np.argmax(F)]
search_result = {'threshold': best_th , 'f1': best_score}
return search_result
r2 = scores['test_r2']
rmse = scores['test_neg_root_mean_squared_error']
print(np.mean(r2),np.mean(rmse))<train_model>
x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
x_tag_test_cuda = torch.tensor(test_tag_X, dtype=torch.float32 ).cuda()
features_test_cuda = torch.tensor(test_features, dtype=torch.float32 ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda, x_tag_test_cuda, features_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )
final_model = LGBMRegressor(boosting_type='dart',max_depth=3,n_estimators=20,random_state=42, subsample=0.3 ).fit(x_train,y_train )<load_from_csv>
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
x_tag_train_fold = torch.tensor(train_tag_X[train_idx], dtype=torch.float32 ).cuda()
feature_train_fold = torch.tensor(train_features[train_idx], dtype=torch.float32 ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
x_tag_val_fold = torch.tensor(train_tag_X[valid_idx], dtype=torch.float32 ).cuda()
feature_val_fold = torch.tensor(train_features[valid_idx], dtype=torch.float32 ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="sum")
 train = torch.utils.data.TensorDataset(x_train_fold, x_tag_train_fold, feature_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, x_tag_val_fold, feature_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 best_f1 = 0.
 train_preds_tmp = np.zeros(( len(train_X)))
 test_preds_tmp = np.zeros(( len(test_X)))
 for epoch in range(train_epochs):
start_time = time.time()
if epoch < 4:
optimizer = torch.optim.Adam(model.parameters())
 else:
optimizer = torch.optim.Adam(model.parameters() , lr=0.0001)
 model.train()
avg_loss = 0.
 for x_batch, x_tag_batch, feature_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch, x_tag_batch, feature_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 valid_true_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, x_tag_batch, feature_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch, x_tag_batch, feature_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
valid_true_fold[i * batch_size:(i+1)* batch_size] = y_batch.cpu().numpy() [:, 0]
elapsed_time = time.time() - start_time
search_result = threshold_search(valid_true_fold, valid_preds_fold)
 if search_result['f1'] > best_f1:
train_preds_tmp[valid_idx] = valid_preds_fold
for i,(x_batch, x_tag_batch, feature_batch)in enumerate(test_loader):
y_pred = model(x_batch, x_tag_batch, feature_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
test_preds_tmp = test_preds_fold / len(splits)
 print('Epoch {}/{}\tloss={:.4f}\tval_loss={:.4f}\tthreshold={:.2f}\tval_f1={:.8f}\ttime={:.2f}s [impoved]'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, search_result['threshold'], search_result['f1'], elapsed_time))
best_f1 = search_result['f1']
else:
print('Epoch {}/{}\tloss={:.4f}\tval_loss={:.4f}\tthreshold={:.2f}\tval_f1={:.8f}\ttime={:.2f}s [not impoved]'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, search_result['threshold'], search_result['f1'], elapsed_time))
train_preds[valid_idx] += train_preds_tmp[valid_idx]
test_preds += test_preds_tmp
test_file = pd.read_csv('.. /input/restaurant-revenue-prediction/test.csv.zip')
 answer = pd.DataFrame(final_model.predict(test_df))
answer.columns = ['Prediction']
answer['Id'] = test_file.index.tolist()
answer.set_index('Id',inplace=True )<save_to_csv>
search_result = threshold_search(train_y, train_preds)
 search_result
answer.to_csv('result.csv' )<load_from_csv>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False)
 grouped = sub.groupby("prediction" ).size()
print(grouped)
 print(grouped[1] / grouped[0] )
!unzip /kaggle/input/restaurant-revenue-prediction/test.csv.zip
!unzip /kaggle/input/restaurant-revenue-prediction/train.csv.zip<import_modules>
def find_word(word, x):
regex = r"\b{}\b".format(word)
 return True if re.search(regex, x)else False
import numpy as np
import pandas as pd
from sklearn.linear_model import BayesianRidge, ElasticNet, Lasso, Ridge, LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV, cross_val_score
from sklearn.metrics import make_scorer, mean_squared_log_error
from sklearn.preprocessing import StandardScaler
from bayes_opt import BayesianOptimization
import matplotlib.pyplot as plt<set_options>
for i,q in enumerate(test_df.question_text.iloc[:]):
if find_word("anal", q):
print(sub.prediction.iloc[i], " : ", q )