Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<choose_model_class>
import re
import time
import gc
import random
import os
import numpy as np
import pandas as pd
from tqdm import tqdm
from tabulate import tabulate
from sklearn.model_selection import train_test_split
from sklearn import metrics, preprocessing
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, roc_auc_score
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import torch
import torch.nn as nn
import torch.utils.data
model3 = Sequential()
model3.add(Conv2D(filters = 32, kernel_size =(3,3), padding = 'valid', activation = 'relu', input_shape = input_shape))
model3.add(Conv2D(filters = 32, kernel_size =(3,3), padding = 'valid', activation = 'relu'))
model3.add(Dropout(0.25))
model3.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'valid', activation = 'relu'))
model3.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'valid', activation = 'relu'))
model3.add(MaxPool2D(pool_size=(2,2)))
 model3.add(Dropout(0.33))
model3.add(Conv2D(filters = 96, kernel_size =(3,3), padding = 'valid', activation = 'relu'))
model3.add(Conv2D(filters = 96, kernel_size =(3,3), padding = 'valid', activation = 'relu'))
model3.add(MaxPool2D(pool_size=(2,2)))
 model3.add(Dropout(0.4))
model3.add(Flatten())
 model3.add(Dense(512, activation='relu'))
model3.add(Dropout(0.50))
model3.add(Dense(1024, activation='relu'))
model3.add(Dropout(0.50))
model3.add(Dense(10, activation='softmax'))<choose_model_class>
if not os.path.exists('./embeddings'):
os.mkdir('./embeddings')
 os.mkdir('./embeddings/glove.840B.300d/')
 os.mkdir('./embeddings/paragram_300_sl999/')
 with zipfile.ZipFile('.. /input/quora-insincere-questions-classification/embeddings.zip', 'r')as z:
with z.open('glove.840B.300d/glove.840B.300d.txt')as zf, open('./embeddings/glove.840B.300d/glove.840B.300d.txt', 'wb')as f:
shutil.copyfileobj(zf, f)
 with z.open('paragram_300_sl999/paragram_300_sl999.txt')as zf, open('./embeddings/paragram_300_sl999/paragram_300_sl999.txt', 'wb')as f:
shutil.copyfileobj(zf, f )
model3.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'] )<train_model>
embed_size = 300
meta_size = 3
max_features = 30000
maxlen = 72
batch_size = 1000
train_epochs = 4
SEED = 1029
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
model3.fit(X_train, y_train, epochs = 100, batch_size = 128, verbose = 1, callbacks = [early_stop], shuffle = True )<predict_on_test>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
''
def clean_and_lower_text(x):
x = str(x)
 x = x.encode("ascii", errors="ignore" ).decode()
for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
x = re.sub('[0-9]{1}', '
return x
mispell_dict = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will",
"how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am",
"I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am",
"i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have",
"it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not",
"mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not",
"needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will",
"she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have",
"so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is",
"there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would",
"they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have",
"to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have",
"we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are",
"what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is",
"why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not",
"wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are",
"y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are",
"you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling',
'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',
'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are',
'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does',
'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist',
'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what',
'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 spaces_re = re.compile('\s')
 number_re = re.compile('[0-9]')
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def have_pattern(text, pattern_re):
return int(bool(pattern_re.search(text)) )
predictions3 = model1.predict(X_test )<define_variables>
def load_and_prec() :
train_df = pd.read_csv(".. /input/quora-insincere-questions-classification/train.csv")
 test_df = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv")
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_and_lower_text(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_and_lower_text(x))
train_df["meta_number"] = train_df["question_text"].progress_apply(lambda x: have_pattern(x,number_re))
test_df["meta_number"] = test_df["question_text"].progress_apply(lambda x: have_pattern(x,number_re))
train_df["meta_mispel"] = train_df["question_text"].progress_apply(lambda x: have_pattern(x,mispellings_re))
test_df["meta_mispel"] = test_df["question_text"].progress_apply(lambda x: have_pattern(x,mispellings_re))
train_df["meta_words_nb"] = train_df["question_text"].progress_apply(lambda x : len(spaces_re.findall(x)))
 test_df["meta_words_nb"] = test_df["question_text"].progress_apply(lambda x : len(spaces_re.findall(x)))
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
meta_train_X = train_df[[col for col in train_df if col.startswith('meta')]]
meta_test_X = test_df[[col for col in test_df if col.startswith('meta')]]
meta_train_X = preprocessing.scale(meta_train_X, with_mean='True', with_std='True')[trn_idx]
meta_test_X = preprocessing.scale(meta_test_X, with_mean='True', with_std='True')
 return train_X, test_X, train_y, tokenizer.word_index, meta_train_X, meta_test_X
predictions =(predictions1 + predictions2 + predictions3)/3
Predictions = []
for i in range(0, len(predictions)) :
Predictions.append(predictions[i].argmax())
 id = []
for i in range(1, len(Predictions)+1):
id.append(i )<save_to_csv>
tqdm.pandas()
if not os.path.exists('./train_X.npy'):
start_time = time.time()
train_X, test_X, train_y, word_index, meta_train_X, meta_test_X = load_and_prec()
total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))

 else :
train_X = np.load('./train_X.npy', allow_pickle=True)
 test_X = np.load('./test_X.npy', allow_pickle=True)
 meta_train_X = np.load('./meta_train_X.npy', allow_pickle=True)
 meta_test_X = np.load('./meta_test_X.npy', allow_pickle=True)
 train_y = np.load('./train_y.npy', allow_pickle=True)
 with open('./word_index.pickle', 'rb')as handle:
tokenizer = pickle.load(handle)
 gc.collect()
output = pd.DataFrame({'ImageId': id, 'Label': Predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!")
<import_modules>
def load_embedding(word_index, EMBEDDING_FILE):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 if(EMBEDDING_FILE=='./embeddings/paragram_300_sl999/paragram_300_sl999.txt'):
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 else :
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index)+1)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
import tensorflow as tf
from tensorflow import keras
import matplotlib.pyplot as plt
import matplotlib.cm as cm<load_from_csv>
if not os.path.exists('./embedding_matrix.npy'):
start_time = time.time()
embedding_glove = load_embedding(word_index, './embeddings/glove.840B.300d/glove.840B.300d.txt')
 embedding_paragram = load_embedding(word_index, './embeddings/paragram_300_sl999/paragram_300_sl999.txt')
 embedding_matrix = np.mean([embedding_glove, embedding_paragram], axis=0)
 print(np.shape(embedding_matrix))
total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))
del embedding_glove, embedding_paragram
else :
embedding_matrix = np.load('./embedding_matrix.npy', allow_pickle=True)
 gc.collect()
train_data_raw = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test_data_raw = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 print('train_data_shape: ' + str(train_data_raw.shape))
print('test_data_shape: ' + str(test_data_raw.shape))<categorify>
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )
def pre_process_data(dataset, has_label=True):
if has_label:
data = dataset.iloc[:,1:].values
else:
data = dataset.iloc[:,:].values
data = data.astype(np.float)
 data = np.multiply(data, 1.0 / 255.0)
 data = data.reshape(data.shape[:1] +(28, 28, 1))
return data
train_images = pre_process_data(train_data_raw)
 print(train_images.shape)
 print(train_images )<categorify>
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 60
self.embedding = nn.Embedding(np.shape(embedding_matrix)[0], embed_size, padding_idx=0)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.linear_meta = nn.Linear(meta_size, meta_size)
 self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.linear = nn.Linear(480, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.softmax = nn.Softmax(dim=1)
 self.out = nn.Linear(16 + meta_size, 1)
 def forward(self, x, meta_x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
h_meta = self.linear_meta(meta_x.type(torch.float))
conc = torch.cat(( conc, h_meta), 1)
 conc = self.dropout(conc)
 out = self.out(conc)
 return out
def pre_process_labels(data):
labels_flat = data.iloc[:,0].values.ravel()
num_classes = np.unique(labels_flat ).shape[0]
num_labels = labels_flat.shape[0]
index_offset = np.arange(num_labels)* num_classes
labels_one_hot = np.zeros(( num_labels, num_classes))
labels_one_hot.flat[index_offset + labels_flat.ravel() ] = 1
return labels_one_hot.astype(np.uint8)
 train_labels = pre_process_labels(train_data_raw)
 print(train_labels.shape)
 print(train_labels )<train_on_grid>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def fit(train_images, train_labels, config):
NUM_EPOCHS = config['NUM_EPOCHS']
BATCH_SIZE = config['BATCH_SIZE']
LEARNING_RATE = config['LEARNING_RATE']
DROPOUT_RATE = config['DROPOUT_RATE']
NETWORK_WIDTH = config['NETWORK_WIDTH']
initializer = tf.keras.initializers.GlorotNormal()
model = keras.models.Sequential([
keras.layers.Conv2D(32,kernel_size=(3, 3),activation='swish',kernel_initializer=initializer, input_shape=(28,28,1), use_bias=True),
keras.layers.Conv2D(32,kernel_size=(3,3), activation='swish',kernel_initializer=initializer, use_bias=True),
keras.layers.MaxPooling2D(pool_size=(2,2)) ,
keras.layers.Dropout(DROPOUT_RATE),
keras.layers.Conv2D(64,(3, 3),activation='swish', padding='same', use_bias=True),
keras.layers.MaxPooling2D(pool_size=(2,2)) ,
keras.layers.Dropout(DROPOUT_RATE),
keras.layers.Conv2D(512,(3, 3),activation='swish', padding='same', use_bias=True),
keras.layers.MaxPooling2D(pool_size=(2,2)) ,
keras.layers.Dropout(DROPOUT_RATE),
keras.layers.Conv2D(256,(2, 2),activation='swish', padding='same', use_bias=True),
keras.layers.MaxPooling2D(pool_size=(2,2)) ,
keras.layers.Dropout(DROPOUT_RATE),
keras.layers.Flatten() ,
keras.layers.Dense(128,activation='swish', kernel_regularizer =tf.keras.regularizers.l1(l=0.01)) ,
keras.layers.Dropout(DROPOUT_RATE),
keras.layers.Dense(10, activation=tf.nn.softmax),
])
 opt = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)
 loss = tf.keras.losses.CategoricalCrossentropy()
model.compile(
 optimizer=opt,
loss=loss,
metrics=['accuracy'] 
)
 val_size = int(train_images.shape[0] * 0.2)
 val_images = train_images[:val_size,:, :]
val_labels = train_labels[:val_size,:]
train_images = train_images[val_size:,:, :]
train_labels = train_labels[val_size:,:]
hist = model.fit(
 x=train_images, y=train_labels,
epochs=NUM_EPOCHS,
batch_size=BATCH_SIZE,
validation_steps=10,
validation_data=(val_images, val_labels),
verbose=1 
)
 show_history(hist)
 loss, acc = model.evaluate(val_images, val_labels)
 return model, loss, acc<train_model>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
config = {
'BATCH_SIZE': 128,
'LEARNING_RATE': 0.001,
'DROPOUT_RATE': 0.4,
'NUM_EPOCHS': 50,
'NETWORK_WIDTH': 512
}
model, loss, acc = fit(train_images, train_labels, config)
 print('Accuracy: ' + str(acc * 100)+ '%' )<predict_on_test>
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED ).split(train_X, train_y))
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 seed_torch(SEED)
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
x_meta_test_cuda = torch.tensor(meta_test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda, x_meta_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
x_meta_train_fold = torch.tensor(meta_train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
x_meta_val_fold = torch.tensor(meta_train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()

 model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="sum")
 optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train_fold, x_meta_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, x_meta_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, meta_x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch, meta_x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, meta_x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch, meta_x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))
for i,(x_batch, meta_x_batch)in enumerate(test_loader):
y_pred = model(x_batch, meta_x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
predictions = model.predict(test_data )<save_to_csv>
search_result = threshold_search(train_y, train_preds)
 search_result
submissions=pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) ,
"Label": np.argmax(predictions, axis=1)})
 submissions.to_csv("my_submissions.csv", index=False, header=True )<load_from_csv>
sub = pd.read_csv('.. /input/quora-insincere-questions-classification/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub['prediction'] = sub['prediction'].apply(lambda x : int(x))
sub.to_csv("submission.csv", index=False )