Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<split>
import re
import time
import gc
import random
import os
import math
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, precision_recall_fscore_support
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import torch
import torch.nn as nn
import torch.utils.data
import torch.nn.functional as F
train_imgs, val_imgs = train_test_split(train_df['Image'].values,
test_size=0.2,
stratify=train_df['Class'].map(lambda x: str(sorted(list(x)))) ,
random_state=10 )<define_variables>
def seed_torch(seed=21330):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = False
SEED = 21330
seed_torch(SEED )
class DataGenenerator(Sequence):
def __init__(self, images_list=None, folder_imgs=train_imgs_folder,
batch_size=8, shuffle=True, augmentation=None,
resized_height=224, resized_width=224, num_channels=3):
self.batch_size = batch_size
self.shuffle = shuffle
self.augmentation = augmentation
if images_list is None:
self.images_list = os.listdir(folder_imgs)
 else:
self.images_list = deepcopy(images_list)
 self.folder_imgs = folder_imgs
self.len = len(self.images_list)// self.batch_size
self.resized_height = resized_height
self.resized_width = resized_width
self.num_channels = num_channels
self.num_classes = 4
self.is_test = not 'train' in folder_imgs
if not shuffle and not self.is_test:
self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]
def __len__(self):
return self.len
def on_epoch_start(self):
if self.shuffle:
random.shuffle(self.images_list)
 def __getitem__(self, idx):
current_batch = self.images_list[idx * self.batch_size:(idx + 1)* self.batch_size]
X = np.empty(( self.batch_size, self.resized_height, self.resized_width, self.num_channels))
y = np.empty(( self.batch_size, self.num_classes))
for i, image_name in enumerate(current_batch):
path = os.path.join(self.folder_imgs, image_name)
 img = cv2.resize(cv2.imread(path),(self.resized_height, self.resized_width)).astype(np.float32)
 if not self.augmentation is None:
augmented = self.augmentation(image=img)
 img = augmented['image']
X[i, :, :, :] = img/255.0
if not self.is_test:
y[i, :] = img_2_ohe_vector[image_name]
return X, y
def get_labels(self):
if self.shuffle:
images_current = self.images_list[:self.len*self.batch_size]
labels = [img_2_ohe_vector[img] for img in images_current]
else:
labels = self.labels
return np.array(labels )<choose_model_class>
embed_size = 300
max_features = 196534
maxlen = 72
log_interval = 250
batch_size = 512
data_generator_train = DataGenenerator(train_imgs, augmentation=albumentations_train)
 data_generator_train_eval = DataGenenerator(train_imgs, shuffle=False)
 data_generator_val = DataGenenerator(val_imgs, shuffle=False )<load_pretrained>
num_partitions = 20
num_cores = psutil.cpu_count()
print('number of cores:', num_cores)
 def df_parallelize_run(df, func):
df_split = np.array_split(df, num_partitions)
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df
class PrAucCallback(Callback):
def __init__(self, data_generator, num_workers=num_cores,
early_stopping_patience=5,
plateau_patience=3, reduction_rate=0.5,
stage='train', checkpoints_path='checkpoints/'):
super(Callback, self ).__init__()
self.data_generator = data_generator
self.num_workers = num_workers
self.class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
self.history = [[] for _ in range(len(self.class_names)+ 1)]
self.early_stopping_patience = early_stopping_patience
self.plateau_patience = plateau_patience
self.reduction_rate = reduction_rate
self.stage = stage
self.best_pr_auc = -float('inf')
 if not os.path.exists(checkpoints_path):
os.makedirs(checkpoints_path)
 self.checkpoints_path = checkpoints_path
def compute_pr_auc(self, y_true, y_pred):
pr_auc_mean = 0
print(f"
{'
")
 for class_i in range(len(self.class_names)) :
precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 pr_auc = auc(recall, precision)
 pr_auc_mean += pr_auc/len(self.class_names)
 print(f"PR AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}
")
 self.history[class_i].append(pr_auc)
 print(f"
{'
PR AUC mean, {self.stage}: {pr_auc_mean:.3f}
{'
")
 self.history[-1].append(pr_auc_mean)
 return pr_auc_mean
def is_patience_lost(self, patience):
if len(self.history[-1])> patience:
best_performance = max(self.history[-1][-(patience + 1):-1])
 return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]
def early_stopping_check(self, pr_auc_mean):
if self.is_patience_lost(self.early_stopping_patience):
self.model.stop_training = True
def model_checkpoint(self, pr_auc_mean, epoch):
if pr_auc_mean > self.best_pr_auc:
for checkpoint in glob.glob(os.path.join(self.checkpoints_path, 'classifier_densenet169_epoch_*')) :
os.remove(checkpoint)
 self.best_pr_auc = pr_auc_mean
self.model.save(os.path.join(self.checkpoints_path, f'classifier_densenet169_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))
print(f"
{'
Saved new checkpoint
{'
")
 def reduce_lr_on_plateau(self):
if self.is_patience_lost(self.plateau_patience):
new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate
keras.backend.set_value(self.model.optimizer.lr, new_lr)
 print(f"
{'
Reduced learning rate to {new_lr}.
 {'
")
 def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict_generator(self.data_generator, workers=self.num_workers)
 y_true = self.data_generator.get_labels()
pr_auc_mean = self.compute_pr_auc(y_true, y_pred)
 if self.stage == 'val':
self.early_stopping_check(pr_auc_mean)
 self.model_checkpoint(pr_auc_mean, epoch)
 self.reduce_lr_on_plateau()
def get_pr_auc_history(self):
return self.history<choose_model_class>
puncts = ['!', '?', '.', ',', '-', "'", '"', ':', ')', '(', '|', ';', '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', '÷', 'π',
'है', 'है',
]
puncts_str = ''.join(puncts)[5:]
puncts = ['!', '?', '.', ',', '-']
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def clean_quotation(x):
x = str(x)
 if "'s" in x:
x = x.replace("'s", "")
 if "s'" in x:
x = x.replace("s'", "")
 if "n't" in x:
x = x.replace("n't", " not")
 if "'ve" in x:
x = x.replace("'ve", " have")
 return x
def get_model() :
K.clear_session()
seresnext101, preprocess_input = Classifiers.get('seresnext101')
 base_model = seresnext101(weights='imagenet', include_top=False, input_shape=(224, 224, 3))
x = base_model.output
x = keras.layers.GlobalAveragePooling2D()(x)
 x = Dense(512 )(x)
 x = BatchNormalization()(x)
 x = LeakyReLU()(x)
 y_pred = Dense(4, activation='sigmoid' )(x)
 return Model(inputs=base_model.input, outputs=y_pred )<choose_model_class>
mispell_dict = {
"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have",
"couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not",
"hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did",
"how'd'y": "how do you", "how'll": "how will", "how's": "how is", "i'd": "i would", "i'd've": "i would have",
"i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would",
"it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us",
"ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have",
"must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not",
"needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have",
"shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would",
"she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is",
"should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have",
"so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is",
"there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is",
"they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have",
"they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would",
"we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have",
"weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are",
"what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did",
"where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is",
"who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have",
"y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are",
"y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will",
"you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center',
'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater',
'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',
'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist',
'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many',
'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation',
'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum',
'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota',
'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp',
'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization',
'quoran': 'quora', 'quorans': 'quora', 'brexit': 'british exit', 'quoras':'quora', 'fortnite': 'video game',
'pubg': 'video game', 'redmi': 'cell phone', 'jinping': 'chinese president', 'lyft': 'uber', 'θ': 'theta', 'σ': 'sigma'
}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )
model = get_model()<train_model>
def preprocess(text):

 text = text.lower()
text = clean_text(text)
 text = replace_typical_misspell(text)
 text = clean_quotation(text)
 return text
def text_clean_wrapper(df):
df["question_text"] = df["question_text"].apply(preprocess)
 return df
for base_layer in model.layers[:-1]:
base_layer.trainable = False
lr=1e-3
adam = Adam(lr=1e-3)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=["accuracy"])
 history_0 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=1,
workers=num_cores, use_multiprocessing=True,
verbose=1
)<train_model>
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df = df_parallelize_run(train_df, text_clean_wrapper)
 test_df = df_parallelize_run(test_df, text_clean_wrapper)
 train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features, filters=puncts_str)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index
for base_layer in model.layers[:-1]:
base_layer.trainable = True
lr=lr/3.0
K.set_value(adam.lr, lr)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=["accuracy"])
 print("training...")
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=20, use_multiprocessing=True,
workers=num_cores,
verbose=1 
)
 model.save("model.h5" )<train_model>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
emb_mean, emb_std = -0.005838499, 0.48782197
embedding_matrix = np.random.normal(emb_mean, emb_std,(len(word_index), embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8")as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= len(word_index):
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
emb_mean, emb_std = -0.0053247833, 0.49346462
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8", errors='ignore')as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= max_features:
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
emb_mean, emb_std = -0.0033469985, 0.109855495
embedding_matrix = np.random.normal(emb_mean, emb_std,(len(word_index), embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8")as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= len(word_index):
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
lr=lr/3.0
K.set_value(adam.lr, lr)
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=15, use_multiprocessing=True,
workers=num_cores,
verbose=1 
)
 model.save("model.h5" )<train_model>
tqdm.pandas()
start_time = time.time()
train_X, test_X, train_y, word_index = load_and_prec()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_para(word_index)
 embedding_matrix_3 = load_fasttext(word_index)
 total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))
print(np.shape(embedding_matrix_1))
lr=lr/3.0
K.set_value(adam.lr, lr)
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=10, use_multiprocessing=True,
workers=num_cores,
verbose=1 
)
 model.save("model.h5" )<train_model>
class RNNModel(nn.Module):
def __init__(self, rnn_type, input_size, hidden_size):
super(RNNModel, self ).__init__()
self.rnn = getattr(nn, rnn_type )(input_size, hidden_size, bidirectional=True, batch_first=True)
 def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
return self.rnn(x)
 class LSTM_TextCNN(nn.Module):
def __init__(self, hidden_size, embedding_matrix, embed_size, embedding_dropout=0.5, initialization=True):
super(LSTM_TextCNN, self ).__init__()
self.embedding = nn.Embedding(len(word_index), embed_size, padding_idx=0)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(embedding_dropout)
 self.lstm = RNNModel("LSTM", embed_size, hidden_size)
 self.lstm.init_weights()
self.conv1 = nn.Conv1d(hidden_size*2, hidden_size, 1, bias=False)
 self.bn1 = nn.BatchNorm1d(hidden_size)
 self.conv2 = nn.Conv1d(hidden_size*2, hidden_size, 2, bias=False)
 self.bn2 = nn.BatchNorm1d(hidden_size)
 self.conv3 = nn.Conv1d(hidden_size*2, hidden_size, 3, bias=False)
 self.bn3 = nn.BatchNorm1d(hidden_size)
 self.conv4 = nn.Conv1d(hidden_size*2, hidden_size, 4, bias=False)
 self.bn4 = nn.BatchNorm1d(hidden_size)
 self.relu = nn.ReLU(inplace=True)
 self.merge = nn.Linear(hidden_size*4, hidden_size)
 self.dropout = nn.Dropout(0.1)
 self.bn_merge = nn.BatchNorm1d(hidden_size)
 self.out = nn.Linear(hidden_size, 1)
 if initialization:
self.init_weights()
def init_weights(self):
self.out.bias.data.fill_(0)
 nn.init.kaiming_normal_(self.out.weight)
 def forward(self, x):
embed = self.embedding(x)
 out = torch.squeeze(
 self.embedding_dropout(torch.unsqueeze(embed, 0)))
 out, _ = self.lstm(embed)
 out = out.transpose(1, 2)
 out1 = self.bn1(self.relu(self.conv1(out)))
 out1 = F.adaptive_max_pool1d(out1, 1)
 out2 = self.bn2(self.relu(self.conv2(out)))
 out2 = F.adaptive_max_pool1d(out2, 1)
 out3 = self.bn3(self.relu(self.conv3(out)))
 out3 = F.adaptive_max_pool1d(out3, 1)
 out4 = self.bn4(self.relu(self.conv4(out)))
 out4 = F.adaptive_max_pool1d(out4, 1)
 conc = torch.squeeze(torch.cat(( out1, out2, out3, out4), 1))
conc = self.relu(self.merge(conc))
conc = self.bn_merge(self.dropout(conc))
out = self.out(conc)
 return out
class LSTM_GRU(nn.Module):
def __init__(self, hidden_size, embedding_matrix, embed_size, embedding_dropout=0.1):
super(LSTM_GRU, self ).__init__()
self.hidden_size = hidden_size
self.embedding_matrix = embedding_matrix
self.embedding = nn.Embedding(len(word_index), embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(embedding_dropout)
 self.lstm = RNNModel("LSTM", embed_size, hidden_size)
 self.lstm.init_weights()
self.gru = RNNModel("GRU", hidden_size*2, hidden_size)
 self.gru.init_weights()
self.linear = nn.Linear(4*hidden_size, 16)
 self.relu = nn.ReLU(inplace=True)
 self.dropout = nn.Dropout(0.0)
 self.bn = nn.BatchNorm1d(16)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.bn(self.dropout(conc))
out = self.out(conc)
 return out
lr=lr/3.0
K.set_value(adam.lr, lr)
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=10, use_multiprocessing=True,
workers=num_cores,
verbose=1 
)
 model.save("model.h5" )<find_best_params>
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED ).split(train_X, train_y))
class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
def get_threshold_for_recall(y_true, y_pred, class_i, recall_threshold=0.95, precision_threshold=0.94, plot=False):
precision, recall, thresholds = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 i = len(thresholds)- 1
best_recall_threshold = None
while best_recall_threshold is None:
next_threshold = thresholds[i]
next_recall = recall[i]
if next_recall >= recall_threshold:
best_recall_threshold = next_threshold
i -= 1
best_precision_threshold = [thres for prec, thres in zip(precision, thresholds)if prec >= precision_threshold][0]
if plot:
plt.figure(figsize=(10, 7))
plt.step(recall, precision, color='r', alpha=0.3, where='post')
 plt.fill_between(recall, precision, alpha=0.3, color='r')
 plt.axhline(y=precision[i + 1])
 recall_for_prec_thres = [rec for rec, thres in zip(recall, thresholds)
 if thres == best_precision_threshold][0]
plt.axvline(x=recall_for_prec_thres, color='g')
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.ylim([0.0, 1.05])
 plt.xlim([0.0, 1.0])
 plt.legend(['PR curve',
f'Precision {precision[i + 1]:.2f} corresponding to selected recall threshold',
f'Recall {recall_for_prec_thres:.2f} corresponding to selected precision threshold'])
 plt.title(f'Precision-Recall curve for Class {class_names[class_i]}')
 return best_recall_threshold, best_precision_threshold
y_pred = model.predict_generator(data_generator_val, workers=num_cores, verbose=1)
 y_true = data_generator_val.get_labels()
recall_thresholds = dict()
precision_thresholds = dict()
for i, class_name in tqdm(enumerate(class_names)) :
recall_thresholds[class_name], precision_thresholds[class_name] = get_threshold_for_recall(y_true, y_pred, i, plot=True )<predict_on_test>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
data_generator_test = DataGenenerator(folder_imgs=test_imgs_folder, shuffle=False)
 y_pred_test = model.predict_generator(data_generator_test, workers=num_cores, verbose=1 )<define_variables>
train_ratio = 80810 / 1306122
public_test_ratio = 3376 / 56370
submit_ratio =(80810 + 3376)/(1306122 + 56370)
 def threshold_search_oof(y_proba, y_true=None, mode="train"):
if mode == "train":
ratio = train_ratio
elif mode == "test":
ratio = public_test_ratio
elif mode == "submit":
ratio = submit_ratio
else:
raise ValueError("Invalid mode")
 best_threshold = 0
best_score = np.Inf
for threshold in tqdm([i * 0.001 for i in range(1000)]):
score = np.abs(( y_proba > threshold ).mean() - ratio)
 if score < best_score:
best_threshold = threshold
best_score = score
if mode == "train":
f1 = f1_score(y_true=y_true, y_pred=y_proba > best_threshold)
 search_result = {'threshold': best_threshold, 'dist': best_score, 'f1': f1}
return search_result
else:
search_result = {'threshold': best_threshold, 'dist': best_score}
return search_result
def threshold_search(y_proba, y_true):
best_threshold = 0
best_score = 0
scores = []
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 scores.append(score)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
plt.figure(figsize=(12,9))
plt.plot([i * 0.01 for i in range(100)], scores)
 plt.plot(best_threshold, best_score, "xr", label="Best threshold")
 plt.show()
return search_result
image_labels_empty = set()
for i,(img, predictions)in enumerate(zip(os.listdir(test_imgs_folder), y_pred_test)) :
for class_i, class_name in enumerate(class_names):
if predictions[class_i] < recall_thresholds[class_name]:
image_labels_empty.add(f'{img}_{class_name}' )<load_from_csv>
class CosineLRWithRestarts() :

 def __init__(self, optimizer, batch_size, epoch_size, restart_period=100,
t_mult=2, last_epoch=-1, eta_threshold=1000, verbose=False):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if last_epoch == -1:
for group in optimizer.param_groups:
group.setdefault('initial_lr', group['lr'])
 else:
for i, group in enumerate(optimizer.param_groups):
if 'initial_lr' not in group:
raise KeyError("param 'initial_lr' is not specified "
"in param_groups[{}] when resuming an"
" optimizer".format(i))
self.base_lrs = list(map(lambda group: group['initial_lr'],
optimizer.param_groups))
self.last_epoch = last_epoch
self.batch_size = batch_size
self.epoch_size = epoch_size
self.eta_threshold = eta_threshold
self.t_mult = t_mult
self.verbose = verbose
self.base_weight_decays = list(map(lambda group: group['weight_decay'],
optimizer.param_groups))
self.restart_period = restart_period
self.restarts = 0
self.t_epoch = -1
def _schedule_eta(self):

 eta_min = 0
eta_max = 1
if self.restarts <= self.eta_threshold:
return eta_min, eta_max
else:
d = self.restarts - self.eta_threshold
k = d * 0.09
return(eta_min + k, eta_max - k)
 def get_lr(self, t_cur):
eta_min, eta_max = self._schedule_eta()
eta_t =(eta_min + 0.5 *(eta_max - eta_min)
 *(1.+ math.cos(math.pi * 
(t_cur / self.restart_period))))
weight_decay_norm_multi = math.sqrt(self.batch_size / 
(self.epoch_size *
self.restart_period))
lrs = [base_lr * eta_t for base_lr in self.base_lrs]
weight_decays = [base_weight_decay * eta_t * weight_decay_norm_multi
for base_weight_decay in self.base_weight_decays]
if self.t_epoch % self.restart_period < self.t_epoch:
if self.verbose:
print("Restart at epoch {}".format(self.last_epoch))
self.restart_period *= self.t_mult
self.restarts += 1
self.t_epoch = 0
return zip(lrs, weight_decays)
 def _set_batch_size(self):
d, r = divmod(self.epoch_size, self.batch_size)
 batches_in_epoch = d + 2 if r > 0 else d + 1
self.batch_increment = iter(torch.linspace(0, 1, batches_in_epoch))
def step(self):
self.last_epoch += 1
self.t_epoch += 1
self._set_batch_size()
self.batch_step()
def batch_step(self):
t_cur = self.t_epoch + next(self.batch_increment)
 for param_group,(lr, weight_decay)in zip(self.optimizer.param_groups,
self.get_lr(t_cur)) :
param_group['lr'] = lr
param_group['weight_decay'] = weight_decay
submission = pd.read_csv('.. /input/efficient-net-b4-unet-clouds/submission.csv')
 submission.head()<filter>
def f1_loss(logits, labels):
__small_value = 1e-6
beta = 1
batch_size = logits.size() [0]
p = torch.sigmoid(logits)
 l = labels
num_pos = torch.sum(p, 1)+ __small_value
num_pos_hat = torch.sum(l, 1)+ __small_value
tp = torch.sum(l * p, 1)
 precise = tp / num_pos
recall = tp / num_pos_hat
fs =(1 + beta * beta)* precise * recall /(beta * beta * precise + recall + __small_value)
 loss = fs.sum() / batch_size
return(1 - loss)
 class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, logit, target):
max_val =(-logit ).clamp(min=0)
 loss = logit - logit * target + max_val +(( -max_val ).exp() +(-logit - max_val ).exp() ).log()
invprobs = F.logsigmoid(-logit *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
if len(loss.size())==2:
loss = loss.sum(dim=1)
 return loss.mean()
def lovasz_grad(gt_sorted):

 p = len(gt_sorted)
 gts = gt_sorted.sum()
intersection = gts - gt_sorted.float().cumsum(0)
 union = gts +(1 - gt_sorted ).float().cumsum(0)
 jaccard = 1.- intersection / union
if p > 1:
jaccard[1:p] = jaccard[1:p] - jaccard[0:-1]
return jaccard
def lovasz_hinge_flat(logits, labels):

 logits = logits.squeeze()
labels = labels.squeeze()
if len(labels)== 0:
return logits.sum() * 0.
 signs = 2.* labels.float() - 1.
 errors =(1.- logits * Variable(signs))
errors_sorted, perm = torch.sort(errors, dim=0, descending=True)
 perm = perm.data
gt_sorted = labels[perm]
grad = lovasz_grad(gt_sorted)
 loss = torch.dot(F.elu(errors_sorted)+ 1, Variable(grad))
return loss
predictions_nonempty = set(submission.loc[~submission['EncodedPixels'].isnull() , 'Image_Label'].values )<concatenate>
train_epochs = 4
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )
print(f'{len(image_labels_empty.intersection(predictions_nonempty)) } masks would be removed' )<save_to_csv>
for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model1 = LSTM_TextCNN(64, np.concatenate(( embedding_matrix_1, embedding_matrix_3), axis=1), embed_size*2, embedding_dropout=0.5)
 model2 = LSTM_GRU(64, np.mean([embedding_matrix_1, embedding_matrix_2], axis=0), embed_size, embedding_dropout=0.1)
 model1.cuda()
model2.cuda()
loss_fn1 = torch.nn.BCEWithLogitsLoss()
loss_fn2 = f1_loss
optimizer1 = torch.optim.Adam(model1.parameters() , lr=0.0035)
 scheduler1 = CosineLRWithRestarts(optimizer1, batch_size, len(x_train_fold), restart_period=4, t_mult=1, verbose=True)
 optimizer2 = torch.optim.Adam(model2.parameters() , lr=0.0035)
 scheduler2 = CosineLRWithRestarts(optimizer2, batch_size, len(x_train_fold), restart_period=4, t_mult=1, verbose=True)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(train_epochs):
start_time = time.time()
scheduler1.step()
scheduler2.step()
model1.train()
model2.train()
for batch_idx,(x_batch, y_batch)in enumerate(train_loader):
y_pred1 = model1(x_batch)
 y_pred2 = model2(x_batch)
 loss1 = loss_fn1(y_pred1, y_batch)+ loss_fn2(y_pred1, y_batch)
 loss2 = loss_fn1(y_pred2, y_batch)+ loss_fn2(y_pred2, y_batch)
 optimizer1.zero_grad()
optimizer2.zero_grad()
loss1.backward()
loss2.backward()
optimizer1.step()
optimizer2.step()
scheduler1.batch_step()
scheduler2.batch_step()
model1.eval()
model2.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred1 = model1(x_batch ).detach()
y_pred2 = model2(x_batch ).detach()
valid_preds_fold[i * batch_size:(i+1)* batch_size] =(
 sigmoid(y_pred1.cpu().numpy())[:, 0] +
sigmoid(y_pred2.cpu().numpy())[:, 0] 
)/ 2
search_result = threshold_search(valid_preds_fold, train_y[valid_idx])
 valid_yhat_fold =(valid_preds_fold > search_result['threshold'] ).astype(int)
 valid_precision, valid_recall, valid_fscore, _ = precision_recall_fscore_support(train_y[valid_idx], valid_yhat_fold, average='binary')
 elapsed_time = time.time() - start_time
print('Epoch {}/{} | val_precision={:.5f} | val_recall={:.5f} | val_f1={:.5f} | threshold={:.2f} | time={:.2f}s'.format(
 epoch, train_epochs - 1, valid_precision, valid_recall, valid_fscore, search_result['threshold'], elapsed_time))
for i,(x_batch,)in enumerate(test_loader):
y_pred1 = model1(x_batch ).detach()
y_pred2 = model2(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] =(
 sigmoid(y_pred1.cpu().numpy())[:, 0] +
sigmoid(y_pred2.cpu().numpy())[:, 0] 
)/ 2
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )
submission.loc[submission['Image_Label'].isin(image_labels_empty), 'EncodedPixels'] = np.nan
submission.to_csv('submission_segmentation_and_classifier.csv', index=None )<import_modules>
search_result = threshold_search(train_preds, train_y)
 train_yhat =(train_preds > search_result['threshold'] ).astype(int)
 train_precision, train_recall, train_fscore, _ = precision_recall_fscore_support(train_y, train_yhat, average='binary')
 print('oof_precision={:.5f} | oof_recall={:.5f} | oof_f1={:.5f} | threshold={:.2f}'.format(train_precision, train_recall, train_fscore, search_result['threshold']))
<set_options><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False )