Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
modelOcSVM = OneClassSVM(nu=0.01, gamma=0.001)
 modelOcSVM.fit(donneesTrain.drop(columns='SalePrice'))
y_pred = modelOcSVM.predict(donneesTrain.drop(columns='SalePrice'))
print('ndividuals considered outlier :',sum(y_pred==-1))<predict_on_test>
DEBUG = False
batch_size = 256
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 2 * LSTM_UNITS
n_seeds = 1
n_splits = 10
n_epochs = 15
EMBED_SIZE = 300
SUBGROUP_NEGATIVE_WEIGHT_COEF = 1
BACKGROUND_POSITIVE_WEIGHT_COEF = 0
ENSEMBLE_START_EPOCH = 3
MAX_LEN = 300
EMB_DROPOUT = 0.3
MIDDLE_DROPOUT = 0.3
if DEBUG:
DEBUG_DATA_SIZE = 1000
newX = modelOcSVM.predict(X_train)
 resultats,yRegressions,regresseurs = calculRegressions(X_train[newX != -1], X_test, y_train[newX != -1], y_test)
 resultats<drop_column>
%matplotlib inline
modelRFE = initRFE(
 donneesTrain.drop(columns='SalePrice'),
donneesTrain.SalePrice,
regresseurs,
modelRFE
)<prepare_x_and_y>
@contextmanager
def suppress_stdout() :
with open(os.devnull, "w")as devnull:
old_stdout = sys.stdout
sys.stdout = devnull
try:
yield
finally:
sys.stdout = old_stdout
resultats,yRegressions,regresseurs,modelRFE = calculRegressionsRFE(donneesTrain.drop(columns='SalePrice'),
np.log(donneesTrain['SalePrice'].values), regresseurs, modelRFE, taux = 0.3)
 resultats<compute_test_metric>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', '
', '\r']
def clean_text(x: str)-> str:
for punct in puncts:
if punct in x:
x = x.replace(punct, ' {} '.format(punct))
return x
def clean_numbers(x):
return re.sub('\d+', ' ', x)
 if DEBUG:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', nrows=DEBUG_DATA_SIZE)
 else:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_test = test['comment_text'].apply(lambda x: clean_text(x))
gc.collect()
affichageMetricsSansLog(np.exp(yRegressions.Observations), np.exp(yRegressions.LinearRegressionRidge))<predict_on_test>
aux_col_list = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
yRegressions = np.zeros(donneesTest.shape[0])
 for nom in regresseurs:
X = donneesTest
newX = modelOcSVM.predict(X)
 X = X[X.columns[modelRFE[nom].support_]]
yRegressions += regresseurs[nom].predict(X)
 yRegressions /= len(regresseurs)
 yRegressions = np.exp(yRegressions )<save_to_csv>
%%time
with open('.. /input/reducing-oov-of-crawl300d2m-no-appos-result/jigsaw-crawl-300d-2M.joblib', 'rb')as f:
crawl_emb_dict = joblib.load(f )
submission = pd.DataFrame(yRegressions,index = donneesTest.index ).reset_index()
submission.columns = ['Id','SalePrice']
submission.to_csv('submission01.csv',index=False )<set_options>
def build_vocab(texts: pd.DataFrame)-> Dict[str, int]:

 sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in tqdm(sentences):
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_coverage(vocab: Dict[str, int], embeddings_index: Dict)-> List[str]:

 known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in tqdm(vocab.keys()):
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(known_words)/ float(len(vocab))))
print('Found embeddings for {:.2%} of all text'.format(float(nb_known_words)/(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
p_stemmer = PorterStemmer()
l_stemmer = LancasterStemmer()
s_stemmer = SnowballStemmer("english")
 def edits1(word):

 letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def known(words, embed):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in embed)
 def spellcheck(word, word_rank_dict):
return min(known(edits1(word), word_rank_dict), key=lambda w: word_rank_dict[w])
 punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "",
"€": "e", "™": "tm", "√": " sqrt ", "×": "x",
"²": "2", "—": "-", "–": "-", "’": "'",
"_": "-", "`": "'", '“': '"', '”': '"',
'“': '"', "£": "e", '∞': 'infinity',
'θ': 'theta', '÷': '/', 'α': 'alpha',
'•': '.', 'à': 'a', '−': '-', 'β': 'beta',
'∅': '', '³': '3', 'π': 'pi', }
def process_stemmer(vocab, embed):
oov_word_set = set()
for word in tqdm(vocab.keys()):
vector = embed.get(word, None)
 if vector is not None:
continue
vector = embed.get(word.lower() , None)
 if vector is not None:
embed[word] = vector
continue
vector = embed.get(word.upper() , None)
 if vector is not None:
embed[word] = vector
continue
vector = embed.get(word.capitalize() , None)
 if vector is not None:
embed[word] = vector
continue
corr_word = punct_mapping.get(word, None)
 if corr_word is not None:
vector = embed.get(corr_word, None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(p_stemmer.stem(word), None)
 except:
vector = embed.get(p_stemmer.stem(word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(l_stemmer.stem(word), None)
 except:
vector = embed.get(l_stemmer.stem(word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(s_stemmer.stem(word), None)
 except:
vector = embed.get(s_stemmer.stem(word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
oov_word_set.add(word)
 return embed, oov_word_set
def process_small_capital(vocab, embed, oov_set):
oov_word_set = set()
for word in tqdm(vocab.keys()):
if word not in oov_set:
continue
char_list = []
any_small_capitial = False
for char in word:
try:
uni_name = unicodedata.name(char)
 except ValueError:
continue
if 'LATIN LETTER SMALL CAPITAL' in uni_name:
char = uni_name[-1]
any_small_capitial = True
if 'CYRILLIC SMALL LETTER GHE WITH STROKE' in uni_name:
char = 'F'
any_small_capitial = True
char_list.append(char)
 if not any_small_capitial:
oov_word_set.add(word)
 continue
legit_word = ''.join(char_list)
 vector = embed.get(legit_word, None)
 if vector is not None:
embed[word] = vector
continue
vector = embed.get(legit_word.lower() , None)
 if vector is not None:
embed[word] = vector
continue
vector = embed.get(legit_word.upper() , None)
 if vector is not None:
embed[word] = vector
continue
vector = embed.get(legit_word.capitalize() , None)
 if vector is not None:
embed[word] = vector
continue
corr_word = punct_mapping.get(legit_word, None)
 if corr_word is not None:
vector = embed.get(corr_word, None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(p_stemmer.stem(legit_word), None)
 except:
vector = embed.get(p_stemmer.stem(legit_word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(l_stemmer.stem(legit_word), None)
 except:
vector = embed.get(l_stemmer.stem(legit_word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(s_stemmer.stem(legit_word), None)
 except:
vector = embed.get(s_stemmer.stem(legit_word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
oov_word_set.add(word)
 return embed, oov_word_set
def process_spellcheck(vocab, embed, word_rank_dict, oov_set):
oov_word_set = set()
for word in tqdm(vocab.keys()):
if word not in oov_set:
continue
try:
vector = embed.get(spellcheck(word, word_rank_dict), None)
 except:
continue
if vector is not None:
embed[word] = vector
continue
oov_word_set.add(word)
 return embed, oov_word_set
def make_word_rank(embed):
word_rank = {}
for i, word in enumerate(embed):
word_rank[word] = i
return word_rank
def head(enumerable, n=10):
for i, item in enumerate(enumerable):
print(str(i)+ '
', item)
 if i > n:
return
InteractiveShell.ast_node_interactivity = "all"<import_modules>
%%time
vocab = build_vocab(x_test)
 oov = check_coverage(vocab, crawl_emb_dict)
 head(oov )
import numpy as np
import pandas as pd
from datetime import datetime
from scipy.stats import skew
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV, Ridge
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler, StandardScaler
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error as mse
from sklearn.metrics import make_scorer
from sklearn.neighbors import LocalOutlierFactor
from sklearn.linear_model import LinearRegression
from mlxtend.regressor import StackingCVRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
import matplotlib.pyplot as plt
import seaborn as sns
import os<load_from_csv>
%%time
emb, oov_stemmer = process_stemmer(vocab, crawl_emb_dict)
 oov = check_coverage(vocab, emb)
 head(oov )
print(os.listdir(".. /input"))
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 print("Train set size:", train.shape)
 print("Test set size:", test.shape)
 print('start data processing', datetime.now() , )<define_variables>
%%time
emb, oov_small_capital = process_small_capital(vocab, emb, oov_stemmer)
 oov = check_coverage(vocab, emb)
 head(oov )
outs = detect_outliers(train['GrLivArea'], train['SalePrice'],top=5)
 outs<feature_engineering>
%%time
word_rank = make_word_rank(emb)
 emb, oov_spell = process_spellcheck(vocab, emb, word_rank, oov_small_capital)
 oov = check_coverage(vocab, emb)
 head(oov )
outs = detect_outliers(train['LowQualFinSF'], train['SalePrice'],top=5)
 outs<define_variables>
with open('.. /input/googleprofanitywords/list.txt', 'r')as f:
p_words = f.readlines()
set_puncts = set(puncts)
 p_word_set = set([t.replace('
', '')for t in p_words])
 def sentence_fetures(text):
word_list = text.split()
word_count = len(word_list)
 n_upper = len([word for word in word_list if any([c.isupper() for c in word])])
 n_unique = len(set(word_list))
n_ex = word_list.count('!')
 n_que = word_list.count('?')
 n_puncts = len([word for word in word_list if word in set_puncts])
 n_prof = len([word for word in word_list if word in p_word_set])
 n_oov = len([word for word in word_list if word not in crawl_emb_dict])
 return word_count, n_upper, n_unique, n_ex, n_que, n_puncts, n_prof, n_oov
sentence_feature_cols = ['word_count', 'n_upper', 'n_unique', 'n_ex', 'n_que', 'n_puncts', 'n_prof', 'n_oov']
feature_dict = defaultdict(list)
 for text in progress_bar(x_test):
feature_list = sentence_fetures(text)
 for i_feature, feature_name in enumerate(sentence_feature_cols):
feature_dict[sentence_feature_cols[i_feature]].append(feature_list[i_feature])
 sentence_df = pd.DataFrame.from_dict(feature_dict)
 for col in ['n_upper', 'n_unique', 'n_ex', 'n_que', 'n_puncts', 'n_prof', 'n_oov']:
sentence_df[col + '_ratio'] = sentence_df[col] / sentence_df['word_count']
sentence_df.head()
outliers = [30, 88, 462, 523, 632, 1298, 1324]<drop_column>
sentence_feature_mat = sentence_df.drop(columns=['n_oov', 'n_oov_ratio'] ).values
gc.collect()
train = train.drop(train.index[outliers])
 train.shape<prepare_x_and_y>
%%time
x_test = x_test.apply(lambda x: clean_numbers(x))
tokenizer = preprocessing.text.Tokenizer(filters="", lower=False)
 tokenizer.fit_on_texts(list(x_test))
x_test = tokenizer.texts_to_sequences(x_test)
 max_features = len(tokenizer.word_index)+ 1
max_features
def build_matrix(word_index, embedding_dict):
embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_dict[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
emb_mat_crawl, oov = build_matrix(tokenizer.word_index, emb)
 del crawl_emb_dict, oov, emb
gc.collect()
emb_mat_torch = torch.tensor(emb_mat_crawl, dtype=torch.float32 ).cuda()
del emb_mat_crawl
gc.collect()
y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 features.drop(['Id'], axis=1, inplace=True)
 print(features.shape )<data_type_conversions>
class DynamicBucketIterator(object):
def __init__(self, data, label, capacity, pad_token, shuffle, length_quantile, max_batch_size, for_bert):
self.data = data
self.label = label
self.pad_token = pad_token
self.capacity = capacity
self.shuffle = shuffle
self.length_quantile = length_quantile
self.for_bert = for_bert
self.index_sorted = sorted(range(len(self.data)) , key=lambda i: len(self.data[i]))
old_separator_index = 0
self.separator_index_list = [0]
for i_sample in range(len(self.data)) :
sample_index = self.index_sorted[i_sample]
sample = self.data[sample_index]
current_batch_size = i_sample - old_separator_index + 1
if min(len(sample), MAX_LEN)* current_batch_size <= self.capacity and current_batch_size <= max_batch_size:
pass
else:
old_separator_index = i_sample
self.separator_index_list.append(i_sample)
 self.separator_index_list.append(len(self.data))
if not self.shuffle:
self.bucket_index = range(self.__len__())
 self.reset_index()
def reset_index(self):
self.i_batch = 0
if self.shuffle:
self.index_sorted = sorted(np.random.permutation(len(self.data)) , key=lambda i: len(self.data[i]))
self.bucket_index = np.random.permutation(self.__len__())
 def __len__(self):
return len(self.separator_index_list)- 1
def __iter__(self):
return self
def __next__(self):
try:
i_bucket = self.bucket_index[self.i_batch]
except IndexError as e:
self.reset_index()
raise StopIteration
start_index, end_index = self.separator_index_list[i_bucket : i_bucket + 2]
index_batch = self.index_sorted[start_index : end_index]
raw_batch_data = [self.data[i] for i in index_batch]
batch_label = self.label[index_batch]
math.ceil(1)
 max_len = int(math.ceil(np.quantile([len(x)for x in raw_batch_data], self.length_quantile)))
 max_len = min([max_len, MAX_LEN])
 if max_len == 0:
max_len = 1
if self.for_bert:
segment_id_batch = np.zeros(( len(raw_batch_data), max_len))
padded_batch = []
input_mask_batch = []
for sample in raw_batch_data:
input_mask = [1] * len(sample)+ [0] *(max_len - len(sample))
input_mask_batch.append(input_mask[:max_len])
 sample = sample + [self.pad_token for _ in range(max_len - len(sample)) ]
padded_batch.append(sample[:max_len])
 self.i_batch += 1
return padded_batch, segment_id_batch, input_mask_batch, batch_label, index_batch
else:
padded_batch = []
for sample in raw_batch_data:
sample = sample + [self.pad_token for _ in range(max_len - len(sample)) ]
padded_batch.append(sample[:max_len])
 self.i_batch += 1
return padded_batch, batch_label, index_batch
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str)
<filter>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class GRULayer(nn.Module):
def __init__(self, input_size, hidden_size):
super(GRULayer, self ).__init__()
self.gru = nn.GRU(input_size=input_size,
hidden_size=hidden_size,
bias=True,
bidirectional=True,
batch_first=True)
 self.init_weights()
def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
gru_outputs, gru_state = self.gru(x)
 return gru_outputs, gru_state
class NeuralNet(nn.Module):
def __init__(self, embed_size, num_aux_targets, num_sentence_features):
super(NeuralNet, self ).__init__()
self.embedding_dropout = SpatialDropout(EMB_DROPOUT)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.middle_dropout = SpatialDropout(MIDDLE_DROPOUT)
 self.lstm2 = GRULayer(LSTM_UNITS * 2, LSTM_UNITS)
 self.linear_sentence1 = nn.Linear(num_sentence_features, num_sentence_features)
 n_hidden = LSTM_UNITS * 2 + num_sentence_features
self.linear1 = nn.Linear(n_hidden, n_hidden)
 self.linear_out = nn.Linear(n_hidden, 1)
 self.linear_aux_out = nn.Linear(n_hidden, num_aux_targets)
 def forward(self, x, sentence_features):
h_embedding = self.embedding_dropout(x)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm1 = self.middle_dropout(h_lstm1)
 h_lstm2, _ = self.lstm2(h_lstm1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_sentence = self.linear_sentence1(sentence_features)
 h_cat = torch.cat(( max_pool, h_sentence), 1)
 h_conc_linear1 = F.relu(self.linear1(h_cat))
hidden = h_cat + h_conc_linear1
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
numeric_features = features.dtypes[features.dtypes != 'object'].index
numeric_features
len(numeric_features)
 category_features = features.dtypes[features.dtypes == 'object'].index
category_features
len(category_features )<define_variables>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
OOF_TRAIN_COL = 'oof_train'
SUBGROUP_AUC_COL = 'subgroup_auc'
BPSN_AUC_COL = 'bpsn_auc'
BNSP_AUC_COL = 'bnsp_auc'
def compute_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except ValueError:
return np.nan
'model'
def compute_subgroup_auc(df, subgroup_col, label_col, oof_col):
subgroup_examples = df[df[subgroup_col]]
return compute_auc(subgroup_examples[label_col], subgroup_examples[oof_col])
 def compute_bpsn_auc(df, subgroup_col, label_col, oof_col):

 subgroup_negative_examples = df[df[subgroup_col] & ~df[label_col]]
non_subgroup_positive_examples = df[~df[subgroup_col] & df[label_col]]
examples = subgroup_negative_examples.append(non_subgroup_positive_examples)
 return compute_auc(examples[label_col], examples[oof_col])
 def compute_bnsp_auc(df, subgroup_col, label_col, oof_col):

 subgroup_positive_examples = df[df[subgroup_col] & df[label_col]]
non_subgroup_negative_examples = df[~df[subgroup_col] & ~df[label_col]]
examples = subgroup_positive_examples.append(non_subgroup_negative_examples)
 return compute_auc(examples[label_col], examples[oof_col])
 def compute_bias_metrics_for_model(df,
subgroup_list,
oof_col,
label_col,
include_asegs=False):

 record_list = []
for subgroup in subgroup_list:
record = {
'subgroup': subgroup,
'subgroup_size': len(df[df[subgroup]])
 }
record[SUBGROUP_AUC_COL] = compute_subgroup_auc(df, subgroup, label_col, oof_col)
 record[BPSN_AUC_COL] = compute_bpsn_auc(df, subgroup, label_col, oof_col)
 record[BNSP_AUC_COL] = compute_bnsp_auc(df, subgroup, label_col, oof_col)
 record_list.append(record)
 return pd.DataFrame(record_list ).sort_values('subgroup_auc', ascending=True)
 TOXICITY_COLUMN = 'target'
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
def convert_to_bool(df, col_name):
df[col_name] = np.where(df[col_name] >= 0.5, True, False)
 def convert_dataframe_to_bool(df):
bool_df = df.copy()
for col in ['target'] + identity_columns:
convert_to_bool(bool_df, col)
 return bool_df
def calculate_overall_auc(df, model_name):
true_labels = df[TOXICITY_COLUMN]
predicted_labels = df[model_name]
return metrics.roc_auc_score(true_labels, predicted_labels)
 def power_mean(series, p):
total = sum(np.power(series, p))
return np.power(total / len(series), 1 / p)
 def get_final_metric(bias_df, overall_auc, POWER=-5, OVERALL_MODEL_WEIGHT=0.25):
bias_score = np.average([
power_mean(bias_df[SUBGROUP_AUC_COL], POWER),
power_mean(bias_df[BPSN_AUC_COL], POWER),
power_mean(bias_df[BNSP_AUC_COL], POWER)
 ])
 return(OVERALL_MODEL_WEIGHT * overall_auc)+(( 1 - OVERALL_MODEL_WEIGHT)* bias_score)
 def get_various_auc(valid_df, y_pred):
valid_df = convert_dataframe_to_bool(valid_df.fillna(0))
valid_df.loc[:, OOF_TRAIN_COL] = y_pred
valid_df = convert_dataframe_to_bool(valid_df.fillna(0))
bias_metrics_df = compute_bias_metrics_for_model(valid_df, identity_columns, OOF_TRAIN_COL, TOXICITY_COLUMN)
 overall_auc = calculate_overall_auc(valid_df, OOF_TRAIN_COL)
 return get_final_metric(bias_metrics_df, overall_auc), overall_auc, bias_metrics_df
special_features = [
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond',
'PoolQC', 'Fence'
]
len(special_features )<data_type_conversions>
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features[numeric_features] = features[numeric_features].apply(
 lambda x: x.fillna(0))
features[category_features] = features[category_features].apply(
 lambda x: x.fillna('None'))<sort_values>
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 auc_array = np.zeros(( n_seeds, n_splits, n_epochs))
test_array = np.zeros(( n_seeds, n_splits, n_epochs, len(x_test)))
 kf = KFold(n_splits=n_splits, shuffle=True)
 for i_seed in range(n_seeds):
print(f'start seed {i_seed}')
 for i_fold,(dev_index, val_index)in enumerate(kf.split(x_test)) :
print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{i_seed}-fold{i_fold}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))),
test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False,
length_quantile=1, max_batch_size=10000000, for_bert=False)
 print(torch.cuda.memory_allocated())
 for i_epoch in range(ENSEMBLE_START_EPOCH, n_epochs):
start_time = time.time()
model = NeuralNet(EMBED_SIZE, len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{i_seed}-fold{i_fold}-epoch{i_epoch}.torchModelState')
 model.load_state_dict(torch.load(load_path))
model.cuda()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
mem_list = []
for batch in test_loader:
x_batch = batch[0]
y_batch = batch[1]
index_batch = batch[2]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
if len(x_batch)>= 32:
y_pred = model(emb_mat_torch[x_batch], sentence_feature_batch)
 else:
len_last = len(x_batch)
 x_batch.extend([[0 for _ in range(len(x_batch[0])) ] for _ in range(32)])
 y_pred = model(emb_mat_torch[x_batch[:32]][:len_last], sentence_feature_batch)
 mem_list.append(torch.cuda.memory_allocated())
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del y_pred
torch.cuda.empty_cache()
test_array[i_seed, i_fold, i_epoch] = epoch_test_pred
elapsed_time = time.time() - start_time
del model
gc.collect()
torch.cuda.empty_cache()
print(f'epoch {i_epoch}: {elapsed_time: 0.3f}, {max(mem_list)/10**9}')
 del epoch_test_pred
gc.collect()
torch.cuda.empty_cache()
skew_features = features[numeric_features].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.15]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))<drop_column>
del test_loader, emb_mat_torch, test_array
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
<categorify>
i_seed = 0
SENTENCE_FEAURE_USED = ['word_count', 'n_unique', 'n_unique_ratio']
batch_size = 32
OUT_DROPOUT = 0.3
BERT_HIDDEN_SIZE = 768
MAX_LEN = 220
print("before get_dummies:",features.shape)
 final_features = pd.get_dummies(features ).reset_index(drop=True)
 print("after get_dummies:",final_features.shape)
 X = final_features.iloc[:len(y), :]	
X_sub = final_features.iloc[len(y):, :]
print("after get_dummies, the dataset size:",'X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape )<train_model>
tqdm.pandas()
%matplotlib inline
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X = np.array(X.drop(overfit, axis=1 ).copy())
 y = np.array(y)
 X_sub = np.array(X_sub.drop(overfit, axis=1 ).copy())
 print('X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape)
 print('feature engineering finished!', datetime.now() )<compute_train_metric>
sentence_feature_mat = sentence_df[SENTENCE_FEAURE_USED].values
gc.collect()
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mse(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )<define_search_space>
%%time
if DEBUG:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', nrows=DEBUG_DATA_SIZE)
 else:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_test = test['comment_text']
del test
gc.collect()
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]<choose_model_class>
logging.basicConfig(level=logging.INFO )
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(
 RobustScaler() ,
LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds))
elasticnet = make_pipeline(
 RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(
 C=20,
epsilon=0.008,
gamma=0.0003, 
))
gbr = GradientBoostingRegressor(n_estimators=3000,
learning_rate=0.05,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 lightgbm = LGBMRegressor(
 objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 xgboost = XGBRegressor(learning_rate=0.01,
n_estimators=3460,
max_depth=3,
min_child_weight=0,
gamma=0,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006 )<choose_model_class>
tokenizer = GPT2Tokenizer(vocab_file='.. /input/gpt2-models/vocab.json', merges_file='.. /input/gpt2-models/merges.txt' )
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True )<compute_test_metric>
%%time
x_test = x_test.progress_apply(lambda x: tokenizer.encode(x[:MAX_LEN]))
print('TEST score on CV')
 score = cv_rmse(ridge)
 print("Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("Lightgbm score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(gbr)
 print("GradientBoosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )<train_model>
class NeuralNet(nn.Module):
def __init__(self, num_aux_targets, num_sentence_features):
super(NeuralNet, self ).__init__()
self.gpt2 = GPT2Model.from_pretrained('.. /input/gpt2-models/')
 self.dropout = nn.Dropout(OUT_DROPOUT)
 self.linear_sentence1 = nn.Linear(num_sentence_features, num_sentence_features)
 n_hidden = BERT_HIDDEN_SIZE + num_sentence_features
self.linear1 = nn.Linear(n_hidden, n_hidden)
 self.linear_out = nn.Linear(n_hidden, 1)
 self.linear_aux_out = nn.Linear(n_hidden, num_aux_targets)
 def forward(self, x_features, sentence_features):
h_gpt2, _ = self.gpt2(x_features)
 h_gpt2, _ = torch.max(h_gpt2, 1)
 h_sentence = self.linear_sentence1(sentence_features)
 h_cat = torch.cat(( h_gpt2, h_sentence), 1)
 h_conc_linear1 = F.relu(self.linear1(h_cat))
hidden = h_cat + h_conc_linear1
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
print('START Fit')
 print(datetime.now() , 'StackingCVRegressor')
 stack_gen_model = stack_gen.fit(X, y )<save_to_csv>
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 aux_col_list = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
test_dict = {}
fold_list = [1]
i_epoch = 1
start_time = time.time()
for i_fold in fold_list:
fold_start_time = time.time()
print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{0}-fold{i_fold}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))), test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False, length_quantile=1, max_batch_size=2048, for_bert=False)
 print(torch.cuda.memory_allocated())
 model = NeuralNet(len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{0}-fold{i_fold}-epoch{i_epoch}.torchModelState')
 model.load_state_dict(torch.load(load_path)['model_state_dict'])
 model.cuda()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
eval_start_time = time.time()
for batch in progress_bar(test_loader):
x_batch = torch.tensor(batch[0], dtype=torch.long ).cuda()
y_batch = batch[1]
index_batch = batch[2]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
y_pred = model(x_batch, sentence_feature_batch)
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del x_batch, y_pred
torch.cuda.empty_cache()
print(f'uncased {i_fold} finishd in {time.time() - fold_start_time}', file=open('time.txt', 'a'))
test_dict[i_fold] = epoch_test_pred
del model, epoch_test_pred
gc.collect()
torch.cuda.empty_cache()
print(f'uncased finishd in {time.time() - start_time}', file=open('time.txt', 'a'))
print('Predict submission', datetime.now() ,)
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(stack_gen_model.predict(X_sub)))
 submission.head()
submission.to_csv("submission_stacking.csv", index=False )<train_model>
del test_loader, test_dict
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
print(datetime.now() , 'ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print(datetime.now() , 'lasso')
 lasso_model_full_data = lasso.fit(X, y)
 print(datetime.now() , 'elasticnet')
 elastic_model_full_data = elasticnet.fit(X, y)
 print(datetime.now() , 'GradientBoosting')
 gbr_model_full_data = gbr.fit(X, y)
 print(datetime.now() , 'xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print(datetime.now() , 'lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y)
 print(datetime.now() , 'svr')
 svr_model_full_data = svr.fit(X, y )<define_variables>
i_seed = 0
SENTENCE_FEAURE_USED = ['word_count', 'n_unique', 'n_unique_ratio']
batch_size = 32
OUT_DROPOUT = 0.3
BERT_HIDDEN_SIZE = 768
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
BERT_DO_LOWER = 'uncased' in BERT_MODEL_PATH
MAX_LEN = 220
models = [
ridge_model_full_data, lasso_model_full_data, elastic_model_full_data,
gbr_model_full_data, xgb_model_full_data, lgb_model_full_data,
svr_model_full_data, stack_gen_model
]
len(models )<define_search_space>
tqdm.pandas()
%matplotlib inline
public_coefs = [0.1, 0.1, 0.1, 0.1, 0.15, 0.1, 0.1, 0.25]
bias = 0<predict_on_test>
%%time
if DEBUG:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', nrows=DEBUG_DATA_SIZE)
 else:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_test = test['comment_text']
del test
gc.collect()
def linear_blend_models_predict(data_x,models,coefs, bias):
tmp=[model.predict(data_x)for model in models]
tmp = [c*d for c,d in zip(coefs,tmp)]
pres=np.array(tmp ).swapaxes(0,1)
 pres=np.sum(pres,axis=1)
 return pres<save_to_csv>
sentence_feature_mat = sentence_df[SENTENCE_FEAURE_USED].values
gc.collect()
print('blending models RMSLE score on train data:')
 print(rmsle(y, linear_blend_models_predict(X,models,public_coefs, bias)))
 print('Predict submission', datetime.now() ,)
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(linear_blend_models_predict(X_sub,models,public_coefs, bias)))
 submission.head()
submission.to_csv("submission_blend_models_with_public_coefs.csv", index=False )<train_model>
logging.basicConfig(level=logging.INFO)
alphas3 = [70]
def blend_models(train_x, train_y, models):
tmp = [model.predict(train_x)for model in models]
pres = np.array(tmp ).swapaxes(0,1)
 print(pres.shape)
 linear = RidgeCV(alphas=alphas3,
cv=kfolds,
fit_intercept=False,
scoring=make_scorer(rmsle, greater_is_better=False)
)
 linear = linear.fit(pres, train_y)
 print('linear coefficient:')
 print(linear.coef_)
 print('linear bias:')
 print(linear.intercept_)
 print('best alpha: %f'%(linear.alpha_))
print('best score: %f'%(rmsle(linear.predict(pres), train_y)))
 return linear.coef_, linear.intercept_<load_from_csv>
tokenizer = BertTokenizer.from_pretrained(
 BERT_MODEL_PATH, cache_dir=None, do_lower_case=BERT_DO_LOWER )
print('blending models RMSLE score on train data:')
 print(rmsle(y, linear_blend_models_predict(X,models,coefs,bias)))
 print('Predict submission', datetime.now() ,)
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(linear_blend_models_predict(X_sub,models,coefs,bias)))
 submission.head()
submission.to_csv("submission_blend_models_with_regression_coefs.csv", index=False )<load_from_csv>
def tokenize(text, max_len, tokenizer):
tokenized_text = tokenizer.tokenize(text)[:max_len-2]
return ["[CLS]"]+tokenized_text+["[SEP]"]
print('Blend with Top Kernals submissions', datetime.now() ,)
 sub_1 = pd.read_csv('.. /input/top-10-0-10943-stacking-mice-and-brutal-force/House_Prices_submit.csv')
 sub_2 = pd.read_csv('.. /input/hybrid-svm-benchmark-approach-0-11180-lb-top-2/hybrid_solution.csv')
 sub_3 = pd.read_csv('.. /input/lasso-model-for-regression-problem/lasso_sol.csv')
 submission.iloc[:,1] = np.floor(( 0.25 * np.floor(np.expm1(linear_blend_models_predict(X_sub,models,public_coefs, bias)))) + 
(0.25 * sub_1.iloc[:,1])+ 
(0.25 * sub_2.iloc[:,1])+ 
(0.25 * sub_3.iloc[:,1]))
submission.to_csv("submission_blend_top.csv", index=False)
 print('Save submission', datetime.now() )<save_to_csv>
x_test = x_test.progress_apply(lambda x: tokenizer.convert_tokens_to_ids(x))
submission = pd.read_csv('.. /input/house-price-best/submission_best.csv')
 submission.to_csv("submission_best.csv", index=False)
 print('Save submission best', datetime.now() )<train_on_grid>
del tokenizer
gc.collect()
<save_to_csv>
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
q1 = submission['SalePrice'].quantile(0.005)
 q2 = submission['SalePrice'].quantile(0.995)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*1.1)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*0.77)
 submission.to_csv("submission_base_blend_top_Scale extremes2.csv", index=False)
 print('Save submission', datetime.now() )<set_options>
with suppress_stdout() :
convert_tf_checkpoint_to_pytorch.convert_tf_checkpoint_to_pytorch(
 BERT_MODEL_PATH + 'bert_model.ckpt',
BERT_MODEL_PATH + 'bert_config.json',
WORK_DIR + 'pytorch_model.bin')
 shutil.copyfile(BERT_MODEL_PATH + 'bert_config.json', WORK_DIR + 'bert_config.json' )
warnings.filterwarnings("ignore" )<load_from_csv>
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
traindf = pd.read_csv('.. /input/titanic/train.csv' ).set_index('PassengerId')
 testdf = pd.read_csv('.. /input/titanic/test.csv' ).set_index('PassengerId')
 submission = pd.read_csv('.. /input/titanic/gender_submission.csv' )<feature_engineering>
def get_model_path(i_seed, i_fold, i_epoch, base_path):
for dir_name in glob(base_path + '*'):
if(i_seed == int(re.match('.*s\d', dir_name ).group() [-1])and
i_fold == int(re.match('.*f\d', dir_name ).group() [-1])and
i_epoch == int(re.match('.*e\d', dir_name ).group() [-1])
):
return dir_name
df = pd.concat([traindf, testdf], axis=0, sort=False)
 df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['IsWomanOrBoy'] =(( df.Title == 'Master')|(df.Sex == 'female'))
df['LastName'] = df.Name.str.split(',' ).str[0]
family = df.groupby(df.LastName ).Survived
df['WomanOrBoyCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).count())
 df['WomanOrBoyCount'] = df.mask(df.IsWomanOrBoy, df.WomanOrBoyCount - 1, axis=0)
 df['FamilySurvivedCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).sum())
 df['FamilySurvivedCount'] = df.mask(df.IsWomanOrBoy, df.FamilySurvivedCount - \
df.Survived.fillna(0), axis=0)
 df['WomanOrBoySurvived'] = df.FamilySurvivedCount / df.WomanOrBoyCount.replace(0, np.nan)
 df.WomanOrBoyCount = df.WomanOrBoyCount.replace(np.nan, 0)
 df['Alone'] =(df.WomanOrBoyCount == 0)
 df['Title'] = df['Title'].replace('Ms','Miss')
 df['Title'] = df['Title'].replace('Mlle','Miss')
 df['Title'] = df['Title'].replace('Mme','Mrs')
 df['Embarked'] = df['Embarked'].fillna('S')
 df['Deck'] = df['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'M')
 med_fare = df.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
df['Fare'] = df['Fare'].fillna(med_fare)
 df['Age'] = df.groupby(['Sex', 'Pclass', 'Title'])['Age'].apply(lambda x: x.fillna(x.median()))
df['Family_Size'] = df['SibSp'] + df['Parch'] + 1
cols_to_drop = ['Name','Ticket','Cabin', 'IsWomanOrBoy', 'WomanOrBoyCount', 'FamilySurvivedCount']
df = df.drop(cols_to_drop, axis=1)
 df.WomanOrBoySurvived = df.WomanOrBoySurvived.fillna(0)
 df.Alone = df.Alone.fillna(0)
 target = df.Survived.loc[traindf.index]
df = df.drop(['Survived'], axis=1)
 train, test = df.loc[traindf.index], df.loc[testdf.index]<set_options>
get_model_path(0, 0, 1, '.. /input/base-uncased' )
pd.set_option('max_columns',100 )<categorify>
class NeuralNet(nn.Module):
def __init__(self, num_aux_targets, num_sentence_features):
super(NeuralNet, self ).__init__()
self.bert = BertModel.from_pretrained(WORK_DIR)
 self.dropout = nn.Dropout(OUT_DROPOUT)
 self.linear_sentence1 = nn.Linear(num_sentence_features, num_sentence_features)
 n_hidden = BERT_HIDDEN_SIZE + num_sentence_features
self.linear1 = nn.Linear(n_hidden, n_hidden)
 self.linear_out = nn.Linear(n_hidden, 1)
 self.linear_aux_out = nn.Linear(n_hidden, num_aux_targets)
 def forward(self, x_features, sentence_features):
_, bert_output = self.bert(*x_features, output_all_encoded_layers=False)
 bert_output = self.dropout(bert_output)
 h_sentence = self.linear_sentence1(sentence_features)
 h_cat = torch.cat(( bert_output, h_sentence), 1)
 h_conc_linear1 = F.relu(self.linear1(h_cat))
hidden = h_cat + h_conc_linear1
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
categorical_columns = []
features = train.columns.values.tolist()
for col in features:
if train[col].dtype in numerics: continue
categorical_columns.append(col)
 for col in categorical_columns:
if col in train.columns:
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))<drop_column>
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 aux_col_list = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
test_dict = {}
fold_list = [0]
i_epoch = 2
start_time = time.time()
for i_fold in fold_list:
fold_start_time = time.time()
MODEL_PATH = '.. /input/bert-base-uncased-ftpe18-e2-5e6'
print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{0}-fold{i_fold}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))), test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False, length_quantile=1, max_batch_size=2048, for_bert=True)
 print(torch.cuda.memory_allocated())
 model = NeuralNet(len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{0}-fold{i_fold}-epoch{i_epoch}.torchModelState')
 model.load_state_dict(torch.load(load_path)['model_state_dict'])
 model.cuda()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
eval_start_time = time.time()
for batch in test_loader:
x_batch = batch[0]
segment_id_batch = batch[1]
input_mask_batch = batch[2]
y_batch = batch[3]
index_batch = batch[4]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
x_features = [torch.tensor(feature, dtype=torch.long ).cuda() for feature in [x_batch, segment_id_batch, input_mask_batch]]
y_pred = model(x_features, sentence_feature_batch)
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del x_features, y_pred
torch.cuda.empty_cache()
print(f'uncased {i_fold} finishd in {time.time() - fold_start_time}', file=open('time.txt', 'a'))
test_dict[i_fold] = epoch_test_pred
del model, epoch_test_pred
gc.collect()
torch.cuda.empty_cache()
print(f'uncased finishd in {time.time() - start_time}', file=open('time.txt', 'a'))
train = reduce_mem_usage(train )<compute_test_metric>
class NeuralNet(nn.Module):
def __init__(self, num_aux_targets, num_sentence_features):
super(NeuralNet, self ).__init__()
self.bert_model = BertModel.from_pretrained(WORK_DIR)
 self.dropout = nn.Dropout(OUT_DROPOUT)
 self.linear_sentence1 = nn.Linear(num_sentence_features, num_sentence_features)
 n_hidden = BERT_HIDDEN_SIZE + num_sentence_features
self.linear1 = nn.Linear(n_hidden, n_hidden)
 self.linear_out = nn.Linear(n_hidden, 1)
 self.linear_aux_out = nn.Linear(n_hidden, num_aux_targets)
 def forward(self, x_features, sentence_features):
_, bert_output = self.bert_model(*x_features, output_all_encoded_layers=False)
 bert_output = self.dropout(bert_output)
 h_sentence = self.linear_sentence1(sentence_features)
 h_cat = torch.cat(( bert_output, h_sentence), 1)
 h_conc_linear1 = F.relu(self.linear1(h_cat))
hidden = h_cat + h_conc_linear1
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
test = reduce_mem_usage(test )<prepare_x_and_y>
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 aux_col_list = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
test_dict = {}
fold_list = [1, 2]
i_epoch = 1
start_time = time.time()
for i_fold in fold_list:
fold_start_time = time.time()
MODEL_PATH = get_model_path(i_seed, i_fold, i_epoch, '.. /input/base-uncased')
 print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{0}-fold{i_fold}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))), test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False, length_quantile=1, max_batch_size=2048, for_bert=True)
 print(torch.cuda.memory_allocated())
 model = NeuralNet(len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{0}-fold{i_fold}-epoch{i_epoch}.torchModelState')
 model.load_state_dict(torch.load(load_path)['model_state_dict'])
 model.cuda()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
eval_start_time = time.time()
for batch in test_loader:
x_batch = batch[0]
segment_id_batch = batch[1]
input_mask_batch = batch[2]
y_batch = batch[3]
index_batch = batch[4]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
x_features = [torch.tensor(feature, dtype=torch.long ).cuda() for feature in [x_batch, segment_id_batch, input_mask_batch]]
y_pred = model(x_features, sentence_feature_batch)
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del x_features, y_pred
torch.cuda.empty_cache()
print(f'uncased {i_fold} finishd in {time.time() - fold_start_time}', file=open('time.txt', 'a'))
test_dict[i_fold] = epoch_test_pred
del model, epoch_test_pred
gc.collect()
torch.cuda.empty_cache()
print(f'uncased finishd in {time.time() - start_time}', file=open('time.txt', 'a'))
X = train
z = target<split>
fold_list = [4, 5]
i_seed = 0
SENTENCE_FEAURE_USED = ['word_count', 'n_unique', 'n_unique_ratio']
batch_size = 32
OUT_DROPOUT = 0.3
BERT_HIDDEN_SIZE = 768
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/cased_l-12_h-768_a-12/cased_L-12_H-768_A-12/'
BERT_DO_LOWER = 'uncased' in BERT_MODEL_PATH
MAX_LEN = 220
Xtrain, Xval, Ztrain, Zval = train_test_split(X, z, test_size=0.2, random_state=0)
 train_set = lgbm.Dataset(Xtrain, Ztrain, silent=False)
 valid_set = lgbm.Dataset(Xval, Zval, silent=False )<init_hyperparams>
if DEBUG:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', nrows=DEBUG_DATA_SIZE)
 else:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_test = test['comment_text']
del test
gc.collect()
params = {
'boosting_type':'gbdt',
'objective': 'regression',
'num_leaves': 31,
'learning_rate': 0.05,
'max_depth': -1,
'subsample': 0.8,
'bagging_fraction' : 1,
'max_bin' : 5000 ,
'bagging_freq': 20,
'colsample_bytree': 0.6,
'metric': 'rmse',
'min_split_gain': 0.5,
'min_child_weight': 1,
'min_child_samples': 10,
'scale_pos_weight':1,
'zero_as_missing': True,
'seed':0,
}
modelL = lgbm.train(params, train_set = train_set, num_boost_round=1000,
early_stopping_rounds=50, verbose_eval=10, valid_sets=valid_set )<create_dataframe>
tokenizer = BertTokenizer.from_pretrained(
 BERT_MODEL_PATH, cache_dir=None, do_lower_case=BERT_DO_LOWER )
feature_score = pd.DataFrame(train.columns, columns = ['feature'])
 feature_score['score_lgb'] = modelL.feature_importance()<predict_on_test>
def tokenize(text, max_len, tokenizer):
tokenized_text = tokenizer.tokenize(text)[:max_len-2]
return ["[CLS]"]+tokenized_text+["[SEP]"]
x_test = x_test.progress_apply(lambda x: tokenize(x, MAX_LEN, tokenizer))
y_train_lgb = modelL.predict(train, num_iteration=modelL.best_iteration ).astype('int')
 y_preds_lgb = modelL.predict(test, num_iteration=modelL.best_iteration )<prepare_x_and_y>
x_test = x_test.progress_apply(lambda x: tokenizer.convert_tokens_to_ids(x))
data_tr = xgb.DMatrix(Xtrain, label=Ztrain)
 data_cv = xgb.DMatrix(Xval , label=Zval)
 data_train = xgb.DMatrix(train)
 data_test = xgb.DMatrix(test)
 evallist = [(data_tr, 'train'),(data_cv, 'valid')]<train_model>
del tokenizer
gc.collect()
parms = {'max_depth':8,
'objective':'reg:logistic',
'eta' :0.3,
'subsample':0.8,
'lambda ' :4,
'colsample_bytree ':0.9,
'colsample_bylevel':1,
'min_child_weight': 10}
modelx = xgb.train(parms, data_tr, num_boost_round=200, evals = evallist,
early_stopping_rounds=30, maximize=False,
verbose_eval=10)
 print('score = %1.5f, n_boost_round =%d.'%(modelx.best_score,modelx.best_iteration))<compute_test_metric>
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
feature_score['score_xgb'] = feature_score['feature'].map(modelx.get_score(importance_type='weight'))
feature_score<predict_on_test>
with suppress_stdout() :
convert_tf_checkpoint_to_pytorch.convert_tf_checkpoint_to_pytorch(
 BERT_MODEL_PATH + 'bert_model.ckpt',
BERT_MODEL_PATH + 'bert_config.json',
WORK_DIR + 'pytorch_model.bin')
 shutil.copyfile(BERT_MODEL_PATH + 'bert_config.json', WORK_DIR + 'bert_config.json' )
y_train_xgb = modelx.predict(data_train ).astype('int')
 y_preds_xgb = modelx.predict(data_test )<normalization>
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
Scaler_train = preprocessing.MinMaxScaler()
train = pd.DataFrame(
 Scaler_train.fit_transform(train),
columns=train.columns,
index=train.index
)<normalization>
get_model_path(i_seed, 4, 1, '.. /input/base-cased' )
test = pd.DataFrame(
 Scaler_train.fit_transform(test),
columns=test.columns,
index=test.index
)<train_on_grid>
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 test_dict = {}
i_epoch = 1
start_time = time.time()
for i_fold in fold_list:
fold_start_time = time.time()
MODEL_PATH = get_model_path(i_seed, i_fold, i_epoch, '.. /input/base-cased')
 print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{0}-fold{i_fold}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))), test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False, length_quantile=1, max_batch_size=2048, for_bert=True)
 print(torch.cuda.memory_allocated())
 model = NeuralNet(len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{0}-fold{i_fold}-epoch{i_epoch}.torchModelState')
 model.load_state_dict(torch.load(load_path)['model_state_dict'])
 model.cuda()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
eval_start_time = time.time()
for batch in test_loader:
x_batch = batch[0]
segment_id_batch = batch[1]
input_mask_batch = batch[2]
y_batch = batch[3]
index_batch = batch[4]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
x_features = [torch.tensor(feature, dtype=torch.long ).cuda() for feature in [x_batch, segment_id_batch, input_mask_batch]]
y_pred = model(x_features, sentence_feature_batch)
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del x_features, y_pred
torch.cuda.empty_cache()
print(f'uncased {i_fold} finishd in {time.time() - fold_start_time}', file=open('time.txt', 'a'))
test_dict[i_fold] = epoch_test_pred
del model, epoch_test_pred
gc.collect()
torch.cuda.empty_cache()
print(f'uncased finishd in {time.time() - start_time}', file=open('time.txt', 'a'))
logreg = LogisticRegression()
logreg.fit(train, target)
 coeff_logreg = pd.DataFrame(train.columns.delete(0))
coeff_logreg.columns = ['feature']
coeff_logreg["score_logreg"] = pd.Series(logreg.coef_[0])
 coeff_logreg.sort_values(by='score_logreg', ascending=False )<merge>
test_pred_base_CASED = np.mean(list(test_dict.values()), axis=0 )
coeff_logreg["score_logreg"] = coeff_logreg["score_logreg"].abs()
feature_score = pd.merge(feature_score, coeff_logreg, on='feature' )<load_pretrained>
del test_loader, test_dict
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
eli5.show_weights(logreg )<predict_on_test>
batch_size = 8
n_seeds = 1
n_splits = 10
n_epochs = 1
TRAIN_ON_N_SPLITS = 1
EMBED_SIZE = 300
SUBGROUP_NEGATIVE_WEIGHT_COEF = 1
BACKGROUND_POSITIVE_WEIGHT_COEF = 0
BERT_HIDDEN_SIZE = 1024
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/uncased_l-24_h-1024_a-16/uncased_L-24_H-1024_A-16/'
BERT_DO_LOWER = 'uncased' in BERT_MODEL_PATH
ENSEMBLE_START_EPOCH = 0
OUT_DROPOUT = 0.3
MAX_LEN = 220
y_train_logreg = logreg.predict(train ).astype('int')
 y_preds_logreg = logreg.predict(test )<train_on_grid>
%%time
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', '
', '\r']
def clean_text(x: str)-> str:
for punct in puncts:
if punct in x:
x = x.replace(punct, ' {} '.format(punct))
return x
def clean_numbers(x):
return re.sub('\d+', ' ', x)
 if DEBUG:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', nrows=DEBUG_DATA_SIZE)
 else:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_test_clean = test['comment_text'].apply(lambda x: clean_text(x)).apply(lambda x: clean_numbers(x))
del test
gc.collect()
x_test_clean.head(20)
 def add_cls_sep(text):
return "[CLS] " + text + " [SEP]"
x_test = x_test_clean.apply(lambda x: add_cls_sep(x))
x_test.head(10)
linreg = LinearRegression()
linreg.fit(train, target)
 coeff_linreg = pd.DataFrame(train.columns.delete(0))
coeff_linreg.columns = ['feature']
coeff_linreg["score_linreg"] = pd.Series(linreg.coef_)
 coeff_linreg.sort_values(by='score_linreg', ascending=False )<load_pretrained>
tokenizer = BertTokenizer.from_pretrained(
 BERT_MODEL_PATH, cache_dir=None, do_lower_case=BERT_DO_LOWER)
 tqdm.pandas()
x_test = x_test.progress_apply(lambda x: tokenizer.tokenize(x))
x_test.head(10 )
eli5.show_weights(linreg )<feature_engineering>
x_test = x_test.progress_apply(lambda x: tokenizer.convert_tokens_to_ids(x)[:MAX_LEN])
 x_test.head(10 )
coeff_linreg["score_linreg"] = coeff_linreg["score_linreg"].abs()<merge>
del tokenizer
gc.collect()
feature_score = pd.merge(feature_score, coeff_linreg, on='feature')
 feature_score = feature_score.fillna(0)
 feature_score = feature_score.set_index('feature')
 feature_score<predict_on_test>
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
y_train_linreg = linreg.predict(train ).astype('int')
 y_preds_linreg = linreg.predict(test )<sort_values>
with suppress_stdout() :
convert_tf_checkpoint_to_pytorch.convert_tf_checkpoint_to_pytorch(
 BERT_MODEL_PATH + 'bert_model.ckpt',
BERT_MODEL_PATH + 'bert_config.json',
WORK_DIR + 'pytorch_model.bin')
 shutil.copyfile(BERT_MODEL_PATH + 'bert_config.json', WORK_DIR + 'bert_config.json' )
feature_score.sort_values('mean', ascending=False )<init_hyperparams>
class NeuralNet(nn.Module):
def __init__(self, num_aux_targets, num_sentence_features):
super(NeuralNet, self ).__init__()
self.bert_model = BertModel.from_pretrained(WORK_DIR)
 self.dropout = nn.Dropout(OUT_DROPOUT)
 self.linear_sentence1 = nn.Linear(num_sentence_features, num_sentence_features)
 n_hidden = BERT_HIDDEN_SIZE + num_sentence_features
self.linear1 = nn.Linear(n_hidden, n_hidden)
 self.linear_out = nn.Linear(n_hidden, 1)
 self.linear_aux_out = nn.Linear(n_hidden, num_aux_targets)
 def forward(self, x_features, sentence_features):
_, bert_output = self.bert_model(*x_features, output_all_encoded_layers=False)
 bert_output = self.dropout(bert_output)
 h_sentence = self.linear_sentence1(sentence_features)
 h_cat = torch.cat(( bert_output, h_sentence), 1)
 h_conc_linear1 = F.relu(self.linear1(h_cat))
hidden = h_cat + h_conc_linear1
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
w_lgb = 0.48
w_xgb = 0.48
w_logreg = 0.03
w_linreg = 1 - w_lgb - w_xgb - w_logreg
w_linreg<sort_values>
aux_col_list = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
feature_score.sort_values('merging', ascending=False )<prepare_output>
sentence_feature_mat = sentence_df.drop(columns=['n_oov'] ).values
y_preds = w_lgb*y_preds_lgb + w_xgb*y_preds_xgb + w_logreg*y_preds_logreg + w_linreg*y_preds_linreg<prepare_output>
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
submission['Survived'] = [1 if x>0.5 else 0 for x in y_preds]
submission.head()<save_to_csv>
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 test_array = np.zeros(len(x_test))
kf = KFold(n_splits=n_splits, shuffle=True)
 for i_seed in range(n_seeds):
print(f'start seed {i_seed}')
 for i_fold,(dev_index, val_index)in enumerate(kf.split(x_test)) :
if i_fold >= TRAIN_ON_N_SPLITS:
break
print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{0}-fold{0}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))),
test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False, length_quantile=1, max_batch_size=2048, for_bert=True)
 print(torch.cuda.memory_allocated())
 mb = master_bar(range(n_epochs))
i_epoch = 1
model = NeuralNet(len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{0}-fold{0}-epoch{i_epoch*4+3}.torchModelState')
 model.load_state_dict(torch.load(load_path))
model.cuda()
print(i_epoch)
 start_time = time.time()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
for batch in progress_bar(test_loader):
x_batch = batch[0]
segment_id_batch = batch[1]
input_mask_batch = batch[2]
y_batch = batch[3]
index_batch = batch[4]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
x_features = [torch.tensor(feature, dtype=torch.long ).cuda() for feature in [x_batch, segment_id_batch, input_mask_batch]]
y_pred = model(x_features, sentence_feature_batch)
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del x_features, y_pred
torch.cuda.empty_cache()
test_array = epoch_test_pred
elapsed_time = time.time() - start_time
del model, epoch_test_pred
gc.collect()
torch.cuda.empty_cache()
submission.to_csv('submission.csv', index=False )<save_to_csv>
test_pred_large_uncased = test_array
submission.to_csv('submission.csv', index=False )<set_options>
del test_loader
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
pd.set_option('max_columns', None)
 plt.style.use('seaborn')
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
<train_model>
batch_size = 8
n_seeds = 1
n_splits = 10
n_epochs = 1
TRAIN_ON_N_SPLITS = 1
EMBED_SIZE = 300
SUBGROUP_NEGATIVE_WEIGHT_COEF = 1
BACKGROUND_POSITIVE_WEIGHT_COEF = 0
BERT_HIDDEN_SIZE = 1024
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/cased_l-24_h-1024_a-16/cased_L-24_H-1024_A-16/'
BERT_DO_LOWER = 'uncased' in BERT_MODEL_PATH
ENSEMBLE_START_EPOCH = 0
OUT_DROPOUT = 0.3
MAX_LEN = 220
class LGBWrapper(object):

 def __init__(self):
self.model = lgb.LGBMClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'])
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if self.model.objective == 'binary':
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_)[:, 1]
else:
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_ )<train_model>
%%time
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', '
', '\r']
def clean_text(x: str)-> str:
for punct in puncts:
if punct in x:
x = x.replace(punct, ' {} '.format(punct))
return x
def clean_numbers(x):
return re.sub('\d+', ' ', x)
 if DEBUG:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', nrows=DEBUG_DATA_SIZE)
 else:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_test_clean = test['comment_text'].apply(lambda x: clean_text(x)).apply(lambda x: clean_numbers(x))
del test
gc.collect()
x_test_clean.head(20)
 def add_cls_sep(text):
return "[CLS] " + text + " [SEP]"
x_test = x_test_clean.apply(lambda x: add_cls_sep(x))
x_test.head(10)
class MainTransformer(BaseEstimator, TransformerMixin):
def __init__(self, convert_cyclical: bool = False, create_interactions: bool = False, n_interactions: int = 20):

 self.convert_cyclical = convert_cyclical
self.create_interactions = create_interactions
self.feats_for_interaction = None
self.n_interactions = n_interactions
def fit(self, X, y=None):
if self.create_interactions:
pass
return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data)
 class FeatureTransformer(BaseEstimator, TransformerMixin):
def __init__(self, main_cat_features: list = None, num_cols: list = None):

 self.main_cat_features = main_cat_features
self.num_cols = num_cols
def fit(self, X, y=None):
self.num_cols = [col for col in X.columns if 'sum' in col or 'mean' in col or 'max' in col or 'std' in col
or 'attempt' in col]
return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data )<train_model>
tokenizer = BertTokenizer.from_pretrained(
 BERT_MODEL_PATH, cache_dir=None, do_lower_case=BERT_DO_LOWER)
 tqdm.pandas()
x_test = x_test.progress_apply(lambda x: tokenizer.tokenize(x))
x_test.head(10 )
class ClassifierModel(object):

 def __init__(self, columns: list = None, model_wrapper=None):

 self.columns = columns
self.model_wrapper = model_wrapper
self.result_dict = {}
self.train_one_fold = False
self.preprocesser = None
def fit(self, X: pd.DataFrame, y,
X_holdout: pd.DataFrame = None, y_holdout=None,
folds=None,
params: dict = None,
eval_metric='auc',
cols_to_drop: list = None,
preprocesser=None,
transformers: dict = None,
adversarial: bool = False,
plot: bool = True):

 self.cols_to_drop = cols_to_drop
if folds is None:
folds = KFold(n_splits=3, random_state=42)
 self.train_one_fold = True
self.columns = X.columns if self.columns is None else self.columns
self.feature_importances = pd.DataFrame(columns=['feature', 'importance'])
 self.trained_transformers = {k: [] for k in transformers}
self.transformers = transformers
self.models = []
self.folds_dict = {}
self.eval_metric = eval_metric
n_target = 1 if len(set(y.values)) == 2 else len(set(y.values))
self.oof = np.zeros(( len(X), n_target))
self.n_target = n_target
X = X[self.columns]
if X_holdout is not None:
X_holdout = X_holdout[self.columns]
if preprocesser is not None:
self.preprocesser = preprocesser
self.preprocesser.fit(X, y)
 X = self.preprocesser.transform(X, y)
 self.columns = X.columns.tolist()
if X_holdout is not None:
X_holdout = self.preprocesser.transform(X_holdout)
 for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y)) :
if X_holdout is not None:
X_hold = X_holdout.copy()
else:
X_hold = None
self.folds_dict[fold_n] = {}
if params['verbose']:
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 self.folds_dict[fold_n] = {}
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if self.train_one_fold:
X_train = X[self.original_columns]
y_train = y
X_valid = None
y_valid = None
datasets = {'X_train': X_train, 'X_valid': X_valid, 'X_holdout': X_hold, 'y_train': y_train}
X_train, X_valid, X_hold = self.transform_(datasets, cols_to_drop)
 self.folds_dict[fold_n]['columns'] = X_train.columns.tolist()
model = copy.deepcopy(self.model_wrapper)
 if adversarial:
X_new1 = X_train.copy()
if X_valid is not None:
X_new2 = X_valid.copy()
elif X_holdout is not None:
X_new2 = X_holdout.copy()
X_new = pd.concat([X_new1, X_new2], axis=0)
 y_new = np.hstack(( np.zeros(( X_new1.shape[0])) , np.ones(( X_new2.shape[0]))))
X_train, X_valid, y_train, y_valid = train_test_split(X_new, y_new)
 model.fit(X_train, y_train, X_valid, y_valid, X_hold, y_holdout, params=params)
 self.folds_dict[fold_n]['scores'] = model.best_score_
if self.oof.shape[0] != len(X):
self.oof = np.zeros(( X.shape[0], self.oof.shape[1]))
if not adversarial:
self.oof[valid_index] = model.predict_proba(X_valid ).reshape(-1, n_target)
 fold_importance = pd.DataFrame(list(zip(X_train.columns, model.feature_importances_)) ,
columns=['feature', 'importance'])
 self.feature_importances = self.feature_importances.append(fold_importance)
 self.models.append(model)
 self.feature_importances['importance'] = self.feature_importances['importance'].astype(float)
 self.calc_scores_()
if plot:
print(classification_report(y,(self.oof > 0.5)* 1))
fig, ax = plt.subplots(figsize=(16, 12))
plt.subplot(2, 2, 1)
 self.plot_feature_importance(top_n=25)
 plt.subplot(2, 2, 2)
 self.plot_metric()
plt.subplot(2, 2, 3)
 g = sns.heatmap(confusion_matrix(y,(self.oof > 0.5)* 1), annot=True, cmap=plt.cm.Blues,fmt="d")
 g.set(ylim=(-0.5, 4), xlim=(-0.5, 4), title='Confusion matrix')
 plt.subplot(2, 2, 4)
 plt.hist(self.oof)
 plt.xticks(range(self.n_target), range(self.n_target))
plt.title('Distribution of oof predictions');
def transform_(self, datasets, cols_to_drop):
for name, transformer in self.transformers.items() :
transformer.fit(datasets['X_train'], datasets['y_train'])
 datasets['X_train'] = transformer.transform(datasets['X_train'])
 if datasets['X_valid'] is not None:
datasets['X_valid'] = transformer.transform(datasets['X_valid'])
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = transformer.transform(datasets['X_holdout'])
 self.trained_transformers[name].append(transformer)
 if cols_to_drop is not None:
cols_to_drop = [col for col in cols_to_drop if col in datasets['X_train'].columns]
self.cols_to_drop = cols_to_drop
datasets['X_train'] = datasets['X_train'].drop(cols_to_drop, axis=1)
 if datasets['X_valid'] is not None:
datasets['X_valid'] = datasets['X_valid'].drop(cols_to_drop, axis=1)
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = datasets['X_holdout'].drop(cols_to_drop, axis=1)
 return datasets['X_train'], datasets['X_valid'], datasets['X_holdout']
def calc_scores_(self):
print()
datasets = [k for k, v in [v['scores'] for k, v in self.folds_dict.items() ][0].items() if len(v)> 0]
self.scores = {}
for d in datasets:
scores = [v['scores'][d][self.eval_metric] for k, v in self.folds_dict.items() ]
print(f"CV mean score on {d}: {np.mean(scores):.4f} +/- {np.std(scores):.4f} std.")
 self.scores[d] = np.mean(scores)
 def predict(self, X_test, averaging: str = 'usual'):

 full_prediction = np.zeros(( X_test.shape[0], self.oof.shape[1]))
if self.preprocesser is not None:
X_test = self.preprocesser.transform(X_test)
 for i in range(len(self.models)) :
X_t = X_test.copy()
for name, transformers in self.trained_transformers.items() :
X_t = transformers[i].transform(X_t)
 if self.cols_to_drop:
cols_to_drop = [col for col in self.cols_to_drop if col in X_t.columns]
X_t = X_t.drop(cols_to_drop, axis=1)
 y_pred = self.models[i].predict_proba(X_t[self.folds_dict[i]['columns']] ).reshape(-1, full_prediction.shape[1])
 if full_prediction.shape[0] != len(y_pred):
full_prediction = np.zeros(( y_pred.shape[0], self.oof.shape[1]))
if averaging == 'usual':
full_prediction += y_pred
elif averaging == 'rank':
full_prediction += pd.Series(y_pred ).rank().values
return full_prediction / len(self.models)
 def plot_feature_importance(self, drop_null_importance: bool = True, top_n: int = 10):

 top_feats = self.get_top_features(drop_null_importance, top_n)
 feature_importances = self.feature_importances.loc[self.feature_importances['feature'].isin(top_feats)]
feature_importances['feature'] = feature_importances['feature'].astype(str)
 top_feats = [str(i)for i in top_feats]
sns.barplot(data=feature_importances, x='importance', y='feature', orient='h', order=top_feats)
 plt.title('Feature importances')
 def get_top_features(self, drop_null_importance: bool = True, top_n: int = 10):

 grouped_feats = self.feature_importances.groupby(['feature'])['importance'].mean()
if drop_null_importance:
grouped_feats = grouped_feats[grouped_feats != 0]
return list(grouped_feats.sort_values(ascending=False ).index)[:top_n]
def plot_metric(self):

 full_evals_results = pd.DataFrame()
for model in self.models:
evals_result = pd.DataFrame()
for k in model.model.evals_result_.keys() :
evals_result[k] = model.model.evals_result_[k][self.eval_metric]
evals_result = evals_result.reset_index().rename(columns={'index': 'iteration'})
 full_evals_results = full_evals_results.append(evals_result)
 full_evals_results = full_evals_results.melt(id_vars=['iteration'] ).rename(columns={'value': self.eval_metric,
'variable': 'dataset'})
 full_evals_results[self.eval_metric] = np.abs(full_evals_results[self.eval_metric])
 sns.lineplot(data=full_evals_results, x='iteration', y=self.eval_metric, hue='dataset')
 plt.title('Training progress' )<load_from_csv>
x_test = x_test.progress_apply(lambda x: tokenizer.convert_tokens_to_ids(x)[:MAX_LEN])
 x_test.head(10 )
data_dict = {}
for i in glob.glob('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/*'):
name = i.split('/')[-1].split('.')[0]
if name != 'MTeamSpellings':
data_dict[name] = pd.read_csv(i)
 else:
data_dict[name] = pd.read_csv(i, encoding='cp1252' )<feature_engineering>
del tokenizer
gc.collect()
data_dict['MNCAATourneySeeds'] = data_dict['MNCAATourneySeeds'].loc[data_dict['MNCAATourneySeeds']['Season'] <= 2014]
data_dict['MNCAATourneySeeds']['Seed'] = data_dict['MNCAATourneySeeds']['Seed'].apply(lambda x: int(x[1:3]))
data_dict['MNCAATourneySeeds'] = data_dict['MNCAATourneySeeds'][['Season', 'TeamID', 'Seed']]
data_dict['MNCAATourneyCompactResults'] = data_dict['MNCAATourneyCompactResults'][['Season','WTeamID', 'LTeamID']]
data_dict['MNCAATourneyCompactResults'] = data_dict['MNCAATourneyCompactResults'].loc[data_dict['MNCAATourneyCompactResults']['Season'] <= 2014]
df = pd.merge(data_dict['MNCAATourneyCompactResults'], data_dict['MNCAATourneySeeds'],
how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])
 df = pd.merge(df, data_dict['MNCAATourneySeeds'], how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'])
 df = df.drop(['TeamID_x', 'TeamID_y'], axis=1)
 df.columns = ['Season', 'WTeamID', 'LTeamID', 'WSeed', 'LSeed']
df.head()<merge>
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
team_win_score = data_dict['MRegularSeasonCompactResults'].groupby(['Season', 'WTeamID'] ).agg({'WScore':['sum', 'count']} ).reset_index()
team_win_score.columns = ['Season', 'WTeamID', 'WScore_sum', 'WScore_count']
team_loss_score = data_dict['MRegularSeasonCompactResults'].groupby(['Season', 'LTeamID'] ).agg({'LScore':['sum', 'count']} ).reset_index()
team_loss_score.columns = ['Season', 'LTeamID', 'LScore_sum', 'LScore_count']
df = pd.merge(df, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 df = pd.merge(df, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 df = pd.merge(df, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 df = pd.merge(df, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 df.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 df.head()<feature_engineering>
with suppress_stdout() :
convert_tf_checkpoint_to_pytorch.convert_tf_checkpoint_to_pytorch(
 BERT_MODEL_PATH + 'bert_model.ckpt',
BERT_MODEL_PATH + 'bert_config.json',
WORK_DIR + 'pytorch_model.bin')
 shutil.copyfile(BERT_MODEL_PATH + 'bert_config.json', WORK_DIR + 'bert_config.json' )
df['x_score'] = df['WScore_sum_x'] + df['LScore_sum_y']
df['y_score'] = df['WScore_sum_y'] + df['LScore_sum_x']
df['x_count'] = df['WScore_count_x'] + df['LScore_count_y']
df['y_count'] = df['WScore_count_y'] + df['LScore_count_x']<rename_columns>
class NeuralNet(nn.Module):
def __init__(self, num_aux_targets, num_sentence_features):
super(NeuralNet, self ).__init__()
self.bert_model = BertModel.from_pretrained(WORK_DIR)
 self.dropout = nn.Dropout(OUT_DROPOUT)
 self.linear_sentence1 = nn.Linear(num_sentence_features, num_sentence_features)
 n_hidden = BERT_HIDDEN_SIZE + num_sentence_features
self.linear1 = nn.Linear(n_hidden, n_hidden)
 self.linear_out = nn.Linear(n_hidden, 1)
 self.linear_aux_out = nn.Linear(n_hidden, num_aux_targets)
 def forward(self, x_features, sentence_features):
_, bert_output = self.bert_model(*x_features, output_all_encoded_layers=False)
 bert_output = self.dropout(bert_output)
 h_sentence = self.linear_sentence1(sentence_features)
 h_cat = torch.cat(( bert_output, h_sentence), 1)
 h_conc_linear1 = F.relu(self.linear1(h_cat))
hidden = h_cat + h_conc_linear1
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
df_win = df.copy()
df_los = df.copy()
df_win = df_win[['WSeed', 'LSeed', 'x_score', 'y_score', 'x_count', 'y_count']]
df_los = df_los[['LSeed', 'WSeed', 'y_score', 'x_score', 'y_count', 'x_count']]
df_win.columns = ['Seed_1', 'Seed_2', 'Score_1', 'Score_2', 'Count_1', 'Count_2']
df_los.columns = ['Seed_1', 'Seed_2', 'Score_1', 'Score_2', 'Count_1', 'Count_2']<feature_engineering>
aux_col_list = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
df_win['Seed_diff'] = df_win['Seed_1'] - df_win['Seed_2']
df_win['Score_diff'] = df_win['Score_1'] - df_win['Score_2']
df_los['Seed_diff'] = df_los['Seed_1'] - df_los['Seed_2']
df_los['Score_diff'] = df_los['Score_1'] - df_los['Score_2']
df_win['Count_diff'] = df_win['Count_1'] - df_win['Count_2']
df_win['Mean_score1'] = df_win['Score_1'] / df_win['Count_1']
df_win['Mean_score2'] = df_win['Score_2'] / df_win['Count_2']
df_win['Mean_score_diff'] = df_win['Mean_score1'] - df_win['Mean_score2']
df_los['Count_diff'] = df_los['Count_1'] - df_los['Count_2']
df_los['Mean_score1'] = df_los['Score_1'] / df_los['Count_1']
df_los['Mean_score2'] = df_los['Score_2'] / df_los['Count_2']
df_los['Mean_score_diff'] = df_los['Mean_score1'] - df_los['Mean_score2']<concatenate>
sentence_feature_mat = sentence_df.drop(columns=['n_oov'] ).values
df_win['result'] = 1
df_los['result'] = 0
data = pd.concat(( df_win, df_los)).reset_index(drop=True )<data_type_conversions>
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()
for col in ['Score_1', 'Score_2', 'Count_1', 'Count_2', 'Score_diff', 'Count_diff']:
print(col)
 data[col] = data[col].fillna(0 ).astype(int )<choose_model_class>
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 test_array = np.zeros(len(x_test))
kf = KFold(n_splits=n_splits, shuffle=True)
 for i_seed in range(n_seeds):
print(f'start seed {i_seed}')
 for i_fold,(dev_index, val_index)in enumerate(kf.split(x_test)) :
if i_fold >= TRAIN_ON_N_SPLITS:
break
print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{0}-fold{0}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))),
test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False, length_quantile=1, max_batch_size=2048, for_bert=True)
 print(torch.cuda.memory_allocated())
 mb = master_bar(range(n_epochs))
i_epoch = 1
model = NeuralNet(len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{0}-fold{0}-epoch{i_epoch*4+3}.torchModelState')
 model.load_state_dict(torch.load(load_path))
model.cuda()
print(i_epoch)
 start_time = time.time()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
for batch in progress_bar(test_loader):
x_batch = batch[0]
segment_id_batch = batch[1]
input_mask_batch = batch[2]
y_batch = batch[3]
index_batch = batch[4]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
x_features = [torch.tensor(feature, dtype=torch.long ).cuda() for feature in [x_batch, segment_id_batch, input_mask_batch]]
y_pred = model(x_features, sentence_feature_batch)
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del x_features, y_pred
torch.cuda.empty_cache()
test_array = epoch_test_pred
elapsed_time = time.time() - start_time
del model, epoch_test_pred
gc.collect()
torch.cuda.empty_cache()
n_fold = 5
folds = RepeatedStratifiedKFold(n_splits=n_fold)
<prepare_x_and_y>
test_pred_large_CASED = test_array
<init_hyperparams><EOS> 
df_submit = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df_submit.prediction = test_pred
df_submit.to_csv('submission.csv', index=False )