Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<count_missing_values>
from fastai.train import Learner
from fastai.train import DataBunch
from fastai.callbacks import *
from fastai.basic_data import DatasetType
train_df_raw.isnull().values.any()
train_df_raw.isna().values.any()<categorify>
import numpy as np
import pandas as pd
import os
import time
import gc
import random
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
from keras.preprocessing import text, sequence
import torch
from torch import nn
from torch.utils import data
from torch.nn import functional as F
train_df = train_df_raw.copy()
X_train = train_df.drop(['label'], 1)
 Y_train = train_df['label']
X_train = X_train.values.reshape(X_train.shape[0], 28, 28, 1 ).astype('float32')/ 255
Y_train = np_utils.to_categorical(Y_train, 10 )<choose_model_class>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
def build_cnn() :
model = Sequential()
model.add(Conv2D(32,kernel_size=3,activation='relu',input_shape=(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model<data_type_conversions>
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
X_test = pd.read_csv(path_test)
 X_test = X_test.values.reshape(X_test.shape[0], 28, 28, 1 ).astype('float32')/ 255<train_model>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
model = build_cnn()
model.fit(X_train, Y_train, batch_size=64, epochs=16 )<save_to_csv>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds
prediction = model.predict_classes(X_test, verbose=0)
 submission = pd.DataFrame({"ImageId": list(range(1,len(prediction)+1)) ,
"Label": prediction})
 submission.to_csv("submission.csv", index=False, header=True )<import_modules>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Dropout, BatchNormalization
from tensorflow.keras.models import Sequential
import optuna
from optuna.samplers import TPESampler
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, f1_score<load_from_csv>
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv' )<feature_engineering>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )
train['LastName'] = train['Name'].str.split(',', expand=True)[0]
test['LastName'] = test['Name'].str.split(',', expand=True)[0]
ds = pd.concat([train, test])
 sur = []
died = []
for index, row in ds.iterrows() :
s = ds[(ds['LastName']==row['LastName'])&(ds['Survived']==1)]
d = ds[(ds['LastName']==row['LastName'])&(ds['Survived']==0)]
s=len(s)
 if row['Survived'] == 1:
s-=1
d=len(d)
 if row['Survived'] == 0:
d-=1
sur.append(s)
 died.append(d)
 ds['FamilySurvived'] = sur
ds['FamilyDied'] = died
ds['FamilySize'] = ds['SibSp'] + ds['Parch'] + 1
ds['IsAlone'] = 0
ds.loc[ds['FamilySize'] == 1, 'IsAlone'] = 1
ds['Fare'] = ds['Fare'].fillna(train['Fare'].median())
 ds['Embarked'] = ds['Embarked'].fillna('Q')
 train = ds[ds['Survived'].notnull() ]
test = ds[ds['Survived'].isnull() ]
test = test.drop(['Survived'], axis=1)
 train['rich_woman'] = 0
test['rich_woman'] = 0
train['men_3'] = 0
test['men_3'] = 0
train.loc[(train['Pclass']<=2)&(train['Sex']=='female'), 'rich_woman'] = 1
test.loc[(test['Pclass']<=2)&(test['Sex']=='female'), 'rich_woman'] = 1
train.loc[(train['Pclass']==3)&(train['Sex']=='male'), 'men_3'] = 1
test.loc[(test['Pclass']==3)&(test['Sex']=='male'), 'men_3'] = 1
train['rich_woman'] = train['rich_woman'].astype(np.int8)
 test['rich_woman'] = test['rich_woman'].astype(np.int8)
 train["Cabin"] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in train['Cabin']])
 test['Cabin'] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in test['Cabin']])
 train = train.drop(['PassengerId', 'Ticket', 'LastName', 'SibSp', 'Parch'], axis=1)
 test = test.drop(['PassengerId', 'Ticket', 'LastName', 'SibSp', 'Parch'], axis=1)
 categorical = ['Pclass', 'Sex', 'Embarked', 'Cabin']
for cat in categorical:
train = pd.concat([train, pd.get_dummies(train[cat], prefix=cat)], axis=1)
 train = train.drop([cat], axis=1)
 test = pd.concat([test, pd.get_dummies(test[cat], prefix=cat)], axis=1)
 test = test.drop([cat], axis=1)
 train = train.drop(['Sex_male', 'Name'], axis=1)
 test = test.drop(['Sex_male', 'Name'], axis=1)
 train = train.fillna(-1)
 test = test.fillna(-1)
 train.head()<init_hyperparams>
x_train = preprocess(train['comment_text'])
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'])
 identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
EPOCHS = 15
initial_keras_params = {
'layers_number': 1,
'n_units_l_0': 128,
'activation_l_0': 'relu',
'dropout_l_0': 0.5,
'lr': 0.001
}<choose_model_class>
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
def keras_classifier(parameters):
model = Sequential()
layers_number = int(parameters['layers_number'])
 for i in range(layers_number):
model.add(Dense(int(parameters['n_units_l_' + str(i)]), activation=parameters['activation_l_' + str(i)]))
model.add(Dropout(int(parameters['dropout_l_' + str(i)])))
 model.add(Dense(2, activation='softmax'))
model.compile(
 loss='categorical_crossentropy',
optimizer=tf.keras.optimizers.Adam(lr=float(parameters['lr'])) ,
metrics=['accuracy'] 
)
 return model<choose_model_class>
max_features = None
model = keras_classifier(initial_keras_params )<split>
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
y = train['Survived']
y = tf.keras.utils.to_categorical(y, num_classes=2, dtype='float32')
 X = train.drop(['Survived', 'Cabin_T'], axis=1)
 X_test = test.copy()
X, X_val, y, y_val = train_test_split(X, y, random_state=0, test_size=0.2, shuffle=False )<train_model>
max_features = max_features or len(tokenizer.word_index)+ 1
max_features
model.fit(X, y, validation_split=0.2, epochs=EPOCHS, batch_size=32 )<compute_test_metric>
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
preds = model.predict(X_val)
 preds = np.argmax(preds, axis=1)
 print('accuracy: ', accuracy_score(np.argmax(y_val, axis=1), preds))
print('f1-score: ', f1_score(np.argmax(y_val, axis=1), preds))<choose_model_class>
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
def create_model(trial):
n_layers = trial.suggest_int("layers_number", 1, 2)
 model = Sequential()
for i in range(n_layers):
num_hidden = trial.suggest_int("n_units_l_{}".format(i), 2, 16)
 activation = trial.suggest_categorical('activation_l_{}'.format(i), ['relu', 'sigmoid', 'tanh', 'elu'])
 model.add(Dense(num_hidden, activation=activation))
dropout = trial.suggest_uniform("dropout_l_{}".format(i), 0.1, 0.4)
 model.add(Dropout(dropout))
model.add(Dense(2, activation='softmax'))
lr = trial.suggest_loguniform("lr", 1e-5, 1e-1)
 model.compile(
 loss='categorical_crossentropy',
optimizer=tf.keras.optimizers.Adam(lr=lr),
metrics=['accuracy'] 
)
 return model<train_model>
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()
def objective(trial):
model = create_model(trial)
 epochs = trial.suggest_int("epochs", 3, 20)
 batch = trial.suggest_int("batch", 1, X.shape[0] / 4)
 model.fit(
 X,
y,
batch_size=batch,
epochs=epochs,
verbose=0 
)
 preds = model.predict(X_val)
 return accuracy_score(np.argmax(y_val, axis=1), np.argmax(preds, axis=1)) 
<find_best_params>
x_train_torch = torch.tensor(x_train, dtype=torch.long)
 y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32 )
def optimize() :
sampler = TPESampler(seed=666)
 study = optuna.create_study(direction="maximize", sampler=sampler)
 study.optimize(objective, n_trials=80)
 return study.best_params<find_best_params>
x_test_torch = torch.tensor(x_test, dtype=torch.long )
params = optimize()<train_model>
batch_size = 512
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 valid_dataset = data.TensorDataset(x_train_torch[:batch_size], y_train_torch[:batch_size])
 test_dataset = data.TensorDataset(x_test_torch)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 databunch = DataBunch(train_dl=train_loader,valid_dl=valid_loader )
epochs = params['epochs']
batch = params['batch']
del params['epochs']
del params['batch']
opt_model = keras_classifier(params)
 opt_model.fit(X, y, validation_split=0.2, epochs=epochs, batch_size=batch )<predict_on_test>
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2
preds = opt_model.predict(X_val)
 preds = np.argmax(preds, axis=1)
 print('accuracy: ', accuracy_score(np.argmax(y_val, axis=1), preds))
print('f1-score: ', f1_score(np.argmax(y_val, axis=1), preds))<predict_on_test>
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch,model,loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )
preds = opt_model.predict(X_test)
 preds = np.argmax(preds, axis=1)
 preds = preds.astype(np.int16 )<save_to_csv>
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )