Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<concatenate>
import numpy as np
import pandas as pd
import os
from tqdm import tqdm, trange
from sklearn.metrics import roc_auc_score
import pickle
import gc
dfMrg = concat_df(df_train, df_test)
 dfMrg.head()<feature_engineering>
BERT_MODEL = 'bert-base-uncased'
CASED = 'uncased' in BERT_MODEL
INPUT = '.. /input/jigsaw-bert-preprocessed-input/'
TEXT_COL = 'comment_text'
MAXLEN = 250
dfMrg['Fare'] = pd.qcut(dfMrg['Fare'], 13 )<feature_engineering>
os.system('pip install --no-index --find-links=".. /input/pytorchpretrainedbert/" pytorch_pretrained_bert' )
dfMrg['Age'] = pd.qcut(dfMrg['Age'],10 )<categorify>
from pytorch_pretrained_bert import BertTokenizer
from pytorch_pretrained_bert.modeling import BertModel
dfMrg['Ticket_Frequency'] = dfMrg.groupby('Ticket')['Ticket'].transform('count' )<string_transform>
BERT_FP = '.. /input/torch-bert-weights/bert-base-uncased/bert-base-uncased/'
def extract_surname(data):
families = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(',')[0]
title = name_no_bracket.split(',')[1].strip().split(' ')[0]
for c in string.punctuation:
family = family.replace(c, '' ).strip()
families.append(family)
 return families
dfMrg['Family'] = extract_surname(dfMrg['Name'])
 df_train = dfMrg.loc[:890]
df_test = dfMrg.loc[891:]
dfs = [df_train, df_test]<define_variables>
def get_bert_embed_matrix() :
bert = BertModel.from_pretrained(BERT_FP)
 bert_embeddings = list(bert.children())[0]
bert_word_embeddings = list(bert_embeddings.children())[0]
mat = bert_word_embeddings.weight.data.numpy()
return mat
mean_survival_rate = np.mean(df_train['Survived'])
 train_family_survival_rate = []
train_family_survival_rate_NA = []
test_family_survival_rate = []
test_family_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Family'][i] in family_rates:
train_family_survival_rate.append(family_rates[df_train['Family'][i]])
 train_family_survival_rate_NA.append(1)
 else:
train_family_survival_rate.append(mean_survival_rate)
 train_family_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Family'].iloc[i] in family_rates:
test_family_survival_rate.append(family_rates[df_test['Family'].iloc[i]])
 test_family_survival_rate_NA.append(1)
 else:
test_family_survival_rate.append(mean_survival_rate)
 test_family_survival_rate_NA.append(0)
 df_train['Family_Survival_Rate'] = train_family_survival_rate
df_train['Family_Survival_Rate_NA'] = train_family_survival_rate_NA
df_test['Family_Survival_Rate'] = test_family_survival_rate
df_test['Family_Survival_Rate_NA'] = test_family_survival_rate_NA
train_ticket_survival_rate = []
train_ticket_survival_rate_NA = []
test_ticket_survival_rate = []
test_ticket_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Ticket'][i] in ticket_rates:
train_ticket_survival_rate.append(ticket_rates[df_train['Ticket'][i]])
 train_ticket_survival_rate_NA.append(1)
 else:
train_ticket_survival_rate.append(mean_survival_rate)
 train_ticket_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Ticket'].iloc[i] in ticket_rates:
test_ticket_survival_rate.append(ticket_rates[df_test['Ticket'].iloc[i]])
 test_ticket_survival_rate_NA.append(1)
 else:
test_ticket_survival_rate.append(mean_survival_rate)
 test_ticket_survival_rate_NA.append(0)
 df_train['Ticket_Survival_Rate'] = train_ticket_survival_rate
df_train['Ticket_Survival_Rate_NA'] = train_ticket_survival_rate_NA
df_test['Ticket_Survival_Rate'] = test_ticket_survival_rate
df_test['Ticket_Survival_Rate_NA'] = test_ticket_survival_rate_NA<feature_engineering>
embedding_matrix = get_bert_embed_matrix()
for df in [df_train, df_test]:
df['Survival_Rate'] =(df['Ticket_Survival_Rate'] + df['Family_Survival_Rate'])/ 2
df['Survival_Rate_NA'] =(df['Ticket_Survival_Rate_NA'] + df['Family_Survival_Rate_NA'])/ 2<categorify>
train = pd.read_csv(INPUT + 'train_bert-base-uncased_ids.csv' ).sample(frac = 1.0, random_state = 23 )
non_numeric_features = ['Embarked', 'Sex', 'Deck', 'Title', 'Family_Size_Grouped', 'Age', 'Fare']
for df in dfs:
for feature in non_numeric_features:
df[feature] = LabelEncoder().fit_transform(df[feature] )<categorify>
test = pd.read_csv(INPUT + 'test_bert-base-uncased_ids.csv' )
cat_features = ['Pclass', 'Sex', 'Deck', 'Embarked', 'Title', 'Family_Size_Grouped']
encoded_features = []
for df in dfs:
for feature in cat_features:
encoded_feat = OneHotEncoder().fit_transform(df[feature].values.reshape(-1, 1)).toarray()
n = df[feature].nunique()
cols = ['{}_{}'.format(feature, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded_feat, columns=cols)
 encoded_df.index = df.index
encoded_features.append(encoded_df)
 df_train = pd.concat([df_train, *encoded_features[:6]], axis=1)
 df_test = pd.concat([df_test, *encoded_features[6:]], axis=1 )<concatenate>
x_train = np.zeros(( train.shape[0],MAXLEN),dtype=np.int)
 for i,ids in tqdm(enumerate(list(train[TEXT_COL]))):
input_ids = [int(i)for i in ids.split() [:MAXLEN]]
inp_len = len(input_ids)
 x_train[i,:inp_len] = np.array(input_ids)
 x_test = np.zeros(( test.shape[0],MAXLEN),dtype=np.int)
 for i,ids in tqdm(enumerate(list(test[TEXT_COL]))):
input_ids = [int(i)for i in ids.split() [:MAXLEN]]
inp_len = len(input_ids)
 x_test[i,:inp_len] = np.array(input_ids)
 with open('temporary.pickle', mode='wb')as f:
pickle.dump(x_test, f)
 del x_test
gc.collect()
dfMrg = concat_df(df_train, df_test)
 drop_cols = ['Deck', 'Embarked', 'Family', 'Family_Size', 'Family_Size_Grouped', 'Survived',
'Name', 'Parch', 'PassengerId', 'Pclass', 'Sex', 'SibSp', 'Ticket', 'Title',
'Ticket_Survival_Rate', 'Family_Survival_Rate', 'Ticket_Survival_Rate_NA', 'Family_Survival_Rate_NA']
dfMrg.drop(columns=drop_cols, inplace=True)
 dfMrg.head()<normalization>
identity_columns = ['male','female','homosexual_gay_or_lesbian','christian','jewish','muslim','black','white','psychiatric_or_mental_illness']
y_identities =(train[identity_columns] >= 0.5 ).astype(int ).values
weights = np.ones(( len(train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']].values
X_train = StandardScaler().fit_transform(df_train.drop(columns=drop_cols))
y_train = df_train['Survived'].values
X_test = StandardScaler().fit_transform(df_test.drop(columns=drop_cols))
print('X_train shape: {}'.format(X_train.shape))
print('y_train shape: {}'.format(y_train.shape))
print('X_test shape: {}'.format(X_test.shape)) 
<prepare_x_and_y>
from keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.losses import binary_crossentropy
from keras import backend as K
import keras.layers as L
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.models import Model
from keras.optimizers import Adam
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import KFold
y_test = dfGenderSubmission.drop("PassengerId", axis=1 ).copy()<define_variables>
def build_model(embedding_matrix, num_aux_targets, loss_weight):

 words = Input(shape=(MAXLEN,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([GlobalMaxPooling1D()(x),GlobalAveragePooling1D()(x),])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss=[custom_loss,'binary_crossentropy'], loss_weights=[loss_weight, 1.0], optimizer='adam')
 return model
SEED = 42 
<choose_model_class>
def custom_loss(y_true, y_pred):
return binary_crossentropy(K.reshape(y_true[:,0],(-1,1)) , y_pred)* y_true[:,1]
single_best_model = RandomForestClassifier(criterion='gini',
n_estimators=1100,
max_depth=5,
min_samples_split=4,
min_samples_leaf=5,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1)
 leaderboard_model = RandomForestClassifier(criterion='gini',
n_estimators=1750,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1 )<import_modules>
tr_ind, val_ind = train_test_split(list(range(len(x_train))),test_size = 0.05, random_state = 23 )
from sklearn.metrics import roc_curve, auc
from sklearn.model_selection import StratifiedKFold 
<train_model>
NUM_MODELS = 1
BATCH_SIZE = 512
EPOCHS = 5
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 512
checkpoint_predictions = []
checkpoint_val_preds = []
weights = []
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1],loss_weight)
 for global_epoch in range(EPOCHS):
model.fit(x_train[tr_ind],[y_train[tr_ind], y_aux_train[tr_ind]],validation_data =(x_train[val_ind],[y_train[val_ind], y_aux_train[val_ind]]),
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
callbacks=[
LearningRateScheduler(lambda epoch: 1e-3 *(0.6 ** global_epoch))
] 
)
 with open('temporary.pickle', mode='rb')as f:
x_test = pickle.load(f)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 checkpoint_val_preds.append(model.predict(x_train[val_ind], batch_size=2048)[0].flatten())
 del x_test
gc.collect()
weights.append(2 ** global_epoch)
 del model
gc.collect()
predictions = np.average(checkpoint_predictions, weights=weights, axis=0 )
N = 5
oob = 0
probs = pd.DataFrame(np.zeros(( len(X_test), N * 2)) , columns=['Fold_{}_Prob_{}'.format(i, j)for i in range(1, N + 1)for j in range(2)])
 importances = pd.DataFrame(np.zeros(( X_train.shape[1], N)) , columns=['Fold_{}'.format(i)for i in range(1, N + 1)], index=dfMrg.columns)
 fprs, tprs, scores = [], [], []
skf = StratifiedKFold(n_splits=N, random_state=N, shuffle=True)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(X_train, y_train), 1):
print('Fold {}
'.format(fold))
leaderboard_model.fit(X_train[trn_idx], y_train[trn_idx])
 trn_fpr, trn_tpr, trn_thresholds = roc_curve(y_train[trn_idx], leaderboard_model.predict_proba(X_train[trn_idx])[:, 1])
 trn_auc_score = auc(trn_fpr, trn_tpr)
 val_fpr, val_tpr, val_thresholds = roc_curve(y_train[val_idx], leaderboard_model.predict_proba(X_train[val_idx])[:, 1])
 val_auc_score = auc(val_fpr, val_tpr)
 scores.append(( trn_auc_score, val_auc_score))
fprs.append(val_fpr)
 tprs.append(val_tpr)
 probs.loc[:, 'Fold_{}_Prob_0'.format(fold)] = leaderboard_model.predict_proba(X_test)[:, 0]
probs.loc[:, 'Fold_{}_Prob_1'.format(fold)] = leaderboard_model.predict_proba(X_test)[:, 1]
importances.iloc[:, fold - 1] = leaderboard_model.feature_importances_
oob += leaderboard_model.oob_score_ / N
print('Fold {} OOB Score: {}
'.format(fold, leaderboard_model.oob_score_))
print('Average OOB Score: {}'.format(oob))<save_to_csv>
val_preds = np.average(checkpoint_val_preds, weights=weights, axis=0 )
class_survived = [col for col in probs.columns if col.endswith('Prob_1')]
probs['1'] = probs[class_survived].sum(axis=1)/ N
probs['0'] = probs.drop(columns=class_survived ).sum(axis=1)/ N
probs['pred'] = 0
pos = probs[probs['1'] >= 0.5].index
probs.loc[pos, 'pred'] = 1
y_pred = probs['pred'].astype(int)
 submission_df = pd.DataFrame(columns=['PassengerId', 'Survived'])
 submission_df['PassengerId'] = df_test['PassengerId']
submission_df['Survived'] = y_pred.values
submission_df.to_csv('submissions.csv', header=True, index=False)
 submission_df.head(10 )<set_options>
def power_mean(x, p=-5):
return np.power(np.mean(np.power(x, p)) ,1/p)
 def get_s_auc(y_true,y_pred,y_identity):
mask = y_identity==1
try:
s_auc = roc_auc_score(y_true[mask],y_pred[mask])
 except:
s_auc = 1
return s_auc
def get_bpsn_auc(y_true,y_pred,y_identity):
mask =(y_identity==1)&(y_true==0)|(y_identity==0)&(y_true==1)
 try:
bpsn_auc = roc_auc_score(y_true[mask],y_pred[mask])
 except:
bpsn_auc = 1
return bpsn_auc
def get_bspn_auc(y_true,y_pred,y_identity):
mask =(y_identity==1)&(y_true==1)|(y_identity==0)&(y_true==0)
 try:
bspn_auc = roc_auc_score(y_true[mask],y_pred[mask])
 except:
bspn_auc = 1
return bspn_auc
def get_total_auc(y_true,y_pred,y_identities):
N = y_identities.shape[1]
saucs = np.array([get_s_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 bpsns = np.array([get_bpsn_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 bspns = np.array([get_bspn_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 M_s_auc = power_mean(saucs)
 M_bpsns_auc = power_mean(bpsns)
 M_bspns_auc = power_mean(bspns)
 rauc = roc_auc_score(y_true,y_pred)
 total_auc = M_s_auc + M_bpsns_auc + M_bspns_auc + rauc
total_auc/= 4
return total_auc
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn<feature_engineering>
get_total_auc(y_train[val_ind][:,0],val_preds,y_identities[val_ind] )
<load_from_csv><EOS> 
df_submit = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df_submit.prediction = predictions
df_submit.to_csv('submission.csv', index=False )