Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<train_on_grid>
import numpy as np
import pandas as pd
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNGRU, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler, ModelCheckpoint
from keras.losses import binary_crossentropy
from keras import backend as K
from tqdm import tqdm_notebook as tqdm
import pickle
import gc
n_estimators = list(range(100,1100,100))
learn_rate = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]
depth = [1,2,3]
hyper={'n_estimators':n_estimators,'learning_rate':learn_rate,'max_depth':depth}
gd=GridSearchCV(estimator=GradientBoostingClassifier(random_state=23),param_grid=hyper,verbose=True)
 gd.fit(X, y)
 print("Best Score: ", gd.best_score_)
 print("Best Model: ", gd.best_estimator_ )<train_on_grid>
EMBEDDING_FILES = ['.. /input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl','.. /input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl']
BATCH_SIZE = 512
TCN_UNITS = 128
DENSE_HIDDEN_UNITS = 2*TCN_UNITS
EPOCHS = 4
MAX_LEN = 220
NUM_MODELS = 1
n_estimators = list(range(100,1100,100))
learn_rate = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]
gd=GridSearchCV(estimator=XGBClassifier(random_state=23),param_grid=hyper,verbose=True)
 gd.fit(X, y)
 print("Best Score: ", gd.best_score_)
 print("Best Model: ", gd.best_estimator_ )<train_model>
def channel_normalization(x):

 max_values = K.max(K.abs(x), 2, keepdims=True)+ 1e-5
out = x / max_values
return out
def wave_net_activation(x):

 tanh_out = Activation('tanh' )(x)
 sigm_out = Activation('sigmoid' )(x)
 return keras.layers.multiply([tanh_out, sigm_out])
 def residual_block(x, s, i, activation, nb_filters, kernel_size, padding, dropout_rate=0, name=''):

 original_x = x
conv = Conv1D(filters=nb_filters, kernel_size=kernel_size,
dilation_rate=i, padding=padding,
name=name + '_dilated_conv_%d_tanh_s%d' %(i, s))(x)
 if activation == 'norm_relu':
x = Activation('relu' )(conv)
 x = Lambda(channel_normalization )(x)
 elif activation == 'wavenet':
x = wave_net_activation(conv)
 else:
x = Activation(activation )(conv)
 x = SpatialDropout1D(dropout_rate, name=name + '_spatial_dropout1d_%d_s%d_%f' %(i, s, dropout_rate))(x)
 x = Convolution1D(nb_filters, 1, padding='same' )(x)
 res_x = keras.layers.add([original_x, x])
 return res_x, x
def process_dilations(dilations):
def is_power_of_two(num):
return num != 0 and(( num &(num - 1)) == 0)
 if all([is_power_of_two(i)for i in dilations]):
return dilations
else:
new_dilations = [2 ** i for i in dilations]
return new_dilations
class TCN(Layer):

 def __init__(self,
nb_filters=64,
kernel_size=2,
nb_stacks=1,
dilations=None,
activation='norm_relu',
padding='causal',
use_skip_connections=True,
dropout_rate=0.0,
return_sequences=True,
name='tcn'):
super().__init__()
self.name = name
self.return_sequences = return_sequences
self.dropout_rate = dropout_rate
self.use_skip_connections = use_skip_connections
self.activation = activation
self.dilations = dilations
self.nb_stacks = nb_stacks
self.kernel_size = kernel_size
self.nb_filters = nb_filters
self.padding = padding
if padding != 'causal' and padding != 'same':
raise ValueError("Only 'causal' or 'same' paddings are compatible for this layer.")
 if not isinstance(nb_filters, int):
print('An interface change occurred after the version 2.1.2.')
 print('Before: tcn.TCN(i, return_sequences=False,...) ')
 print('Now should be: tcn.TCN(return_sequences=False,...)(i)')
 print('Second solution is to pip install keras-tcn==2.1.2 to downgrade.')
 raise Exception()
def __call__(self, inputs):
if self.dilations is None:
self.dilations = [1, 2, 4, 8, 16, 32]
x = inputs
x = Convolution1D(self.nb_filters, 1, padding=self.padding, name=self.name + '_initial_conv' )(x)
 skip_connections = []
for s in range(self.nb_stacks):
for i in self.dilations:
x, skip_out = residual_block(x, s, i, self.activation, self.nb_filters,
self.kernel_size, self.padding, self.dropout_rate, name=self.name)
 skip_connections.append(skip_out)
 if self.use_skip_connections:
x = keras.layers.add(skip_connections)
 x = Activation('relu' )(x)
 if not self.return_sequences:
output_slice_index = -1
x = Lambda(lambda tt: tt[:, output_slice_index, :] )(x)
 return x
f,(ax1, ax2)= plt.subplots(2,2,figsize=(15,7))
model1 = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',
max_depth=None, max_features='auto', max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=1,
oob_score=False, random_state=23, verbose=0, warm_start=False)
 model1.fit(X,y)
 pd.Series(model1.feature_importances_, X.columns ).sort_values(ascending=True ).plot.barh(width=0.8,ax=ax1[0])
 ax1[0].set_title('Feature Importance in Random Forests')
 model2 = AdaBoostClassifier(algorithm='SAMME.R', base_estimator=None,
learning_rate=0.1, n_estimators=100, random_state=23)
 model2.fit(X,y)
 pd.Series(model2.feature_importances_, X.columns ).sort_values(ascending=True ).plot.barh(width=0.8,ax=ax1[1])
 ax1[1].set_title('Feature Importance in AdaBoost')
 model3 = GradientBoostingClassifier(criterion='friedman_mse', init=None,
learning_rate=0.5, loss='deviance', max_depth=2,
max_features=None, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=300,
presort='auto', random_state=23, subsample=1.0, verbose=0,
warm_start=False)
 model3.fit(X,y)
 pd.Series(model3.feature_importances_, X.columns ).sort_values(ascending=True ).plot.barh(width=0.8,ax=ax2[0])
 ax2[0].set_title('Feature Importance in Gradient Boosting')
 model4 = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bytree=1, gamma=0, learning_rate=0.3, max_delta_step=0,
max_depth=3, min_child_weight=1, missing=None, n_estimators=200,
n_jobs=1, nthread=None, objective='binary:logistic', random_state=23,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=True, subsample=1)
 model4.fit(X,y)
 pd.Series(model4.feature_importances_, X.columns ).sort_values(ascending=True ).plot.barh(width=0.8,ax=ax2[1])
 ax2[1].set_title('Feature Importance in XGBoost' )<load_from_csv>
def load_embeddings(path):
with open(path,'rb')as f:
return pickle.load(f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
def custom_loss(y_true, y_pred):
return binary_crossentropy(K.reshape(y_true[:,0],(-1,1)) , y_pred)* y_true[:,1]
def build_model(embedding_matrix, num_aux_targets, loss_weight):
words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x1 = TCN(TCN_UNITS, return_sequences=True, dilations = [1, 2, 4, 8, 16], name = 'tnc1_forward' )(x)
 x2 = Lambda(lambda z: K.reverse(z,axes=-1))(x)
 x2 = TCN(TCN_UNITS, return_sequences=True, dilations = [1, 2, 4, 8, 16],name = 'tnc1_backward' )(x2)
 x = add([x1,x2])
 x1 = TCN(TCN_UNITS, return_sequences=True, dilations = [1, 2, 4, 8, 16], name = 'tnc2_forward' )(x)
 x2 = Lambda(lambda z: K.reverse(z,axes=-1))(x)
 x2 = TCN(TCN_UNITS, return_sequences=True, dilations = [1, 2, 4, 8, 16],name = 'tnc2_backward' )(x2)
 x = add([x1,x2])
 hidden = concatenate([GlobalMaxPooling1D()(x),GlobalAveragePooling1D()(x)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss=[custom_loss,'binary_crossentropy'], loss_weights=[loss_weight, 1.0], optimizer='adam')
 return model
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data
test = pd.read_csv('.. /input/test.csv')
 test.head()<count_missing_values>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = preprocess(train['comment_text'])
 x_test = preprocess(test['comment_text'])
 identity_columns = ['male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish','muslim', 'black', 'white', 'psychiatric_or_mental_illness']
y_identities =(train[identity_columns] >= 0.5 ).astype(int ).values
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']].values
test.isnull().sum()<count_missing_values>
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN)
 embedding_matrix = np.concatenate([build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
 with open('temporary.pickle', mode='wb')as f:
pickle.dump(x_test, f)
 del identity_columns, weights, tokenizer, train, test, x_test
gc.collect()
test.isnull().sum()<feature_engineering>
tr_ind, val_ind = train_test_split(list(range(len(x_train))),test_size = 0.05, random_state = 23 )
test['NameTitle'] = test['Name'].apply(extract_title)
 fix_title(test)
 test.loc[test['Age'].isnull() & test['NameTitle'].str.contains('Master.'), 'Age'] = 5
test.loc[test['Age'].isnull() & test['NameTitle'].str.contains('Miss.'), 'Age'] = 22
test.loc[test['Age'].isnull() & test['NameTitle'].str.contains('Mr.'), 'Age'] = 32
test.loc[test['Age'].isnull() & test['NameTitle'].str.contains('Mrs.'), 'Age'] = 36
test.loc[test['Age'].isnull() & test['NameTitle'].str.contains('Other.'), 'Age'] = 46
test['Age'].isnull().any()
test['AgeGroup'] = test['Age'].apply(age_group)
 test['Fare'].fillna(test['Fare'].dropna().median() , inplace=True)
 test['FareGroup'] = test['Fare'].apply(fare_group)
 test['Embarked'] = test['Embarked'].map({'S': 0, 'Q': 1, 'C': 2} ).astype(int)
 test['Sex'] = test['Sex'].map({'male': 0, 'female': 1} ).astype(int)
 test['NameTitle'] = test['NameTitle'].map({'Mr.': 0, 'Other.': 1, 'Master.': 2, 'Miss.': 3, 'Mrs.': 4, 'Dona.': 1} ).astype(int )<feature_engineering>
checkpoint_predictions = []
checkpoint_val_preds = []
weights = []
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1],loss_weight)
 for global_epoch in range(EPOCHS):
es = EarlyStopping(patience = 1, verbose = True)
 model.fit(x_train[tr_ind],[y_train[tr_ind], y_aux_train[tr_ind]],validation_data =(x_train[val_ind],[y_train[val_ind], y_aux_train[val_ind]]),
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
callbacks=[
LearningRateScheduler(lambda epoch: 2e-3 *(0.6 ** global_epoch)) ,es
] 
)
 with open('temporary.pickle', mode='rb')as f:
x_test = pickle.load(f)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 checkpoint_val_preds.append(model.predict(x_train[val_ind], batch_size=2048)[0].flatten())
 del x_test
gc.collect()
weights.append(2 ** global_epoch)
 del model
gc.collect()
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
test['FamilySize'] = test['SibSp'] + test['Parch']
test.head()<drop_column>
val_preds = np.average(checkpoint_val_preds, weights=weights, axis=0 )
test_id = test[test.columns[0]]
test.drop('PassengerId', axis=1, inplace=True)
 test.drop('Name', axis=1, inplace=True)
 test.drop('Sex', axis=1, inplace=True)
 test.drop('Age', axis=1, inplace=True)
 test.drop('Ticket', axis=1, inplace=True)
 test.drop('Fare', axis=1, inplace=True)
 test.drop('Cabin', axis=1, inplace=True)
 test.drop('SibSp', axis=1, inplace=True)
 test.drop('Parch', axis=1, inplace=True)
 test.head()<predict_on_test>
def power_mean(x, p=-5):
return np.power(np.mean(np.power(x, p)) ,1/p)
 def get_s_auc(y_true,y_pred,y_identity):
mask = y_identity==1
s_auc = roc_auc_score(y_true[mask],y_pred[mask])
 return s_auc
def get_bpsn_auc(y_true,y_pred,y_identity):
mask =(y_identity==1)&(y_true==0)|(y_identity==0)&(y_true==1)
 bpsn_auc = roc_auc_score(y_true[mask],y_pred[mask])
 return bpsn_auc
def get_bspn_auc(y_true,y_pred,y_identity):
mask =(y_identity==1)&(y_true==1)|(y_identity==0)&(y_true==0)
 bspn_auc = roc_auc_score(y_true[mask],y_pred[mask])
 return bspn_auc
def get_total_auc(y_true,y_pred,y_identities):
N = y_identities.shape[1]
saucs = np.array([get_s_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 bpsns = np.array([get_bpsn_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 bspns = np.array([get_bspn_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 M_s_auc = power_mean(saucs)
 M_bpsns_auc = power_mean(bpsns)
 M_bspns_auc = power_mean(bspns)
 rauc = roc_auc_score(y_true,y_pred)
 total_auc = M_s_auc + M_bpsns_auc + M_bspns_auc + rauc
total_auc/= 4
return total_auc
prediction = model3.predict(test)
 submission = pd.DataFrame({
"PassengerId": test_id,
"Survived": prediction
} )<save_to_csv>
get_total_auc(y_train[val_ind][:,0],val_preds,y_identities[val_ind] )
submission.to_csv('submission.csv', index=False )<save_to_csv>
df_submit = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df_submit.prediction = predictions
df_submit.to_csv('submission.csv', index=False )