Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<feature_engineering>
tqdm.pandas()
test['Survived'] = [1 if(x == 'female')else 0 for x in test['Sex']]
test.loc[(test.Boy == 1), 'Survived'] = 1
test.loc[(( test.Pclass == 3)&(test.Embarked == 'S')) , 'Survived'] = 0
test.loc[(( test.Pclass == 3)&(test.Embarked == 'S')&(test.Boy == 1)&(test.Family > 0)&(test.Family < 4)) , 'Survived'] = 1<feature_engineering>
TEXT_COL = 'comment_text'
EMB_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv', index_col='id')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', index_col='id' )
test['Survived'] = [1 if(x == 'female')else 0 for x in test['Sex']]
test.loc[(test.Boy == 1), 'Survived'] = 1
test.loc[(( test.Pclass == 3)&(test.Embarked == 'S')& ~(( test.Boy == 1)&(test.Family > 0)&(test.Family < 4))), 'Survived'] = 0<define_variables>
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
def convert_to_bool(df, col_name):
df[col_name] = np.where(df[col_name] >= 0.5, True, False)
 def convert_dataframe_to_bool(df):
bool_df = df.copy()
for col in ['target'] + identity_columns:
convert_to_bool(bool_df, col)
 return bool_df
train = convert_dataframe_to_bool(train )
start_time = time.time() 
<install_modules>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 def load_embeddings(embed_dir=EMB_PATH):
embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in tqdm(open(embed_dir)))
 return embedding_index
def build_embedding_matrix(word_index, embeddings_index, max_features, lower = True, verbose = True):
embedding_matrix = np.zeros(( max_features, 300))
for word, i in tqdm(word_index.items() ,disable = not verbose):
if lower:
word = word.lower()
if i >= max_features: continue
try:
embedding_vector = embeddings_index[word]
except:
embedding_vector = embeddings_index["unknown"]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def build_matrix(word_index, embeddings_index):
embedding_matrix = np.zeros(( len(word_index)+ 1,300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embeddings_index[word]
except:
embedding_matrix[i] = embeddings_index["unknown"]
return embedding_matrix
!pip install tensorflow-text==2.0.0 --user<import_modules>
maxlen = 220
max_features = 100000
embed_size = 300
tokenizer = Tokenizer(num_words=max_features, lower=True)
 print('fitting tokenizer')
 tokenizer.fit_on_texts(list(train[TEXT_COL])+ list(test[TEXT_COL]))
word_index = tokenizer.word_index
X_train = tokenizer.texts_to_sequences(list(train[TEXT_COL]))
y_train = train['target'].values
X_test = tokenizer.texts_to_sequences(list(test[TEXT_COL]))
X_train = pad_sequences(X_train, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 del tokenizer
gc.collect()
import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_text as textb<set_options>
embeddings_index = load_embeddings()
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 310 )<load_from_csv>
embedding_matrix = build_matrix(word_index, embeddings_index )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 train_copy = train.copy()
length_train = len(train.index)
 length_train<prepare_x_and_y>
del embeddings_index
gc.collect()
index_hot = [4415, 4400, 4399,4403,4397,4396, 4394,4414, 4393,4392,4404,4407,4420,4412,4408,4391,4405]
train_example = train.copy()
train_example.loc[index_hot,:]<prepare_output>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
index_hot = [6840,6834,6837,6841,6816]
train_example = train.copy()
train_example.loc[index_hot,:]<split>
def build_model(verbose = False, compile = True):
sequence_input = L.Input(shape=(maxlen,), dtype='int32')
 embedding_layer = L.Embedding(len(word_index)+ 1,
300,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)
 x = embedding_layer(sequence_input)
 x = L.SpatialDropout1D(0.2 )(x)
 x = L.Bidirectional(L.CuDNNLSTM(64, return_sequences=True))(x)
 att = Attention(maxlen )(x)
 avg_pool1 = L.GlobalAveragePooling1D()(x)
 max_pool1 = L.GlobalMaxPooling1D()(x)
 x = L.concatenate([att,avg_pool1, max_pool1])
 preds = L.Dense(1, activation='sigmoid' )(x)
 model = Model(sequence_input, preds)
 if verbose:
model.summary()
if compile:
model.compile(loss='binary_crossentropy',optimizer=Adam(0.005),metrics=['acc'])
 return model
index_hot = [6828,6831]
train_example = train.copy()
train_example.loc[index_hot,:]<train_model>
SUBGROUP_AUC = 'subgroup_auc'
BPSN_AUC = 'bpsn_auc'
BNSP_AUC = 'bnsp_auc'
TOXICITY_COLUMN = 'target'
def compute_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except ValueError:
return np.nan
def compute_subgroup_auc(df, subgroup, label, model_name):
subgroup_examples = df[df[subgroup]]
return compute_auc(subgroup_examples[label], subgroup_examples[model_name])
 def compute_bpsn_auc(df, subgroup, label, model_name):

 subgroup_negative_examples = df[df[subgroup] & ~df[label]]
non_subgroup_positive_examples = df[~df[subgroup] & df[label]]
examples = subgroup_negative_examples.append(non_subgroup_positive_examples)
 return compute_auc(examples[label], examples[model_name])
 def compute_bnsp_auc(df, subgroup, label, model_name):

 subgroup_positive_examples = df[df[subgroup] & df[label]]
non_subgroup_negative_examples = df[~df[subgroup] & ~df[label]]
examples = subgroup_positive_examples.append(non_subgroup_negative_examples)
 return compute_auc(examples[label], examples[model_name])
 def compute_bias_metrics_for_model(dataset,
subgroups,
model,
label_col,
include_asegs=False):

 records = []
for subgroup in subgroups:
record = {
'subgroup': subgroup,
'subgroup_size': len(dataset[dataset[subgroup]])
 }
record[SUBGROUP_AUC] = compute_subgroup_auc(dataset, subgroup, label_col, model)
 record[BPSN_AUC] = compute_bpsn_auc(dataset, subgroup, label_col, model)
 record[BNSP_AUC] = compute_bnsp_auc(dataset, subgroup, label_col, model)
 records.append(record)
 return pd.DataFrame(records ).sort_values('subgroup_auc', ascending=True )
index_hot = [591, 587]
train_example = train.copy()
train_example.loc[index_hot,:]<split>
def calculate_overall_auc(df, model_name):
true_labels = df[TOXICITY_COLUMN]
predicted_labels = df[model_name]
return metrics.roc_auc_score(true_labels, predicted_labels)
 def power_mean(series, p):
total = sum(np.power(series, p))
return np.power(total / len(series), 1 / p)
 def get_final_metric(bias_df, overall_auc, POWER=-5, OVERALL_MODEL_WEIGHT=0.25):
bias_score = np.average([
power_mean(bias_df[SUBGROUP_AUC], POWER),
power_mean(bias_df[BPSN_AUC], POWER),
power_mean(bias_df[BNSP_AUC], POWER)
 ])
 return(OVERALL_MODEL_WEIGHT * overall_auc)+(( 1 - OVERALL_MODEL_WEIGHT)* bias_score )
index_hot = [601,576,584,608,606,603,592,604]
train_example = train.copy()
train_example.loc[index_hot,:]<prepare_output>
pd.options.mode.chained_assignment = None
class CompetitionMetricEarlyStopping(EarlyStopping):
def __init__(self, model, df, X_valid, *args, **kwargs):
kwargs['monitor'] = 'val_competition_metric'
self.model = model
self.df = df
self.X_valid = X_valid
self.bias_metrics_df = None
super().__init__(*args, **kwargs)
 def get_monitor_value(self, logs):
preds = self.model.predict(self.X_valid, batch_size=2048)[:, 0]
self.df['predictions'] = preds
bias_metrics_df = compute_bias_metrics_for_model(self.df, identity_columns, 'predictions', TOXICITY_COLUMN)
 self.bias_metrics_df = bias_metrics_df
val_competition_metric = get_final_metric(bias_metrics_df, calculate_overall_auc(self.df, 'predictions'))
logs['val_competition_metric'] = val_competition_metric
return val_competition_metric
index_hot = [3667,3674,3688,3696]
train_example = train.copy()
train_example.loc[index_hot,:]<prepare_output>
splits = list(KFold(n_splits=5 ).split(X_train,y_train))
BATCH_SIZE = 2048
NUM_EPOCHS = 100
oof_preds = np.zeros(( X_train.shape[0]))
test_preds = np.zeros(( X_test.shape[0]))
for fold in [0,1,2,3,4]:
K.clear_session()
tr_ind, val_ind = splits[fold]
model = build_model()
es = CompetitionMetricEarlyStopping(model, train.iloc[val_ind], X_train[val_ind],
restore_best_weights=False,
mode='max',
verbose=1,
patience=2)
 ckpt = ModelCheckpoint(f'gru_{fold}.hdf5', save_best_only = True, monitor='val_competition_metric', mode='max', verbose=1)
 model.fit(X_train[tr_ind],
y_train[tr_ind]>0.5,
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
validation_data=(X_train[val_ind], y_train[val_ind]>0.5),
callbacks = [es, ckpt])
 model.load_weights(f'gru_{fold}.hdf5')
 print(es.bias_metrics_df)
 oof_preds[val_ind] += model.predict(X_train[val_ind])[:,0]
test_preds += model.predict(X_test)[:,0]
test_preds /= 5
index_hot = [3913,3914,3936,3921,3941,3937,3938]
train_example = train.copy()
train_example.loc[index_hot,:]<split>
train['predictions'] = oof_preds
bias_metrics_df = compute_bias_metrics_for_model(train, identity_columns, 'predictions', TOXICITY_COLUMN)
 print(bias_metrics_df)
 off_competition_metric = get_final_metric(bias_metrics_df, calculate_overall_auc(train, 'predictions')) ; off_competition_metric
index_hot = [3136, 3133]
train_example = train.copy()
train_example.loc[index_hot,:]<prepare_output>
roc_auc_score(y_train>0.5,oof_preds )
index_hot = [3930,3933,3924,3917]
train_example = train.copy()
train_example.loc[index_hot,:]<split>
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 submission['prediction'] = test_preds
submission.reset_index(drop=False, inplace=True)
 submission.head()
index_hot = [246,270,266,259,253,251,250,271]
train_example = train.copy()
train_example.loc[index_hot,:]<categorify>
submission.to_csv('submission.csv', index=False )
<categorify><EOS> 
submission.to_csv('submission.csv', index=False )