Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
data['Embarked'].fillna('S', inplace=True)
 data['Embarked'].isnull().sum()<feature_engineering>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv', index_col='id')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', index_col='id' )
def extract_title(x):
res = re.findall(r'[A-Za-z]+[.]', x)
 if res:
return res[0]
else:
return None
data['NameTitle'] = data['Name'].apply(extract_title)
 data.head(3 )<feature_engineering>
stopwords = set(STOPWORDS)
 def show_wordcloud(data, title = None):
wordcloud = WordCloud(
 background_color='white',
stopwords=stopwords,
max_words=50,
max_font_size=40,
scale=5,
random_state=1
).generate(str(data))
fig = plt.figure(1, figsize=(10,10))
plt.axis('off')
 if title:
fig.suptitle(title, fontsize=20)
 fig.subplots_adjust(top=2.3)
 plt.imshow(wordcloud)
 plt.show()
data.loc[data['Age'].isnull() & data['NameTitle'].str.contains('Master.'), 'Age'] = 5
data.loc[data['Age'].isnull() & data['NameTitle'].str.contains('Miss.'), 'Age'] = 22
data.loc[data['Age'].isnull() & data['NameTitle'].str.contains('Mr.'), 'Age'] = 32
data.loc[data['Age'].isnull() & data['NameTitle'].str.contains('Mrs.'), 'Age'] = 36
data.loc[data['Age'].isnull() & data['NameTitle'].str.contains('Other.'), 'Age'] = 46
data['Age'].isnull().any()<count_values>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
data['NameTitle'].value_counts()<drop_column>
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
from keras.losses import binary_crossentropy
from keras import backend as K
data.drop('PassengerId', axis=1, inplace=True)
 data.drop('Ticket', axis=1, inplace=True)
 data.drop('Cabin', axis=1, inplace=True)
 data.drop('Name', axis=1, inplace=True )<categorify>
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
def age_group(x):
if x <= 16:
return 4
elif x > 16 and x <= 32:
return 1
elif x > 32 and x <= 48:
return 2
elif x > 48 and x <= 64:
return 3
else:
return 0<feature_engineering>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
data['AgeGroup'] = data['Age'].apply(age_group)
 data.head()<drop_column>
def custom_loss(y_true, y_pred):
return binary_crossentropy(K.reshape(y_true[:,0],(-1,1)) , y_pred)* y_true[:,1]
data.drop('AgeBand', axis=1, inplace=True)
 data.drop('Age', axis=1, inplace=True)
 data.head()<categorify>
def build_model(embedding_matrix, num_aux_targets, loss_weight):
words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
Attention(MAX_LEN )(x),
GlobalMaxPooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss=[custom_loss,'binary_crossentropy'], loss_weights=[loss_weight, 1.0], optimizer='adam')
 return model
def fare_group(x):
if x <= 7.854:
return 0
elif x > 7.854 and x <= 10.5:
return 1
elif x > 10.5 and x <= 21.679:
return 2
elif x > 21.679 and x <= 39.688:
return 3
else:
return 4<feature_engineering>
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data
data['FareGroup'] = data['Fare'].apply(fare_group)
 data.head()<drop_column>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = preprocess(train['comment_text'] )
data.drop('Fare', axis=1, inplace=True)
 data.drop('FareBand', axis=1, inplace=True)
 data.head()<data_type_conversions>
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
data['Embarked'] = data['Embarked'].map({'S': 0, 'Q': 1, 'C': 2} ).astype(int)
 data['Sex'] = data['Sex'].map({'male': 0, 'female': 1} ).astype(int)
 data['NameTitle'] = data['NameTitle'].map({'Mr.': 0, 'Other.': 1, 'Master.': 2, 'Miss.': 3, 'Mrs.': 4} ).astype(int)
 data.head()<drop_column>
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']].values
x_test = preprocess(test['comment_text'] )
data.drop('Sex', axis=1, inplace=True)
 data.head()<feature_engineering>
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
data['FamilySize'] = data['SibSp'] + data['Parch']
data.head()<drop_column>
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )
data.drop('SibSp', axis=1, inplace=True)
 data.drop('Parch', axis=1, inplace=True )<split>
with open('temporary.pickle', mode='wb')as f:
pickle.dump(x_test, f)
 del identity_columns, weights, tokenizer, train, test, x_test
gc.collect()
checkpoint_predictions = []
weights = []
X = data[data.columns[1:]]
y = data[data.columns[0]]
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42 )<choose_model_class>
build_model(embedding_matrix, y_aux_train.shape[-1], loss_weight ).summary()
label = ['Linear SVM', 'Radial SVM', 'Decision Tree', 'Random Forest', 'Logistic Regression']
models = [svm.LinearSVC() , svm.SVC() , DecisionTreeClassifier() , RandomForestClassifier(n_estimators=100), LogisticRegression() ]
accuracy = []
fscore = []
for model in models:
model.fit(X_train, y_train)
 prediction = model.predict(X_val)
 accuracy.append(metrics.accuracy_score(prediction, y_val))
fscore.append(metrics.f1_score(y_val, prediction))
model_df = pd.DataFrame({
'accuracy': accuracy,
'fscore': fscore
}, index=label)
 model_df<compute_train_metric>
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1], loss_weight)
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
callbacks=[
LearningRateScheduler(lambda epoch: 1e-3 *(0.6 ** global_epoch))
] 
)
 with open('temporary.pickle', mode='rb')as f:
x_test = pickle.load(f)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 del x_test
gc.collect()
weights.append(2 ** global_epoch)
 del model
gc.collect()
cv_accuracy = []
cv_std = []
kfold = KFold(n_splits=10, random_state=23)
 for model in models:
cv_result = cross_val_score(model, X, y, cv=kfold)
 cv_accuracy.append(cv_result.mean())
 cv_std.append(cv_result.std())
 model_df['cv_accuracy'] = cv_accuracy
model_df['cv_std'] = cv_std
model_df<train_on_grid>
predictions = np.average(checkpoint_predictions, weights=weights, axis=0 )
<train_on_grid><EOS> 
df_submit = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df_submit.prediction = predictions
df_submit.to_csv('submission.csv', index=False )