Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<concatenate>
tqdm.pandas()
submission = pd.concat([titanic['PassengerId'], sub], axis=1 )<save_to_csv>
TEXT_COL = 'comment_text'
EMB_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv', index_col='id')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', index_col='id')
submission.to_csv('sub.csv', index=False )<load_from_csv>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 def load_embeddings(embed_dir=EMB_PATH):
embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in tqdm(open(embed_dir)))
 return embedding_index
def build_embedding_matrix(word_index, embeddings_index, max_features, lower = True, verbose = True):
embedding_matrix = np.zeros(( max_features, 300))
for word, i in tqdm(word_index.items() ,disable = not verbose):
if lower:
word = word.lower()
if i >= max_features: continue
try:
embedding_vector = embeddings_index[word]
except:
embedding_vector = embeddings_index["unknown"]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def build_matrix(word_index, embeddings_index):
embedding_matrix = np.zeros(( len(word_index)+ 1,300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embeddings_index[word]
except:
embedding_matrix[i] = embeddings_index["unknown"]
return embedding_matrix
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 test_df.head()<count_missing_values>
maxlen = 220
max_features = 100000
embed_size = 300
tokenizer = Tokenizer(num_words=max_features, lower=True)
 print('fitting tokenizer')
 tokenizer.fit_on_texts(list(train[TEXT_COL])+ list(test[TEXT_COL]))
word_index = tokenizer.word_index
X_train = tokenizer.texts_to_sequences(list(train[TEXT_COL]))
y_train = train['target'].values
X_test = tokenizer.texts_to_sequences(list(test[TEXT_COL]))
X_train = pad_sequences(X_train, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 del tokenizer
gc.collect()
train_df.isnull().sum() / len(train_df)*100<count_missing_values>
embeddings_index = load_embeddings()
test_df.isnull().sum() / len(test_df)*100<count_values>
embedding_matrix = build_matrix(word_index, embeddings_index )
train_df['Sex'].value_counts()<groupby>
del embeddings_index
gc.collect()
train_df.groupby('Sex',as_index=False ).Survived.mean()<sort_values>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
train_df[['Pclass','Survived']].groupby(['Pclass'],as_index=False ).mean().sort_values(by='Survived',ascending = False )<sort_values>
def build_model(verbose = False, compile = True):
sequence_input = L.Input(shape=(maxlen,), dtype='int32')
 embedding_layer = L.Embedding(len(word_index)+ 1,
300,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)
 x = embedding_layer(sequence_input)
 x = L.SpatialDropout1D(0.2 )(x)
 x = L.Bidirectional(L.CuDNNLSTM(64, return_sequences=True))(x)
 att = Attention(maxlen )(x)
 avg_pool1 = L.GlobalAveragePooling1D()(x)
 max_pool1 = L.GlobalMaxPooling1D()(x)
 x = L.concatenate([att,avg_pool1, max_pool1])
 preds = L.Dense(1, activation='sigmoid' )(x)
 model = Model(sequence_input, preds)
 if verbose:
model.summary()
if compile:
model.compile(loss='binary_crossentropy',optimizer=Adam(0.005),metrics=['acc'])
 return model
train_df[["Embarked", "Survived"]].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
splits = list(KFold(n_splits=5 ).split(X_train,y_train))
BATCH_SIZE = 2048
NUM_EPOCHS = 100
oof_preds = np.zeros(( X_train.shape[0]))
test_preds = np.zeros(( X_test.shape[0]))
for fold in [0,1,2,3,4]:
K.clear_session()
tr_ind, val_ind = splits[fold]
ckpt = ModelCheckpoint(f'gru_{fold}.hdf5', save_best_only = True)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)
 model = build_model()
model.fit(X_train[tr_ind],
y_train[tr_ind]>0.5,
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
validation_data=(X_train[val_ind], y_train[val_ind]>0.5),
callbacks = [es,ckpt])
 oof_preds[val_ind] += model.predict(X_train[val_ind])[:,0]
test_preds += model.predict(X_test)[:,0]
test_preds /= 5
train_df[['Parch','Survived']].groupby(['Parch'],as_index=False ).mean().sort_values(by='Survived',ascending=False )<sort_values>
roc_auc_score(y_train>0.5,oof_preds )
train_df[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<groupby>
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 submission['prediction'] = test_preds
submission.reset_index(drop=False, inplace=True)
 submission.head()
train_df.groupby('Sex',as_index='False')['Age'].median()<drop_column>
submission.to_csv('submission.csv', index=False )
<drop_column><EOS> 
submission.to_csv('submission.csv', index=False )