Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<count_values>
tqdm.pandas()
train.Survived.value_counts()<count_missing_values>
TEXT_COL = 'comment_text'
EMB_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv', index_col='id')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', index_col='id')
train.isna().sum()<feature_engineering>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 def load_embeddings(embed_dir=EMB_PATH):
embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in tqdm(open(embed_dir)))
 return embedding_index
def build_embedding_matrix(word_index, embeddings_index, max_features, lower = True, verbose = True):
embedding_matrix = np.zeros(( max_features, 300))
for word, i in tqdm(word_index.items() ,disable = not verbose):
if lower:
word = word.lower()
if i >= max_features: continue
try:
embedding_vector = embeddings_index[word]
except:
embedding_vector = embeddings_index["unknown"]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def build_matrix(word_index, embeddings_index):
embedding_matrix = np.zeros(( len(word_index)+ 1,300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embeddings_index[word]
except:
embedding_matrix[i] = embeddings_index["unknown"]
return embedding_matrix
train.loc[train.Age.isna() , 'Age'] = train[~train.Age.isna() ].Age.mean()<feature_engineering>
maxlen = 220
max_features = 100000
embed_size = 300
tokenizer = Tokenizer(num_words=max_features, lower=True)
 print('fitting tokenizer')
 tokenizer.fit_on_texts(list(train[TEXT_COL])+ list(test[TEXT_COL]))
word_index = tokenizer.word_index
X_train = tokenizer.texts_to_sequences(list(train[TEXT_COL]))
y_train = train['target'].values
X_test = tokenizer.texts_to_sequences(list(test[TEXT_COL]))
X_train = pad_sequences(X_train, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 del tokenizer
gc.collect()
train.loc[train.Cabin.isna() ,'Cabin'] = "No Cabin"<count_values>
embeddings_index = load_embeddings()
print(train.Embarked.value_counts())
 train.loc[train.Embarked.isna() ,'Embarked'] = "S"<count_missing_values>
embedding_matrix = build_matrix(word_index, embeddings_index )
train.isna().sum()<filter>
del embeddings_index
gc.collect()
train.loc[train.Fare > 200]<count_values>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
train.Sex.value_counts()<categorify>
def build_model(verbose = False, compile = True):
sequence_input = L.Input(shape=(maxlen,), dtype='int32')
 embedding_layer = L.Embedding(len(word_index)+ 1,
300,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)
 x = embedding_layer(sequence_input)
 x = L.SpatialDropout1D(0.2 )(x)
 x = L.Bidirectional(L.CuDNNLSTM(64, return_sequences=True))(x)
 att = Attention(maxlen )(x)
 avg_pool1 = L.GlobalAveragePooling1D()(x)
 max_pool1 = L.GlobalMaxPooling1D()(x)
 x = L.concatenate([att,avg_pool1, max_pool1])
 preds = L.Dense(1, activation='sigmoid' )(x)
 model = Model(sequence_input, preds)
 if verbose:
model.summary()
if compile:
model.compile(loss='binary_crossentropy',optimizer=Adam(0.005),metrics=['acc'])
 return model
labelencoder = LabelEncoder()
train['Sex'] = labelencoder.fit_transform(train['Sex'])
 train.Sex.value_counts()<data_type_conversions>
splits = list(KFold(n_splits=5 ).split(X_train,y_train))
BATCH_SIZE = 2048
NUM_EPOCHS = 100
oof_preds = np.zeros(( X_train.shape[0]))
test_preds = np.zeros(( X_test.shape[0]))
for fold in [0,1,2,3,4]:
K.clear_session()
tr_ind, val_ind = splits[fold]
ckpt = ModelCheckpoint(f'gru_{fold}.hdf5', save_best_only = True)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)
 model = build_model()
model.fit(X_train[tr_ind],
y_train[tr_ind]>0.5,
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
validation_data=(X_train[val_ind], y_train[val_ind]>0.5),
callbacks = [es,ckpt])
 oof_preds[val_ind] += model.predict(X_train[val_ind])[:,0]
test_preds += model.predict(X_test)[:,0]
test_preds /= 5
def features_engineering(df):
df.loc[df.Age.isna() , 'Age'] = df[~df.Age.isna() ].Age.mean()
df.loc[df.Cabin.isna() ,'Cabin'] = "No Cabin"
df.loc[df.Embarked.isna() ,'Embarked'] = "S"
df['persons_abroad_size'] =(df['Parch']+df['SibSp'] ).astype(int)
 df['alone'] = np.where(df['Parch']==0,1,0)
 df['Embarked'] = df['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 df['Sex'] = df['Sex'].map({'male': 1, 'female': 2} ).astype(int)
 df['log_fare'] = df['Fare'].apply(np.log)
 df['Room'] =(df['Cabin'] 
.str.slice(1,5 ).str.extract('([0-9]+)', expand=False)
.fillna(0)
.astype(int))
df['RoomBand'] = 0
df.loc[(df.Room > 0)&(df.Room <= 20), 'RoomBand'] = 1
df.loc[(df.Room > 20)&(df.Room <= 40), 'RoomBand'] = 2
df.loc[(df.Room > 40)&(df.Room <= 80), 'RoomBand'] = 3
df.loc[df.Room > 80, 'RoomBand'] = 4
df_id = df.PassengerId
df = df.drop('PassengerId', axis=1)
 return df,df_id<load_from_csv>
roc_auc_score(y_train>0.5,oof_preds )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train,train_id = features_engineering(train)
 test,test_id = features_engineering(test )<prepare_x_and_y>
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 submission['prediction'] = test_preds
submission.reset_index(drop=False, inplace=True)
 submission.head()
X_train = train.drop('Survived',axis=1 ).select_dtypes(include=['int32','int64','float64'])
 y_train = train['Survived']
X_test = test.select_dtypes(include=['int32','int64','float64'])
 xg_boost = xgb.XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bytree=0.65, gamma=2, learning_rate=0.3, max_delta_step=1,
max_depth=4, min_child_weight=2, missing=None, n_estimators=280,
n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=True, subsample=1 )<train_model>
submission.to_csv('submission.csv', index=False )
<compute_train_metric><EOS> 
submission.to_csv('submission.csv', index=False )