Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<feature_engineering>
%matplotlib inline
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 200
MAX_FEATURES = 120000
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
model_df['Age'] = model_df['Age'].astype(int)
 model_df.loc[ model_df['Age'] <= 11, 'Age'] = 0
model_df.loc[(model_df['Age'] > 11)&(model_df['Age'] <= 18), 'Age'] = 1
model_df.loc[(model_df['Age'] > 18)&(model_df['Age'] <= 22), 'Age'] = 2
model_df.loc[(model_df['Age'] > 22)&(model_df['Age'] <= 27), 'Age'] = 3
model_df.loc[(model_df['Age'] > 27)&(model_df['Age'] <= 33), 'Age'] = 4
model_df.loc[(model_df['Age'] > 33)&(model_df['Age'] <= 40), 'Age'] = 5
model_df.loc[(model_df['Age'] > 40)&(model_df['Age'] <= 66), 'Age'] = 6
model_df.loc[ model_df['Age'] > 66, 'Age'] = 6<feature_engineering>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
model_df['Fare'] = model_df['Fare'].astype(int)
 model_df.loc[ model_df['Fare'] <= 7.91, 'Fare'] = 0
model_df.loc[(model_df['Fare'] > 7.91)&(model_df['Fare'] <= 14.454), 'Fare'] = 1
model_df.loc[(model_df['Fare'] > 14.454)&(model_df['Fare'] <= 31), 'Fare'] = 2
model_df.loc[(model_df['Fare'] > 31)&(model_df['Fare'] <= 99), 'Fare'] = 3
model_df.loc[(model_df['Fare'] > 99)&(model_df['Fare'] <= 250), 'Fare'] = 4
model_df.loc[ model_df['Fare'] > 250, 'Fare'] = 5<drop_column>
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = train_df[TEXT_COLUMN].astype(str)
 y_aux_train = train_df[AUX_COLUMNS].values
y_train = train_df[TARGET_COLUMN].values
x_test = test_df[TEXT_COLUMN].astype(str )
model_df.drop(["SibSp","Parch","Name"],axis=1,inplace=True)
 model_df.head()<prepare_x_and_y>
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, True, False)
 tokenizer = text.Tokenizer(num_words=MAX_FEATURES, filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )
X_train=model_df[model_df.train==1]
X_test=model_df[model_df.train==0]
Y_train=y
X_train.drop("train",axis=1,inplace=True)
 X_test.drop("train",axis=1,inplace=True )<train_model>
x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
sgd = linear_model.SGDClassifier(max_iter=5, tol=None)
 sgd.fit(X_train, Y_train)
 acc_sgd = round(sgd.score(X_train, Y_train)* 100, 2 )<train_model>
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2 )<train_on_grid>
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
GBC = GradientBoostingClassifier()
GBC.fit(X_train, Y_train)
 acc_GBC = round(GBC.score(X_train, Y_train)* 100, 2 )<train_model>
checkpoint_predictions = []
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2 )<train_model>
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten() )
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2 )<train_on_grid>
predictions = np.average(checkpoint_predictions, axis=0 )
<train_model><EOS> 
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 submission['prediction'] = predictions
submission.to_csv('submission.csv', index=False )