Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
log_reg = linear_model.LogisticRegression()
baseline_log_reg = log_reg.fit(X_train, y)
 predicted_survivors = baseline_log_reg.predict(X_test)
 print('Accuracy: ', metrics.accuracy_score(train.Survived, baseline_log_reg.predict(X_train)))
 sns.heatmap(metrics.confusion_matrix(train.Survived, baseline_log_reg.predict(X_train)) ,\
cmap="Blues", annot=True)
 plt.xlabel('Pred Label')
 plt.ylabel('True Label')
 test_baseline_model = test
test_baseline_model['Survived'] = predicted_survivors
test_baseline_model = test_baseline_model[['PassengerId', 'Survived']] 
<find_best_model_class>
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
from keras.losses import binary_crossentropy
from keras import backend as K
first_tree = tree.DecisionTreeClassifier()
first_tree_fit = first_tree.fit(X_train, y)
 predicted_survivors = first_tree_fit.predict(X_test)
 print('Accuracy: ', metrics.accuracy_score(train.Survived, first_tree_fit.predict(X_train)))
 sns.heatmap(metrics.confusion_matrix(train.Survived, first_tree_fit.predict(X_train)) , \
cmap="Blues", annot=True)
 plt.xlabel('Pred Label')
 plt.ylabel('True Label')
 test_first_model = test
test_first_model['Survived'] = predicted_survivors
test_first_model = test_first_model[['PassengerId', 'Survived']] 
<prepare_x_and_y>
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
X_train = train[['male', 'female', 'Pclass', 'new_Parch', 'new_SibSp']]
y = train['Survived']
test['new_SibSp'] = test.SibSp
test['new_SibSp'] = test.SibSp.astype(int)
 test.loc[train.new_SibSp > 1, 'new_SibSp'] = 2
test['new_Parch'] = test.Parch
test['new_Parch'] = test.new_Parch.astype(int)
 test.loc[train.new_Parch > 1, 'new_Parch'] = 2
X_test = test[['male', 'female', 'Pclass', 'new_Parch', 'new_SibSp']]<find_best_model_class>
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
second_tree = tree.DecisionTreeClassifier()
second_tree_fit = second_tree.fit(X_train, y)
 predicted_survivors = second_tree_fit.predict(X_test)
 print('Accuracy: ', metrics.accuracy_score(train.Survived, second_tree_fit.predict(X_train)))
 sns.heatmap(metrics.confusion_matrix(train.Survived, second_tree_fit.predict(X_train)) , \
cmap="Blues", annot=True)
 plt.xlabel('Pred Label')
 plt.ylabel('True Label')
 test_second_model = test
test_second_model['Survived'] = predicted_survivors
test_second_model = test_second_model[['PassengerId', 'Survived']] 
<import_modules>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32' )
model_names = [
'KNeighborsClassifier',
'SVC(kernel="linear")',
'RandomForestClassifier',
'AdaBoostClassifier() ',
'GaussianNB() ',
]
models = [
KNeighborsClassifier() ,
SVC(kernel="linear"),
RandomForestClassifier() ,
AdaBoostClassifier() ,
GaussianNB()
]
fit_models = []
for i in range(len(models)) :
temp_score = cross_val_score(models[i], X_train, y, cv=5)
 fit_models.append(models[i].fit(X_train,y))
print(model_names[i],' ', temp_score.mean())
 print('+/- ',temp_score.var() )<save_to_csv>
def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f )
test_rf_model = test
test_rf_model['Survived'] = fit_models[2].predict(X_test)
 test_rf_model = test_rf_model[['PassengerId', 'Survived']] 
<count_values>
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
print('S embarked survival %:',\
round(( train[train.Embarked == 'S']\ 
.Survived.sum() /train[train.Embarked == 'S'].Survived.count())*100., 3))
print('C embarked survival %:',\
round(( train[train.Embarked == 'C']\ 
.Survived.sum() /train[train.Embarked == 'C'].Survived.count())*100., 3))
print('Q embarked survival %:',\
round(( train[train.Embarked == 'Q']\ 
.Survived.sum() /train[train.Embarked == 'Q'].Survived.count())*100., 3))<filter>
def custom_loss(y_true, y_pred):
return binary_crossentropy(K.reshape(y_true[:,0],(-1,1)) , y_pred)* y_true[:,1]
train[train.Embarked.isnull() ]<categorify>
def build_model(embedding_matrix, num_aux_targets, loss_weight):
words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss=[custom_loss,'binary_crossentropy'], loss_weights=[loss_weight, 1.0], optimizer='adam')
 return model
train.loc[train.Embarked.isnull() , 'Embarked'] = 'S'
train = train.join(pd.get_dummies(train.Embarked))<categorify>
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data
test = test.join(pd.get_dummies(test.Embarked))<filter>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )
test[test.Fare.isnull() ]<feature_engineering>
x_train = preprocess(train['comment_text'])
 identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
test.loc[test.Fare.isnull() , 'Fare'] = test[(test.Pclass == 3)].Fare.median()
print(test[test.PassengerId == 1044] )<categorify>
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']].values
x_test = preprocess(test['comment_text'])
 tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN)
 embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )
train.loc[train.Age.isnull() , 'Age'] = -0.5
train = train.join(pd.get_dummies(pd.cut(train.Age, [-1,0,5, 12,18,35,60, 100],\
labels=['missing', 'infant', 'child', 'teen', 'youngAdult', 'adult', 'senior'])))
 train = train.join(pd.cut(train.Age, [-1,0,5, 12,18,35,60, 100],\
labels=['missing', 'infant', 'child', 'teen', 'youngAdult', 'adult', 'senior']),\
rsuffix='_class')
 test.loc[~test.Age.isnull() , 'Age'] = -0.5
test = test.join(pd.get_dummies(pd.cut(test.Age, [-1,0,5, 12,18,35,60, 100],\
labels=['missing', 'infant', 'child', 'teen', 'youngAdult', 'adult', 'senior'])) )<categorify>
with open('temporary.pickle', mode='wb')as f:
pickle.dump(x_test, f )
train = train.join(pd.get_dummies(train.new_SibSp, prefix='SibSp_'))
train = train.join(pd.get_dummies(train.new_Parch, prefix='Parch_'))<categorify>
del identity_columns, weights, tokenizer, train, test, x_test
gc.collect()
checkpoint_predictions = []
weights = []
test = test.join(pd.get_dummies(test.new_SibSp, prefix='SibSp_'))
test = test.join(pd.get_dummies(test.new_Parch, prefix='Parch_'))<define_variables>
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1], loss_weight)
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
callbacks=[
LearningRateScheduler(lambda epoch: 1e-3 *(0.6 ** global_epoch))
] 
)
 with open('temporary.pickle', mode='rb')as f:
x_test = pickle.load(f)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 del x_test
gc.collect()
weights.append(2 ** global_epoch)
 del model
gc.collect()
<prepare_x_and_y><EOS> 
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 df_submit = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df_submit.prediction = predictions
df_submit.to_csv('submission.csv', index=False )