Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
total = all_data.isnull().sum().sort_values(ascending=False)
 percent =(all_data.isnull().sum() /all_data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data<drop_column>
import numpy as np
import pandas as pd
from keras.models import Model
from keras.layers import Input, Dense, LSTM, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNLSTM, CuDNNGRU, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.optimizers import Adam, RMSprop, SGD
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
import sys, os, re, csv, codecs
from keras.preprocessing.text import Tokenizer
all_data.drop(['Cabin'], axis=1, inplace=True )<count_values>
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt']
all_data['SibSp'].loc[np.isnan(all_data['Age'])].value_counts()<groupby>
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
all_data["Age"] = all_data.loc[all_data["SibSp"]>1].groupby("SibSp")["Age"].transform(
 lambda x: x.fillna(x.median()))<groupby>
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
all_data["Age"] = all_data.groupby("Pclass")["Age"].transform(
 lambda x: x.fillna(x.median()))<count_values>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32' )
all_data["Embarked"].value_counts()<feature_engineering>
def load_embedding(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f )
all_data["Embarked"] = all_data["Embarked"].fillna("S" )<groupby>
def build_matrix(word_index, path):
embedding_index = load_embedding(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
all_data["Fare"] = all_data.groupby("Pclass")["Fare"].transform(
 lambda x: x.fillna(x.median()))<feature_engineering>
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
all_data["TotalRelatives"] = all_data['SibSp'] + all_data['Parch']
all_data['IsAlone'] = 1
all_data['IsAlone'].loc[all_data["TotalRelatives"] > 0] = 0
all_data['Title'] = all_data['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]
all_data['FareBin'] = pd.qcut(all_data['Fare'], 4)
 all_data['AgeBin'] = pd.cut(all_data['Age'].astype(int), 5 )<categorify>
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )
final_features = pd.get_dummies(all_data ).reset_index(drop=True)
 final_features.shape<split>
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str )
train = final_features[:ntrain]
test = final_features[ntrain:]<compute_train_metric>
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, True, False )
n_folds = 5
def score_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 score = cross_val_score(model, train.values, y_train, cv = kf)
 return("score: {:.4f}({:.4f})".format(score.mean() , score.std()))<train_on_grid>
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
params = {'logisticregression__C' : [0.001,0.01,0.1,1,10,100,1000]}
pipe = make_pipeline(RobustScaler() , LogisticRegression())
 gridsearch_logistic = GridSearchCV(pipe, params, cv=10)
 gridsearch_logistic.fit(train, y_train)
 print("Meilleurs parametres: ", gridsearch_logistic.best_params_ )<find_best_params>
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen = MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen = MAX_LEN )
score_cv(gridsearch_logistic.best_estimator_ )<train_on_grid>
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN] *(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])* train_df[IDENTITY_COLUMNS].sum(axis=1)* 5
sample_weights /= sample_weights.mean()
params = {'kneighborsclassifier__n_neighbors' : [3,4,5,6,7],
'kneighborsclassifier__weights' : ['uniform','distance'],
'kneighborsclassifier__algorithm' : ['auto', 'ball_tree', 'kd_tree', 'brute']}
pipe = make_pipeline(RobustScaler() , KNeighborsClassifier())
 gridsearch_KNC = GridSearchCV(pipe, params, cv=5)
 gridsearch_KNC.fit(train, y_train)
 print("Meilleurs parametres: ", gridsearch_KNC.best_params_ )<find_best_params>
embedding_matrix = np.concatenate([build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
 checkpoint_predictions = []
weights = []
score_cv(gridsearch_KNC.best_estimator_ )<compute_train_metric>
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 predictions = np.average(checkpoint_predictions, weights=weights, axis=0 )
<train_model><EOS> 
submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions})
 submission.to_csv('submission.csv', index=False )