Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
for dataset in data_combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 data_train.head()<define_variables>
pd.set_option('display.max_colwidth', 300 )
guess_ages = np.zeros(( 2,3))
guess_ages<find_best_params>
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt']
for dataset in data_combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
 data_train.head()<sort_values>
NUM_MODELS = 2
BATCH_SIZE = 1024
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4*LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
data_train['AgeBand'] = pd.cut(data_train['Age'], 5)
 data_train[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True )<feature_engineering>
IDENTITY_COLUMNS = [
'transgender', 'female', 'homosexual_gay_or_lesbian', 'muslim', 'hindu',
'white', 'black', 'psychiatric_or_mental_illness', 'jewish']
for dataset in data_combine:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age']
data_train.head()<drop_column>
AUX_COLUMNS = ['target', 'severe_toxicity','obscene','identity_attack','insult','threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
data_train = data_train.drop(['AgeBand'], axis=1)
 data_combine = [data_train, data_test]
data_train.head()<sort_values>
contractions = joblib.load('.. /input/contractions/contraction_mapping' )
for dataset in data_combine:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
data_train[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<feature_engineering>
swear_words = pd.read_table('.. /input/swear-list/swear_wordlist.txt')
 swear_words.columns = ['words']
swear_words['words'] = swear_words['words'].map(lambda x: x.strip())
 swear_words = swear_words['words'].tolist()
for dataset in data_combine:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
data_train[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean()<drop_column>
replace_with_fuck = []
for swear in swear_words:
replace_with_fuck.append(swear)
 replace_with_fuck = '|'.join(replace_with_fuck )
data_train = data_train.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 data_test = data_test.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 data_combine = [data_train, data_test]
data_train.head()<feature_engineering>
def handle_swears(text):
text = re.sub(replace_with_fuck, ' fuck ', text)
 return text
def clean_sentence(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 text = handle_swears(text)
 wordnet_lemmatizer = WordNetLemmatizer()
stop = set(stopwords.words('english'))
VERB_CODES = {'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', }
ADJ_CODES = {'JJ','JJR','JJS'}
ADV_CODES = {'RB','RBR','RBS'}
NOUN_CODES = {'NN', 'NNP','NNS','NNPS'}
new_sentence = []
for i, word in enumerate(words):
if role[i][1] in VERB_CODES:
new_word = wordnet_lemmatizer.lemmatize(word, 'v')
 else:
if role[i][1] in ADJ_CODES:
new_word = wordnet_lemmatizer.lemmatize(word, 'a')
 else:
if role[i][1] in ADV_CODES:
new_word = wordnet_lemmatizer.lemmatize(word, 'r')
 else:
if role[i][1] in NOUN_CODES:
new_word = wordnet_lemmatizer.lemmatize(word, 'n')
 else:
new_word = wordnet_lemmatizer.lemmatize(word)
 if new_word not in stop and new_word.isalpha() and len(new_word)>1:
new_sentence.append(new_word)
 s = ' '.join(new_sentence)
 return s
def get_coefs(word, *arr):

 return word, np.asarray(arr,dtype='float32')
for dataset in data_combine:
dataset['Age*Class'] = dataset.Age * dataset.Pclass
data_train.loc[:, ['Age*Class', 'Age', 'Pclass']].head(10 )<sort_values>
def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f )
freq_port = data_train.Embarked.dropna().mode() [0]
for dataset in data_combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 data_train[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<categorify>
def build_matrix(word_index, path):

 embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
for dataset in data_combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 data_train.head()<data_type_conversions>
def build_model(embedding_matrix, num_aux_targets):

 words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights = [embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x)
 ])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result =Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs =words, outputs =[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
data_test['Fare'].fillna(data_test['Fare'].dropna().median() , inplace=True)
 data_test.head()<feature_engineering>
train_df = pd.read_csv('.. /input/cleaned/cleaned_train_df.csv')
 test_df = pd.read_csv('.. /input/cleaned/cleaned_test_df.csv' )
for dataset in data_combine:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 data_train = data_train.drop(['FareBand'], axis=1)
 data_combine = [data_train, data_test]<prepare_x_and_y>
X_train = data_train.drop("Survived", axis=1)
 Y_train = data_train["Survived"]
X_test = data_test.drop("PassengerId", axis=1 ).copy()
X_train.shape, Y_train.shape, X_test.shape<drop_column>
print(X_train.columns)
 print("
print(X_test.columns)
 print("
X_train.drop("Embarked", axis=1)
 X_test.drop("Embarked", axis=1 )<import_modules>
gc.collect()
gc.collect()
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn import metrics<compute_train_metric>
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str )
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2)
 acc_log
print('Test ACC Logistic Regression -- > ', acc_log)
 X_pred = logreg.predict(X_train)
 X_predprob = logreg.predict_proba(X_train)[:,1]
t_lr_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_lr_score)
 t_lr_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_lr_roc)
 t_lr_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_lr_rmse )<compute_train_metric>
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >=0.5, True, False )
svc = SVC(probability=True)
 svc.fit(X_train, Y_train)
 Y_pred = svc.predict(X_test)
 acc_svc = round(svc.score(X_train, Y_train)* 100, 2)
 acc_svc
print('Test ACC SVM -- > ', acc_svc)
 X_pred = svc.predict(X_train)
 X_predprob = svc.predict_proba(X_train)[:,1]
t_svm_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_svm_score)
 t_svm_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_svm_roc)
 t_svm_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_svm_rmse )<train_on_grid>
tokenizer = text.Tokenizer(filters = CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )
param_grid = {'C': [0.1, 1, 10, 100, 1000],
'gamma': [1, 0.1, 0.01, 0.001, 0.0001],
'kernel': ['linear', 'rbf']}
grid = LinearSVC(penalty='l2', loss='squared_hinge', dual=True, tol=0.0001, C=1.0, multi_class='ovr')
 grid.fit(X_train, Y_train)
 Y_pred = grid.predict(X_test)
 acc_svc1 = round(grid.score(X_train, Y_train)* 100, 2)
 print('Test ACC LinearSVC -- > ', acc_svc )<compute_train_metric>
x_train = sequence.pad_sequences(x_train,maxlen=MAX_LEN)
 x_test= sequence.pad_sequences(x_test, maxlen=MAX_LEN)
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 acc_knn
print('Test ACC KNN ', acc_knn)
 X_pred = knn.predict(X_train)
 X_predprob = knn.predict_proba(X_train)[:,1]
t_knn_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_knn_score)
 t_knn_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_knn_roc)
 t_knn_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_knn_rmse )<compute_train_metric>
sample_weights = np.ones(len(train_df), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN]*(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])*train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights/=sample_weights.mean()
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred = gaussian.predict(X_test)
 acc_gaussian = round(gaussian.score(X_train, Y_train)* 100, 2)
 acc_gaussian
print('Test ACC Naive ', acc_gaussian)
 X_pred = gaussian.predict(X_train)
 X_predprob = gaussian.predict_proba(X_train)[:,1]
t_nb_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_nb_score)
 t_nb_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_nb_roc)
 t_nb_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_nb_rmse )<compute_train_metric>
embedding_matrix = np.concatenate([build_matrix(tokenizer.word_index,f)for f in EMBEDDING_FILES], axis =-1)
 del train_df, tokenizer
gc.collect()
checkpoint_predictions = []
weights = []
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
 Y_pred = perceptron.predict(X_test)
 acc_perceptron = round(perceptron.score(X_train, Y_train)* 100, 2)
 acc_perceptron<compute_train_metric>
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks = [
LearningRateScheduler(lambda _: 1e-3*(0.55**global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=BATCH_SIZE)[0].flatten())
 weights.append(2 ** global_epoch)
 del model
gc.collect()
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred = linear_svc.predict(X_test)
 acc_linear_svc = round(linear_svc.score(X_train, Y_train)* 100, 2)
 acc_linear_svc<compute_train_metric>
gc.collect()
gd = GradientBoostingClassifier()
gd.fit(X_train, Y_train)
 Y_pred = gd.predict(X_test)
 acc_gd = round(gd.score(X_train, Y_train)* 100, 2)
 acc_gd
print('Test ACC Gradient Descent', acc_gd)
 X_pred = gd.predict(X_train)
 X_predprob = gd.predict_proba(X_train)[:,1]
t_gd_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_gd_score)
 t_gd_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_gd_roc)
 t_gd_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_gd_rmse )<compute_train_metric>
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions})
 submission.to_csv('submission.csv', index=False )