Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<train_on_grid>
import numpy as np
import pandas as pd
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 acc_linear_svc = round(linear_svc.score(X_train, Y_train)* 100, 2 )<train_model>
from sklearn.model_selection import train_test_split
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2 )<prepare_output>
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
NUM_MODELS = 4
BATCH_SIZE = 330
LSTM_UNITS = 120
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 3
MAX_LEN = 210
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
results = pd.DataFrame({
'Model': ['Support Vector Machines','GBC', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron',
'Stochastic Gradient Decent',
'Decision Tree'],
'Score': [acc_linear_svc,acc_GBC, acc_knn, acc_log,
acc_random_forest, acc_gaussian, acc_perceptron,
acc_sgd, acc_decision_tree]})
 result_df = results.sort_values(by='Score', ascending=False)
 result_df = result_df.set_index('Score')
 result_df.head(10 )<compute_train_metric>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32' )
rf = RandomForestClassifier(n_estimators=100)
 scores = cross_val_score(rf, X_train, Y_train, cv=10, scoring = "accuracy")
 print("Scores:", scores)
 print("Mean:", scores.mean())
 print("Standard Deviation:", scores.std() )<choose_model_class>
def load_embeddings(path):
with open(path,'r', encoding='UTF-8')as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f )
random_forest = RandomForestClassifier(criterion = "gini",
min_samples_leaf = 1,
min_samples_split = 10,
n_estimators=100,
max_features='auto',
oob_score= True,
random_state=1,
n_jobs=-1)
 random_forest.fit(X_train, Y_train)
 print("oob score:", round(random_forest.score(X_train,Y_train), 4)*100, "%" )<compute_train_metric>
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
predictions = cross_val_predict(random_forest, X_train, Y_train, cv=10)
 confusion_matrix(Y_train, predictions )<compute_test_metric>
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )
print("Precision:", precision_score(Y_train, predictions))
print("Recall:",recall_score(Y_train, predictions))<predict_on_test>
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str )
y_scores = random_forest.predict_proba(X_train)
 y_scores = y_scores[:,1]<compute_test_metric>
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.55, True, False )
r_a_score = roc_auc_score(Y_train, y_scores)
 print("ROC-AUC-Score:", r_a_score )<predict_on_test>
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
predictionss=random_forest.predict(X_test )<load_from_csv>
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
test_df=pd.read_csv("/kaggle/input/titanic/test.csv")
 submission =pd.DataFrame({'PassengerId':test_df.PassengerId,'Survived':predictionss})
 submission.head()<save_to_csv>
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN] *(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])* train_df[IDENTITY_COLUMNS].sum(axis=1)* 5
sample_weights /= sample_weights.mean()
submission.to_csv("submission.csv",index=False )<set_options>
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
plt.style.use('ggplot')
 warnings.filterwarnings('ignore')
 plt.rc('font', size=18)
 plt.rc('axes', titlesize=22)
 plt.rc('axes', labelsize=18)
 plt.rc('xtick', labelsize=12)
 plt.rc('ytick', labelsize=12)
 plt.rc('legend', fontsize=12)
 plt.rcParams['font.sans-serif'] = ['Verdana']
def format_1000(value, tick_number):
return int(value / 1_000)
 pd.options.mode.chained_assignment = None
pd.options.display.max_seq_items = 500
pd.options.display.max_rows = 500
pd.set_option('display.float_format', lambda x: '%.5f' % x)
 local = False
debug = False
if(local):
BASE_PATH = "./"
else:
BASE_PATH = ".. /input/titanic/" 
<load_from_csv>
X, X_t, y, y_t , ya, ya_t, sw, sw_t = train_test_split(x_train, y_train , y_aux_train ,sample_weights, test_size=0.18, random_state=42 )
df = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId' )<create_dataframe>
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
 return model
missing = [(c, df[c].isna().mean() *100)for c in df]
missing = pd.DataFrame(missing, columns=["column_name", "percentage"])
 missing = missing[missing.percentage > 0]
display(missing.sort_values("percentage", ascending=False))<feature_engineering>
model = build_model(embedding_matrix, ya.shape[-1] )
def cabin_location(df):
df['Cabin'].unique()
df['Cabin'].fillna('Unknown',inplace = True)
 df['Cabin_Location'] = df['Cabin'].str[0]
df['Cabin_Location'] = np.where(( df.Pclass==1)&(df['Cabin_Location']=='U'),'C',
np.where(( df.Pclass==2)&(df['Cabin_Location']=='U'),'D',
np.where(( df.Pclass==3)&(df['Cabin_Location']=='U'),'G',
np.where(df['Cabin_Location']=='T','C',df['Cabin_Location']))))
return df
def title_extract(df):
name_splice = df['Name'].str.split(pat=', ',n=-1,expand=True)
 df['Title'] = name_splice[1].str.split(pat='.',n=-1,expand=True)[0]
title_correction_map = {'Mme':'Mrs','Mlle':'Mss','Miss':'Mss', 'Don':'Mr','Donna':'Mrs',
'Lady':'Mrs','Sir':'Mr'}
title_list = {'Mr','Mrs','Mss','Dr','Master','Rev'}
df['Title'].replace(title_list,inplace=True)
 df['Title'] = df['Title'].apply(lambda x:'Other' if(not any(x in title for title in title_list)) else x)
 return df
def age_fill(df):
df_train = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId')
 df_total = pd.concat([df_train, df_test] ).copy()
df_total = title_extract(df_total)
 df = title_extract(df)
 mean_age_by_title = np.round(df_total.groupby('Title' ).Age.mean() ).astype('int64')
 df['Age'] = df.groupby('Title')['Age'].apply(lambda x: x.fillna(mean_age_by_title[x.name]))
return df
def fare_fill(df):
df_train = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId')
 df_total = pd.concat([df_train, df_test] ).copy()
mean_fare_by_class = df_total.groupby('Pclass' ).Fare.mean()
df['Fare'] = df.groupby('Pclass')['Fare'].apply(lambda x: x.fillna(mean_fare_by_class[x.name]))
df['is_free'] = np.where(df['Fare']== 0,1,0)
 df['Fare'] = np.where(( df.Pclass==1)&(df['Fare']== 0),mean_fare_by_class[1],
np.where(( df.Pclass==2)&(df['Fare']== 0),mean_fare_by_class[2],
np.where(( df.Pclass==3)&(df['Fare']== 0),mean_fare_by_class[3],
df['Fare'])))
 return df
def embarked_fill(df):
df_train = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId')
 df_total = pd.concat([df_train, df_test] ).copy()
mode_embarked = df_total['Embarked'].mode()
df['Embarked'] = df['Embarked'].fillna(mode_embarked[0])
 return df
def totally_cleaned(df):
df_copy = df.copy()
df_copy = cabin_location(df)
 df_copy = title_extract(df)
 df_copy = age_fill(df)
 df_copy = fare_fill(df)
 df_copy = embarked_fill(df)
 return df_copy 
<feature_engineering>
checkpoint_predictions = []
def is_female(df):
df['is_female']=df['Sex'].apply(lambda x: 1 if x=='female' else 0)
 return df
def traveling_party(df):
df['traveling_party']=df['Parch'] + df['SibSp'] + 1
df['is_mother'] = np.where(((df['Sex']=='female')&(df['Parch'] > 0)&(df['Age'] > 30.)),1,0)
 df['is_wife'] = np.where(((df['Sex']=='female')&(df['Title'] == 'Mrs')&(df['Age'] > 30.)),1,0)
 df['is_alone'] = np.where(df['traveling_party'] == 1,1,0)
 return df
def bin_data(df,columns,nbins,log=False):
frames = []
for cut_col in columns:
if(log):
tmp = pd.DataFrame(pd.cut(np.log1p(feat[cut_col]), bins=nbins, labels=np.arange(0,nbins)))
 else:
tmp = pd.DataFrame(pd.cut(feat[cut_col], bins=nbins, labels=np.arange(0,nbins)))
 tmp.columns = [cut_col + "_binned"]
frames.append(tmp)
 binned = pd.concat(frames, axis=1 ).astype(int)
 df = pd.concat([df, binned], axis=1)
 return df
def ticket_pre(df):
ticket_prefix = []
for i in list(df.Ticket):
if not i.isdigit() :
ticket_prefix.append(( i.replace(".","" ).replace("/","" ).strip().split(' ')[0] ).upper())
 else:
ticket_prefix.append("X")
 df["Ticket_Pre"] = ticket_prefix
return df
def feature_engineering(df):
df = is_female(df)
 df = traveling_party(df)
 df = ticket_pre(df)
 return df 
<load_from_csv>
for global_model in range(NUM_MODELS):
model.fit(X, [y, ya], validation_data=[X_t, [y_t,ya_t]],
batch_size=BATCH_SIZE, epochs= 2, verbose=2,sample_weight=[sw.values, np.ones_like(sw)],
callbacks=[LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_model)) ])
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten() )
def catboost_encode(df,columns):
df_train = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId')
 df_total = pd.concat([df_train, df_test] ).copy()
df_total = totally_cleaned(df_total)
 df_total = feature_engineering(df_total)
 cb_enc = ce.CatBoostEncoder(cols=columns)
 count_encoded = cb_enc.fit(df_total[columns],df_total['Survived'])
 df = df.join(cb_enc.transform(df[columns] ).add_suffix('_cb'))
return df
def onehot_encode(df,columns):
df_obj = df[columns].copy()
cols = []
for c in df_obj:
dummies = pd.get_dummies(df_obj[c])
 dummies.columns = [c + "_" + str(x)for x in dummies.columns]
cols.append(dummies)
 df_obj = pd.concat(cols, axis=1)
 df = pd.concat([df, df_obj], axis=1)
 df.reset_index(inplace=True, drop=True)
 return df
def get_dummy(df,columns):
for col in columns:
df = pd.get_dummies(df, columns = [col], prefix = col)
 return df
def drop_catagotical(df):
catagorical_cols = [col for col in df.columns if df[col].dtype == "object"]
df = df.drop(catagorical_cols,axis=1)
 return df 
<load_from_csv>
predictions = np.average(checkpoint_predictions, axis=0 )
<categorify><EOS> 
submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )