Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
sgd = SGDClassifier(loss='log')
 sgd.fit(X_train, Y_train)
 Y_pred = sgd.predict(X_test)
 acc_sgd = round(sgd.score(X_train, Y_train)* 100, 2)
 acc_sgd
print('Test ACC Stochastic Gradient Descent', acc_sgd)
 X_pred = sgd.predict(X_train)
 X_predprob = sgd.predict_proba(X_train)[:,1]
t_sgd_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_sgd_score)
 t_sgd_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_sgd_roc)
 t_sgd_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_sgd_rmse )<compute_train_metric>
import numpy as np
import pandas as pd
import os
import time
import gc
import random
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
from keras.preprocessing import text, sequence
import torch
from torch import nn
from torch.utils import data
from torch.nn import functional as F
from fastai.train import Learner
from fastai.train import DataBunch
from fastai.callbacks import *
from fastai.basic_data import DatasetType
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 Y_pred = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2)
 acc_decision_tree
print('Test ACC Decision Tree', acc_decision_tree)
 X_pred = decision_tree.predict(X_train)
 X_predprob = decision_tree.predict_proba(X_train)[:,1]
t_dt_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_dt_score)
 t_dt_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_dt_roc)
 t_dt_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_dt_rmse )<compute_train_metric>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
bag_cla = BaggingClassifier()
bag_cla.fit(X_train, Y_train)
 y_pred=bag_cla.predict(X_test)
 acc_bag_cla = round(bag_cla.score(X_train, Y_train)* 100, 2)
 print('Test ACC Bagging Classifier', acc_bag_cla)
 X_pred = bag_cla.predict(X_train)
 X_predprob = bag_cla.predict_proba(X_train)[:,1]
t_bc_score = round(metrics.accuracy_score(Y_train, X_pred)*100,2)
 print('Score -- >', t_bc_score)
 t_bc_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_bc_roc)
 t_bc_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_bc_rmse)
<compute_train_metric>
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 Y_pred = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 acc_random_forest
print('Test ACC Random Forest', acc_random_forest)
 X_pred = random_forest.predict(X_train)
 X_predprob = random_forest.predict_proba(X_train)[:,1]
t_rm_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_rm_score)
 t_rm_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_rm_roc)
 t_rm_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_rm_rmse )<compute_train_metric>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
xgb = XGBClassifier(n_estimators=100)
 xgb.fit(X_train, Y_train)
 Y_pred_xgb=xgb.predict(X_test)
 xgb.score(X_train, Y_train)
 acc_xgb = round(xgb.score(X_train, Y_train)* 100, 2)
 acc_xgb
print('Test ACC XGBoost', acc_xgb)
 X_pred = xgb.predict(X_train)
 X_predprob = xgb.predict_proba(X_train)[:,1]
t_xgb_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_xgb_score)
 t_xgb_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_xgb_roc)
 t_xgb_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_xgb_rmse)
<predict_on_test>
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
xgb_tuned = XGBClassifier(
 learning_rate =0.1,
n_estimators=143,
max_depth=5,
min_child_weight=1,
gamma=0.0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27 
)
 xgb_tuned.fit(X_train, Y_train)
 Y_pred_tuned=xgb_tuned.predict(X_test)
 xgb_tuned.score(X_train, Y_train)
 acc_xgb_tuned = round(xgb_tuned.score(X_train, Y_train)* 100, 2)
 acc_xgb_tuned
print('Test ACC XGBoost Tunned', acc_xgb_tuned)
 X_pred = xgb_tuned.predict(X_train)
 X_predprob = xgb_tuned.predict_proba(X_train)[:,1]
t_xgbt_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_xgbt_score)
 t_xgbt_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_xgbt_roc)
 t_xgbt_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_xgbt_rmse )<predict_on_test>
def sigmoid(x):
return 1/(1 + np.exp(-x))
catb=CatBoostClassifier(iterations=2500, depth=5, learning_rate=0.3, verbose=0,
allow_writing_files=False, loss_function='CrossEntropy', random_strength=0.1, leaf_estimation_method='Gradient')
 catb.fit(X_train, Y_train)
 y_pred=catb.predict(X_test)
 acc_catb = round(catb.score(X_train, Y_train)* 100, 2)
 print('Test ACC CatBoost', acc_catb)
 X_pred = catb.predict(X_train)
 X_predprob = catb.predict_proba(X_train)[:,1]
t_catb_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_catb_score)
 t_catb_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_catb_roc)
 t_catb_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_catb_rmse )<choose_model_class>
def train_model(learn,test,output_dim,lr=0.00083,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds
clr = LogisticRegression()
csvc = SVC(probability=True)
 cknn = KNeighborsClassifier(n_neighbors = 3)
 cgau = GaussianNB()
cgb = GradientBoostingClassifier()
csgb = SGDClassifier(loss='log')
 crf = RandomForestClassifier(n_estimators=100)
 cxgbt = XGBClassifier(learning_rate =0.1, n_estimators=143, max_depth=5, min_child_weight=1, gamma=0.0, subsample=0.8,
colsample_bytree=0.8, objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27)
 cdt = DecisionTreeClassifier()
cbc = BaggingClassifier()
ccb = CatBoostClassifier(iterations=2500, depth=5, learning_rate=0.3, verbose=0,
allow_writing_files=False
, loss_function='CrossEntropy', random_strength=0.1, leaf_estimation_method='Gradient')

eclf1 = VotingClassifier(estimators=[('RandomForest', crf),('DecisionTree', cdt),('BaggClassifier', cbc),('CatBoost', ccb)], voting='soft')
 eclf1.fit(X_train, Y_train)
 y_pred=eclf1.predict(X_test)
 acc_eclf1 = round(eclf1.score(X_train.astype(float), Y_train.astype(float)).astype(float)* 100, 2)
 print('Test ACC Voting Classifier ', acc_eclf1)
 X_pred = eclf1.predict(X_train.astype(float))
X_predprob = eclf1.predict_proba(X_train)[:,1]
t_eclf1_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_eclf1_score)
 t_eclf1_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_eclf1_roc)
 t_eclf1_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_eclf1_rmse)
 eclf1.fit(X_train, Y_train)
 y_pred=catb.predict(X_test)
 acc_eclf1 = round(eclf1.score(X_train.astype(float), Y_train.astype(float)).astype(float)* 100, 2)
<compute_train_metric>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
for clf, label in zip([crf, cdt, cbc, ccb, eclf1], ['RandomForest', 'Decision Tree', 'Bagging Classifier',
'CatBoost', 'Voting Classifier']):
scores = cross_val_score(clf, X_train, Y_train, cv=5, scoring='accuracy')
 print("Accuracy: %0.2f(+/- %0.2f)[%s]" %(round(scores.mean() *100,2), scores.std() , label))<create_dataframe>
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron', 'Gradient Descent',
'Stochastic Gradient Descent', 'Linear SVC',
'Decision Tree', 'XgBoost', 'XgBoost_Tuned', 'Bagging Classifer', 'Cat Boost', 'Voting Classifier'],
'Score': [acc_svc, acc_knn, acc_log,
acc_random_forest, acc_gaussian, acc_perceptron,
acc_gd,acc_sgd, acc_linear_svc, acc_decision_tree, acc_xgb,acc_xgb_tuned, acc_bag_cla, acc_catb,
acc_eclf1],
'ROC': [t_svm_roc, t_knn_roc, t_lr_roc,
t_rm_roc, t_nb_roc, -100,
t_gd_roc,t_sgd_roc, -100, t_dt_roc, t_xgb_roc,t_xgbt_roc, t_bc_roc, t_catb_roc, t_eclf1_roc],
'RMSE': [t_svm_rmse, t_knn_rmse, t_lr_rmse,
t_rm_rmse, t_nb_rmse, 100,
t_gd_rmse,t_sgd_rmse, 100, t_dt_rmse, t_xgb_rmse,t_xgbt_rmse, t_bc_rmse, t_catb_rmse, t_eclf1_rmse]
})
 models.sort_values(by='RMSE', ascending=True )<save_to_csv>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = preprocess(train['comment_text'])
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'] )
X_pred = decision_tree.predict(X_train)
 submission = pd.DataFrame({
"PassengerId": data_test["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv(".. /working/submission_TitanicSurvived_pred_26Feb20_1757hr.csv", index=False)
 print('Validation Data Distribution:
', submission['Survived'].value_counts(normalize = True))
submission.sample(10)
 print("Not Normalized Counts")
 submission.Survived.value_counts()<set_options>
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn<load_from_csv>
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
max_features = None
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )<drop_column>
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
train_ID = train['PassengerId']
test_ID = test['PassengerId']
train = train.drop('PassengerId', axis=1)
 test = test.drop('PassengerId', axis=1 )<count_values>
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()
train['Sex'].value_counts()<count_values>
x_train_torch = torch.tensor(x_train, dtype=torch.long)
 y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32)
 x_test_torch = torch.tensor(x_test, dtype=torch.long)
 batch_size = 512
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 valid_dataset = data.TensorDataset(x_train_torch[:batch_size], y_train_torch[:batch_size])
 test_dataset = data.TensorDataset(x_test_torch)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 databunch = DataBunch(train_dl=train_loader,valid_dl=valid_loader )
train['Embarked'].value_counts()<prepare_x_and_y>
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch,model,loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.Survived.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['Survived'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))<sort_values>
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]})
 submission.to_csv('submission.csv', index=False )