Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<feature_engineering>
import numpy as np
import pandas as pd
import os
import time
import gc
import random
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
from keras.preprocessing import text, sequence
import torch
from torch import nn
from torch.utils import data
from torch.nn import functional as F
from sklearn.model_selection import train_test_split
train_df["Fare"] = train_df["Fare"].fillna(np.mean(train_df[train_df["Pclass"]==3]["Fare"]))
train_df[train_df["Fare"].isnull() ]<set_options>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
pd.set_option('mode.chained_assignment', None)
<load_from_csv>
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 1
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 224
gender_submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
 train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv' )<count_values>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
train_df.PassengerId[train_df.Cabin.notnull() ].count()<count_values>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def train_model(model, train,test, loss_fn, output_dim, lr=0.001,
batch_size=512, n_epochs=1,
enable_checkpoint_ensemble=True):
param_lrs = [{'params': param, 'lr': lr} for param in model.parameters() ]
optimizer = torch.optim.Adam(param_lrs, lr=lr)
 scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda epoch: 1 ** epoch)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
for epoch in range(n_epochs):
start_time = time.time()
scheduler.step()
model.train()
avg_loss = 0.
 for data in tqdm(train_loader, disable=False):
x_batch = data[:-1]
y_batch = data[-1]
y_pred = model(*x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
y_pred = sigmoid(model(*x_batch ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t triain_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, elapsed_time))
if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds
train_df.PassengerId[train_df.Age.notnull() ].count()<feature_engineering>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.5)
 self.lstm = nn.LSTM(embed_size,LSTM_UNITS,num_layers=2,dropout=0.2,bidirectional=True,batch_first=True)
 for name, param in self.lstm.named_parameters() :
if 'bias' in name:
torch.nn.init.constant_(param, 0.0)
 elif 'weight' in name:
torch.nn.init.orthogonal_(param)
 self.conv1=nn.Conv2d(1,3, kernel_size=3,stride=1,padding=1)
 self.maxpool1=nn.MaxPool2d(kernel_size=(56,4))
self.batch1=torch.nn.BatchNorm2d(3)
 self.droupout_out=torch.nn.Dropout(0.05)
 self.droupout_aux_out=torch.nn.Dropout(0.05)
 self.linear_out = nn.Linear(3*int(MAX_LEN/56)*int(LSTM_UNITS/2), 1)
 self.linear_aux_out = nn.Linear(3*int(MAX_LEN/56)*int(LSTM_UNITS/2), num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm, _ = self.lstm(h_embedding)
 addaxis_lstm=h_lstm.unsqueeze(1)
 conv_1=self.conv1(addaxis_lstm)
 max_pool1=F.relu(self.maxpool1(conv_1))
max_pool1_batch=self.batch1(max_pool1)
 flatten=torch.flatten(max_pool1_batch, start_dim=1)
 result = self.linear_out(flatten)
 aux_result = self.linear_aux_out(flatten)
 result = self.droupout_out(result)
 aux_result = self.droupout_aux_out(aux_result)
 out = torch.cat([result, aux_result], 1)
 return out
train_df.Age = train_df.Age.median()
train_df.Age<filter>
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data
train_df[train_df.Embarked.isnull() ]<groupby>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )
MaxPassEmbarked = train_df.groupby('Embarked' ).count() ['PassengerId']
train_df.Embarked[train_df.Embarked.isnull() ] = MaxPassEmbarked[MaxPassEmbarked == MaxPassEmbarked.max() ].index[0]<drop_column>
x_train = preprocess(train['comment_text'])
 y_train = np.where(train['target'] >= 0.5, 1, 0)
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'] )
train_df = train_df.drop(['PassengerId','Name','Ticket','Cabin'],axis=1)
 train_df<features_selection>
max_features = None
train_df.columns[train_df.isnull().any() ]<categorify>
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
label = LabelEncoder()
dicts = {}
label.fit(train_df.Sex.drop_duplicates())
 dicts['Sex'] = list(label.classes_)
 train_df.Sex = label.transform(train_df.Sex)
 label.fit(train_df.Embarked.drop_duplicates())
 dicts['Embarked'] = list(label.classes_)
 train_df.Embarked = label.transform(train_df.Embarked)
 train_df<categorify>
max_features = max_features or len(tokenizer.word_index)+ 1
max_features
test_df.Age[test_df.Age.isnull() ] = test_df.Age.mean()
test_df.Fare[test_df.Fare.isnull() ] = test_df.Fare.median()
MaxPassEmbarked = test_df.groupby('Embarked' ).count() ['PassengerId']
test_df.Embarked[test_df.Embarked.isnull() ] = MaxPassEmbarked[MaxPassEmbarked == MaxPassEmbarked.max() ].index[0]
result = pd.DataFrame(test_df.PassengerId)
 test_df = test_df.drop(['Name','Ticket','Cabin','PassengerId'],axis=1)
 label.fit(dicts['Sex'])
 test_df.Sex = label.transform(test_df.Sex)
 label.fit(dicts['Embarked'])
 test_df.Embarked = label.transform(test_df.Embarked )<prepare_x_and_y>
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
target = train_df.Survived
train_df = train_df.drop(['Survived'], axis=1)
 kfold = 5
itog_val = {}<split>
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
ROCtrainTRN, ROCtestTRN, ROCtrainTRG, ROCtestTRG = train_test_split(train_df, target, test_size=0.25 )<choose_model_class>
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()
model_rfc = RandomForestClassifier(n_estimators = 80, max_features='auto', criterion='entropy',max_depth=4)
 model_knc = KNeighborsClassifier(n_neighbors = 18)
 model_lr = LogisticRegression(penalty='l2', tol=0.01)
 model_svc = svm.SVC()<save_to_csv>
x_train_torch = torch.tensor(x_train, dtype=torch.long ).cuda()
x_test_torch = torch.tensor(x_test, dtype=torch.long ).cuda()
y_train_torch = torch.tensor(np.hstack([y_train[:, np.newaxis], y_aux_train]), dtype=torch.float32 ).cuda()
model_rfc.fit(train_df, target)
 result.insert(1,'Survived', model_rfc.predict(test_df))
result.to_csv('predictions.csv', index=False )<import_modules>
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 test_dataset = data.TensorDataset(x_test_torch)
 all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 model.cuda()
test_preds = train_model(model, train_dataset,test_dataset, output_dim=y_train_torch.shape[-1], batch_size=512,
n_epochs=4,loss_fn=nn.BCEWithLogitsLoss(reduction='mean'))
all_test_preds.append(test_preds)
 print()
<import_modules><EOS> 
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )