Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<count_values>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
titanic['Embarked'].value_counts()<data_type_conversions>
%%time
! pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext".. /input/nvidiaapex/repository/NVIDIA-apex-39e153a
titanic['Embarked'] = titanic['Embarked'].fillna('S')
 titanic_t['Embarked'] = titanic_t['Embarked'].fillna('S' )<drop_column>
package_dir_a = ".. /input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT"
sys.path.insert(0, package_dir_a )
titanic = titanic.drop(['PassengerId', 'Ticket'], axis=1)
 titanic_t = titanic_t.drop(['PassengerId', 'Ticket'], axis=1 )<feature_engineering>
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function
import numpy as np
import pandas as pd
from typing import *
from pathlib import Path
import torch
import torch.optim as optim
from fastai import *
from fastai.text import *
from fastai.vision import *
from fastai.callbacks import *
from sklearn.model_selection import train_test_split
titanic['With_someone'] = titanic['SibSp'] | titanic['Parch']
titanic_t['With_someone'] = titanic_t['SibSp'] | titanic_t['Parch']<feature_engineering>
tqdm.pandas(desc="my bar!")
 warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore' )
titanic['With_someone'] = titanic['With_someone'].apply(lambda x:1 if x >=1 else 0)
 titanic_t['With_someone'] = titanic_t['With_someone'].apply(lambda x:1 if x >=1 else 0 )<feature_engineering>
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
WORK_DIR = '.. /working/'
convert_tf_checkpoint_to_pytorch.convert_tf_checkpoint_to_pytorch(BERT_MODEL_PATH + 'bert_model.ckpt',
BERT_MODEL_PATH + 'bert_config.json',
WORK_DIR + 'pytorch_model.bin')
 shutil.copyfile(BERT_MODEL_PATH + 'bert_config.json', WORK_DIR + 'bert_config.json' )
titanic['Title'] = titanic['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 titanic_t['Title'] = titanic_t['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 titanic['Title'].value_counts()<categorify>
warnings.filterwarnings(action='once')
 device = torch.device('cuda')
 MAX_SEQUENCE_LENGTH = 300
SEED = 620402
BATCH_SIZE = 16
BERT_MODEL_BIN = '.. /working'
bert_config = BertConfig(BERT_MODEL_PATH + 'bert_config.json')
 np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
title_mapping = {"Mr": 0, "Miss": 1, "Mrs": 2,
"Master": 3, "Dr": 3, "Rev": 3, "Col": 3, "Major": 3, "Mlle": 3,"Countess": 3,
"Ms": 3, "Lady": 3, "Jonkheer": 3, "Don": 3, "Dona" : 3, "Mme": 3,"Capt": 3,"Sir": 3 }
titanic['Title'] = titanic['Title'].map(title_mapping)
 titanic_t['Title'] = titanic_t['Title'].map(title_mapping )<drop_column>
bert_tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None, do_lower_case=True )
titanic = titanic.drop(['Name'], axis=1)
 titanic_t = titanic_t.drop(['Name'], axis=1 )<feature_engineering>
class FastAiBertTokenizer(BaseTokenizer):

 def __init__(self, tokenizer: BertTokenizer, max_seq_len: int=300, **kwargs):
self._pretrained_tokenizer = tokenizer
self.max_seq_len = max_seq_len
def __call__(self, *args, **kwargs):
return self
def tokenizer(self, t:str)-> List[str]:

 return ["[CLS]"] + self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2] + ["[SEP]"]
titanic['family members'] = titanic['SibSp'] + titanic['Parch'] + 1
titanic_t['family members'] = titanic_t['SibSp'] + titanic_t['Parch'] + 1<categorify>
fastai_tokenizer = Tokenizer(
 tok_func = FastAiBertTokenizer(bert_tokenizer, max_seq_len = MAX_SEQUENCE_LENGTH),
pre_rules=[],
post_rules=[]
)
titanic = pd.get_dummies(titanic, columns = ['Pclass', 'Sex', 'Embarked', 'Title'], drop_first = True)
 titanic_t = pd.get_dummies(titanic_t, columns = ['Pclass', 'Sex', 'Embarked', 'Title'], drop_first = True )<drop_column>
fastai_bert_vocab = Vocab(list(bert_tokenizer.vocab.keys()))
titanic = titanic.drop(['SibSp', 'Parch', 'Age'], axis=1)
 titanic_t = titanic_t.drop(['SibSp', 'Parch', 'Age'], axis=1 )<prepare_x_and_y>
%%time
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv" )
X = titanic.drop(['Survived'], axis=1)
 y = titanic['Survived']<prepare_x_and_y>
len_test = len(test_df )
X_t = titanic_t<normalization>
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data
X_scale = X[['Fare', 'family members']]
X_noscale = X.drop(['Fare', 'family members'], axis=1)
 X_scale_t = X_t[['Fare', 'family members']]
X_noscale_t = X_t.drop(['Fare', 'family members'], axis=1 )<normalization>
%%time
test_df['comment_text'] = preprocess(test_df['comment_text'] ).astype(str )
sc_X = MinMaxScaler()
X_scaled = sc_X.fit_transform(X_scale)
 X_scaled_t = sc_X.fit_transform(X_scale_t )<create_dataframe>
test = test_df
X_scaled = pd.DataFrame(X_scaled, columns=['Fare', 'family members'])
 X_scaled_t = pd.DataFrame(X_scaled_t, columns=['Fare', 'family members'] )<concatenate>
class BertTokenizeProcessor(TokenizeProcessor):
def __init__(self, tokenizer):
super().__init__(tokenizer=tokenizer, include_bos=False, include_eos=False)
 class BertNumericalizeProcessor(NumericalizeProcessor):
def __init__(self, *args, **kwargs):
super().__init__(*args, vocab=Vocab(list(bert_tokenizer.vocab.keys())) , **kwargs)
 def get_bert_processor(tokenizer:Tokenizer=None, vocab:Vocab=None):

 return [BertTokenizeProcessor(tokenizer=tokenizer),
NumericalizeProcessor(vocab=vocab)]
X = pd.concat([X_scaled, X_noscale], axis=1)
 X_t = pd.concat([X_scaled_t, X_noscale_t], axis=1 )<define_variables>
class BertDataBunch(TextDataBunch):
@classmethod
def from_df(cls, path:PathOrStr, train_df:DataFrame, valid_df:DataFrame, test_df:Optional[DataFrame]=None,
tokenizer:Tokenizer=None, vocab:Vocab=None, classes:Collection[str]=None, text_cols:IntsOrStrs=1,
label_cols:IntsOrStrs=0, label_delim:str=None, **kwargs)-> DataBunch:
"Create a `TextDataBunch` from DataFrames."
p_kwargs, kwargs = split_kwargs_by_func(kwargs, get_bert_processor)
 processor = get_bert_processor(tokenizer=tokenizer, vocab=vocab, **p_kwargs)
 if classes is None and is_listy(label_cols)and len(label_cols)> 1: classes = label_cols
src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),
TextList.from_df(valid_df, path, cols=text_cols, processor=processor))
src = src.label_for_lm() if cls==TextLMDataBunch else src.label_from_df(cols=label_cols, classes=classes)
 if test_df is not None: src.add_test(TextList.from_df(test_df, path, cols=text_cols))
return src.databunch(**kwargs )
k_range = [4]
weight_options = ['uniform']
norm = [1]
algo = ['ball_tree']<define_search_space>
bert_model = BertForSequenceClassification.from_pretrained(BERT_MODEL_BIN, num_labels=2)
 loss_func = nn.CrossEntropyLoss()
param_grid = dict(n_neighbors = k_range, weights = weight_options, p = norm, algorithm = algo)
 param_grid<import_modules>
learn = load_learner('.. /input/fastai-sigsaw-1-epoch', 'train_epoch_9.pkl' )
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV<choose_model_class>
test_list = TextList.from_df(test, cols=['comment_text'], path='.. /input/jigsaw-unintended-bias-in-toxicity-classification' )
knn = KNeighborsClassifier()<train_on_grid>
learn.data.add_test(test_list )
grid_knn = GridSearchCV(knn, param_grid, cv = 10, scoring='accuracy', return_train_score=False)
 grid_knn.fit(X, y )<find_best_params>
preds = learn.get_preds(DatasetType.Test )
grid_knn.best_params_<find_best_score>
preds = [learn.predict(test['comment_text'][i])for i in tqdm(range(len(test_df)))]
grid_knn.best_score_<choose_model_class>
preds_sub = [np.array(x[2][1])for x in tqdm(preds)]
forest_clf = RandomForestClassifier()<define_search_space>
submission_bert = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': preds_sub
})
 submission_bert.head()
param_grid = dict(n_estimators = [10], criterion = ['gini'], max_depth = [135, 140, 145] )<train_on_grid>
submission_bert.to_csv("submission.csv", index=False )