Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<data_type_conversions>
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
from pandas import DataFrame
import numpy as np
import json
for col in('BsmtFinSF1','TotalBsmtSF'):
all_data[col] = all_data[col].fillna(0 )<categorify>
!cp -r cased_L-12_H-768_A-12/* .
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )<categorify>
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, '')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )<drop_column>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 train.comment_text = preprocess(train['comment_text'])
 test.comment_text = preprocess(test['comment_text'])
 df_bert = pd.DataFrame({'id':train['id'],
'target':np.where(train.target >=.8,1,0),
'alpha':['a']*train.shape[0],
'comment_text':train["comment_text"].replace(r'
',' ',regex=True)})
 df_0 = df_bert[df_bert['target'] == 0].sample(1000, random_state = 101)
 df_1 = df_bert[df_bert['target'] == 1].sample(1000, random_state = 101)
 df_calibrated = pd.concat([df_0, df_1], axis=0 ).reset_index(drop=True)
 df_calibrated = shuffle(df_calibrated)
 df_bert_test = pd.DataFrame({'User_ID':test['id'],
'text':test['comment_text'].replace(r'
',' ',regex=True)})
 del(train)
 df_calibrated.to_csv('data/train.tsv', sep='\t', index=False, header=False)
 df_bert_test.to_csv('data/test.tsv', sep='\t', index=False, header=True )
all_data = all_data.drop(['Utilities'], axis=1 )<data_type_conversions>
!python.. /input/bertclassifier/repository/google-research-bert-0fce551/run_classifier.py \
--task_name=cola --do_train=True --do_predict=True \
--data_dir='data' \
--vocab_file='vocab.txt' \
--bert_config_file='bert_config.json' \
--init_checkpoint='bert_model.ckpt' \
--max_seq_length=220 --train_batch_size=8 --learning_rate=2e-5 --num_train_epochs=3.0 \
--output_dir='bertoutput' \
--do_lower_case=False
all_data["Functional"] = all_data["Functional"].fillna("Typ" )<feature_engineering>
result = pd.read_csv("bertoutput/test_results.tsv",sep="\t",header=None)
 preds_id = test['id'].values
preds = result.iloc[:,1].values
submission_file = pd.DataFrame({'id':preds_id,
'prediction':preds})
 submission_file.to_csv('submission.csv',sep=",",index=None )
<categorify><EOS> 
sub = pd.read_csv('submission.csv' )