Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<create_dataframe>
warnings.filterwarnings(action='once')
 device = torch.device('cuda' )
final_df = df7.copy()<feature_engineering>
def convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"])+[0] *(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens )
test_df = test_df.dropna(subset=['Embarked'])
 test_df = test_df.drop('Cabin',axis=1)
 test_df['Title'] = test_df['Name'].str.replace(r' (.*,)|(\.. *)','')
 test_df = test_df.drop(['Name','Ticket'],axis=1)
 test_df['Title'] = test_df['Title'].apply(lambda x: 'others' if x not in ['Mr','Mrs','Miss','Master'] else x)
 def age_miss(df):
if np.isnan(df['Age']):
if df['Pclass']==1:
return 38
elif df['Pclass']==2:
return 29
elif df['Pclass']==3:
return 25
else:
return df['Age']
test_df['Age'] = test_df.apply(age_miss,axis=1 )<prepare_x_and_y>
MAX_SEQUENCE_LENGTH = 220
SEED = 1234
BATCH_SIZE = 32
BERT_MODEL_PATH = '.. /input/bertpretrainedmodels/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
features = ['Pclass', 'Sex','Age','Embarked', 'Title']
X = final_df[features]
y = final_df.Survived
test = test_df[features]<split>
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=101 )<import_modules>
bert_config = BertConfig('.. /input/bertjson/bert_config_new.json')
 tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True )
from sklearn.pipeline import make_pipeline,Pipeline
from sklearn.compose import make_column_transformer
from sklearn.preprocessing import OneHotEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import GridSearchCV<categorify>
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv")
 test_df['comment_text'] = test_df['comment_text'].astype(str)
 X_test = convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), MAX_SEQUENCE_LENGTH, tokenizer )
ohe = OneHotEncoder()
params = [{'classifier':[RandomForestClassifier() ],
'classifier__n_estimators':[10,25,50,75,100],
'classifier__max_depth':[5,10,15,20]},
{'classifier':[LogisticRegression() ],
'classifier__penalty':['l1','l2'],
'classifier__C':[0.01,0.1,1,10,100,1000],
'classifier__max_iter':[10,100,1000]},
{'classifier':[KNeighborsClassifier() ],
'classifier__n_neighbors':[2,5,10]}]
preprocessing = make_column_transformer(( ohe,['Pclass', 'Sex','Embarked', 'Title']),remainder='passthrough')
 pipe = Pipeline([('preprocessing',preprocessing), 
('classifier',RandomForestClassifier())] )<train_on_grid>
model = BertForSequenceClassification(bert_config, num_labels=16)
 model.load_state_dict(torch.load(".. /input/bert2epaux01wwd1e4decay95weachrebasev1/bert_pytorch.bin"))
model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
gr = GridSearchCV(pipe,param_grid=params,cv=5 ).fit(X_train,y_train )<find_best_params>
test_preds = np.zeros(( len(X_test)))
 test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)
 tk0 = tqdm(test_loader)
 for i,(x_batch,)in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 test_preds[i * 32:(i + 1)* 32] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()
gr.best_params_<predict_on_test>
from IPython.display import HTML
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.utils.validation import check_X_y, check_is_fitted
from sklearn.linear_model import LogisticRegression
from scipy import sparse
import re
import string
submit = pd.DataFrame(gr.predict(test),index=test_df.index,columns=['Survived'] )<save_to_csv>
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )
submit.to_csv('Submission.csv',index=False )<load_from_csv>
text = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s): return text.sub(r' \1 ', s ).split()
length = train_df.shape[0]
Vectorize = TfidfVectorizer(ngram_range=(1,2), tokenizer=tokenize,
min_df=3, max_df=0.9, strip_accents='unicode', use_idf=1,
smooth_idf=1, sublinear_tf=1 )
target = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')['target']
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 ssub = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
<import_modules>
train = Vectorize.fit_transform(train_df["comment_text"])
 test = Vectorize.transform(test_df["comment_text"] )
import pandas_profiling as pp<feature_engineering>
y = np.where(train_df['target'] >= 0.5, 1, 0 )
train['hashtags']=train['text'].apply(lambda x:re.findall('<feature_engineering>
class NbSvmClassifier(BaseEstimator, ClassifierMixin):
def __init__(self, C=1.0, dual=False, n_jobs=1):
self.C = C
self.dual = dual
self.n_jobs = n_jobs
def predict(self, x):
check_is_fitted(self, ['_r', '_clf'])
 return self._clf.predict(x.multiply(self._r))
def predict_proba(self, x):
check_is_fitted(self, ['_r', '_clf'])
 return self._clf.predict_proba(x.multiply(self._r))
def fit(self, x, y):
y = y
x, y = check_X_y(x, y, accept_sparse=True)
 def pr(x, y_i, y):
p = x[y==y_i].sum(0)
 return(p+1)/(( y==y_i ).sum() +1)
 self._r = sparse.csr_matrix(np.log(pr(x,1,y)/ pr(x,0,y)))
 x_nb = x.multiply(self._r)
 self._clf = LogisticRegression(C=self.C, dual=self.dual, n_jobs=self.n_jobs ).fit(x_nb, y)
 return self
labels=['Negative','Positive']
no_clusters=2
for c in range(2):
print('Target:-',labels[c])
 hts=list(train[train['target']==c]['hashtags'])
 hashes=[]
for ht in hts:
for h in ht:
hashes.append(h.strip())
 string_hash=' '.join(hashes)
 hash_values=pd.Series(hashes ).value_counts()
hval=hash_values.reset_index()
d = {}
for a, x in hval.values:
d[a] = x
wordcloud = WordCloud(max_font_size=40)
 wordcloud.generate_from_frequencies(frequencies=d)
 plt.figure(figsize=(70,70))
plt.imshow(wordcloud, interpolation="bilinear")
 plt.axis("off")
 plt.show()<categorify>
NbSvm = NbSvmClassifier(C=1.5, dual=True, n_jobs=-1)
 NbSvm.fit(train, y )
embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder/3")
 X_train_embeddings = embed(train.text.values)
 X_test_embeddings = embed(test.text.values )<prepare_x_and_y>
prediction=NbSvm.predict_proba(test)[:,1]
<set_options><EOS> 
submission = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv")
 submission['prediction'] = 0.2*prediction + 0.3*test_pred
submission.to_csv('submission.csv', index=False )