Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<count_values>
import numpy as np
import pandas as pd
import re, string
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.utils.validation import check_X_y, check_is_fitted
from sklearn.linear_model import LogisticRegression
from sklearn.multioutput import ClassifierChain
from sklearn.pipeline import Pipeline
from scipy import sparse
type_counts=train_labels['Target'].value_counts()<create_dataframe>
class NbSvmClassifier(BaseEstimator, ClassifierMixin):
def __init__(self, C=1.0, dual=False, n_jobs=1):
self.C = C
self.dual = dual
self.n_jobs = n_jobs
self.classes_ = [0,1]
def predict(self, x):
check_is_fitted(self, ['_r', '_clf'])
 return self._clf.predict(x.multiply(self._r))
def predict_proba(self, x):
check_is_fitted(self, ['_r', '_clf'])
 return self._clf.predict_proba(x.multiply(self._r))
def fit(self, x, y):
x, y = check_X_y(x, y, accept_sparse=True)
 def pr(x, y_i, y):
p = x[y==y_i].sum(0)
 return(p+1)/(( y==y_i ).sum() +1)
 self._r = sparse.csr_matrix(np.log(pr(x,1,y)/ pr(x,0,y)))
 x_nb = x.multiply(self._r)
 self._clf = LogisticRegression(C=self.C, dual=self.dual, n_jobs=self.n_jobs ).fit(x_nb, y)
 return self
type_counts=pd.DataFrame(type_counts )<feature_engineering>
labels = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
type_counts['level']=type_counts.index<feature_engineering>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv' )
type_counts['level']=type_counts['level'].replace(poverty_mapping)
<feature_engineering>
train['comment_text'].fillna('unknown', inplace=True)
 test['comment_text'].fillna('unknown', inplace=True )
type_counts['Count']=type_counts['Target']<groupby>
def tokenize(s):
pattern = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 return pattern.sub(r' \1 ', s ).split()
all_equal=train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() ==1 )<filter>
vec = TfidfVectorizer(ngram_range=(1,2), tokenizer=tokenize,
min_df=3, max_df=0.9, strip_accents='unicode', use_idf=1,
smooth_idf=1, sublinear_tf=1)
 X_train = vec.fit_transform(train['comment_text'])
 X_test = vec.transform(test['comment_text'] )
not_equal= all_equal[all_equal!=True]<filter>
y_train = train[labels].values
example=train.loc[train['idhogar']==not_equal.index[0],['idhogar', 'parentesco1', 'Target']]<groupby>
chains = [ClassifierChain(NbSvmClassifier() , order='random', random_state=i)for i in range(len(labels)) ]
households_leader=train.groupby('idhogar')['parentesco1'].sum()<filter>
for chain in chains:
chain.fit(X_train, y_train )
households_leader_flase=train.loc[train['idhogar'].isin(households_leader
[households_leader !=1].index),:]<feature_engineering>
y_pred_chains = np.array([chain.predict_proba(X_test)for chain in chains] )
for household in not_equal.index:
true_target= int(train[(train['idhogar']==household)& 
(train['parentesco1']==1.0)]['Target'])
 train.loc[train['idhogar']==household, 'Target']=true_target<prepare_output>
y_pred_ensemble = y_pred_chains.mean(axis=0 )
missing = pd.DataFrame(data.isnull().sum() ).rename(columns={0:'total'})
 missing['Percent']=missing['total']/len(data)
 missing.sort_values('Percent',ascending=False ).head(10 )<groupby>
sub[labels] = y_pred_ensemble
heads.groupby('v18q')['v18q1'].apply(lambda x:x.isnull().sum() )<feature_engineering>
sub.to_csv('nbsvm_chain.csv', index=False )