Kaggle competition: Toxic Comment Classification Challenge
data['v18q1']=data['v18q1'].fillna(0 )<define_variables>
train = pd.read_csv(".. /input/train.csv" ).fillna('unknown')
 test = pd.read_csv(".. /input/test.csv" ).fillna('unknown' )
own_variables=[x for x in data if x.startswith('tipo')]<create_dataframe>
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s): return re_tok.sub(r' \1 ', s ).split()
owns=pd.DataFrame(data.loc[data['tipovivi3']==1,'Target'] )<count_values>
from scipy.sparse import hstack
from sklearn.pipeline import make_union
owns['Target'].value_counts()<feature_engineering>
repl = {
"yay!": " good ",
"yay": " good ",
"yaay": " good ",
"yaaay": " good ",
"yaaaay": " good ",
"yaaaaay": " good ",
":/": " bad ",
":&gt;": " sad ",
":')": " sad ",
":-(": " frown ",
":(": " frown ",
":s": " frown ",
":-s": " frown ",
"&lt;3": " heart ",
":d": " smile ",
":p": " smile ",
":dd": " smile ",
"8)": " smile ",
":-)": " smile ",
":)": " smile ",
";)": " smile ",
"(-:": " smile ",
"(:": " smile ",
":/": " worry ",
":&gt;": " angry ",
":')": " sad ",
":-(": " sad ",
":(": " sad ",
":s": " sad ",
":-s": " sad ",
r"\br\b": "are",
r"\bu\b": "you",
r"\bhaha\b": "ha",
r"\bhahaha\b": "ha",
r"\bdon't\b": "do not",
r"\bdoesn't\b": "does not",
r"\bdidn't\b": "did not",
r"\bhasn't\b": "has not",
r"\bhaven't\b": "have not",
r"\bhadn't\b": "had not",
r"\bwon't\b": "will not",
r"\bwouldn't\b": "would not",
r"\bcan't\b": "can not",
r"\bcannot\b": "can not",
r"\bi'm\b": "i am",
"m": "am",
"r": "are",
"u": "you",
"haha": "ha",
"hahaha": "ha",
"don't": "do not",
"doesn't": "does not",
"didn't": "did not",
"hasn't": "has not",
"haven't": "have not",
"hadn't": "had not",
"won't": "will not",
"wouldn't": "would not",
"can't": "can not",
"cannot": "can not",
"i'm": "i am",
"m": "am",
"i'll" : "i will",
"its" : "it is",
"it's" : "it is",
"'s" : " is",
"that's" : "that is",
"weren't" : "were not",
}
data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0 
<feature_engineering>
new_train_data = []
new_test_data = []
list_train = train['comment_text'].tolist()
list_test = test['comment_text'].tolist()
for i in list_train:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in repl.keys() :
j = repl[j]
xx = xx + j + " "
new_train_data.append(xx)
 for i in list_test:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in repl.keys() :
j = repl[j]
xx = xx + j + " "
new_test_data.append(xx)
 train["clean_comment_text"] = new_train_data
test["clean_comment_text"] = new_test_data
data['v2a1-missing'] = data['v2a1'].isnull() 
<count_values>
pattern = re.compile(r'[^a-zA-Z ?!]+')
 train_text = train["clean_comment_text"].tolist()
test_text = test["clean_comment_text"].tolist()
for i,c in enumerate(train_text):
train_text[i] = pattern.sub('',train_text[i].lower())
 for i,c in enumerate(test_text):
test_text[i] = pattern.sub('',test_text[i].lower() )
data['v2a1-missing'].value_counts()<data_type_conversions>
train['comment_text'] = train_text
test["comment_text"] = test_text
del train_text, test_text
train.drop(['clean_comment_text'], inplace = True, axis = 1)
 test.drop(['clean_comment_text'], inplace = True, axis = 1 )
data['v2a1'] = data['v2a1'].fillna(value=data['tipovivi3'] )<feature_engineering>
all_text = pd.concat([train['comment_text'],test['comment_text']] )
data.loc[(( data['age']>19)|(data['age']<7)) &(data['rez_esc'].isnull()), 'rez_esc']=0<feature_engineering>
word_vectorizer = TfidfVectorizer(ngram_range =(1,3),
tokenizer=tokenize,
min_df=3, max_df=0.9,
strip_accents='unicode',
stop_words = 'english',
analyzer = 'word',
use_idf=1,
smooth_idf=1,
sublinear_tf=1 )
data['rez_esc-missing'] = data['rez_esc'].isnull()<feature_engineering>
char_vectorizer = TfidfVectorizer(ngram_range =(1,4),
min_df=3, max_df=0.9,
strip_accents='unicode',
analyzer = 'char',
stop_words = 'english',
use_idf=1,
smooth_idf=1,
sublinear_tf=1,
max_features=50000 )
data['rez_esc']=data['rez_esc'].fillna(0 )<feature_engineering>
vectorizer = make_union(word_vectorizer, char_vectorizer )
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5<define_variables>
vectorizer.fit(all_text )
id_ = ['Id', 'idhogar', 'Target']<define_variables>
train_matrix =vectorizer.transform(train['comment_text'])
 test_matrix = vectorizer.transform(test['comment_text'] )
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']<define_variables>
train_matrix,test_matrix
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']<define_variables>
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error, accuracy_score
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']<define_variables>
val_score = []
def cross_validation(model,y_train):
score = cross_val_score(model,train_matrix,y_train,scoring='accuracy',cv=5)
 val_score.append(score.mean() )
heads = data.loc[data['parentesco1'] == 1, :]
heads = heads[id_ + hh_bool + hh_cont + hh_ordered]
heads.shape<feature_engineering>
class_names = ['toxic','severe_toxic','obscene','threat','insult','identity_hate']
model = MultinomialNB()
for clas in class_names:
print(clas)
 cross_validation(model,train[clas] )
elec = []
for i, row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull() 
<groupby>
val_score = []
LR_model = LogisticRegression(C=3, dual=True)
 for clas in class_names:
print(clas)
 cross_validation(LR_model,train[clas] )
heads = heads.drop(columns = 'area2')
 heads.groupby('area1')['Target'].value_counts(normalize = True )<feature_engineering>
LR_score = val_score
LR_score
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
<feature_engineering>
DF_score = pd.DataFrame(index=class_names)
 DF_score['NB'] = NB_score
DF_score['LR'] = LR_score
DF_score
heads['walls+roof+floor'] = heads['walls'] + heads['roof'] + heads['floor'] 
<feature_engineering>
def pr(y_i, y):
p = train_matrix[y==y_i].sum(0)
 return(p+1)/(( y==y_i ).sum() +1 )
heads['warning'] = 1 *(heads['sanitario1'] + 
(heads['elec'] == 0)+
heads['pisonotiene'] +
heads['abastaguano'] + 
(heads['cielorazo'] == 0))<feature_engineering>
def get_mdl(y):
y = y.values
r = np.log(pr(1,y)/ pr(0,y))
m = LogisticRegression(C=3, dual=True)
 x_nb = train_matrix.multiply(r)
 return m.fit(x_nb, y), r
heads['bonus'] = 1 *(heads['refrig'] +
heads['computer'] + 
(heads['v18q1'] > 0)+
heads['television'])
<feature_engineering>
model = LogisticRegression(C=3,dual = True)
 NBLR_score=[]
for clas in class_names:
print(clas)
 y = train[clas].values
r = np.log(pr(1,y)/ pr(0,y))
x_nb = train_matrix.multiply(r)
 score = cross_val_score(model,x_nb,y,scoring='accuracy',cv=5)
 NBLR_score.append(score.mean() )
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']<feature_engineering>
DF_score = pd.DataFrame(index=class_names)
 DF_score['NB'] = NB_score
DF_score['LR'] = LR_score
DF_score['NBLR'] = NBLR_score
DF_score
heads.loc[(heads.v14a == 1)&(heads.sanitario1 == 1)&(heads.abastaguano == 0), "v14a"] = 0
heads.loc[(heads.v14a == 1)&(heads.sanitario1 == 1)&(heads.abastaguano == 0), "sanitario1"] = 0<groupby>
preds = np.zeros(( len(test), len(class_names)))
 for i, j in enumerate(class_names):
print('fit', j)
 m,r = get_mdl(train[j])
 preds[:,i] = m.predict_proba(test_matrix.multiply(r)) [:,1]
<rename_columns><EOS> 
submid = pd.DataFrame({'id': test["id"]})
 submission = pd.concat([submid, pd.DataFrame(preds, columns = class_names)], axis=1)
 submission.to_csv('submission_1.csv', index=False )