Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<import_modules>
start_real = datetime.now()
np.random.seed(123 )
<load_from_csv><EOS> 
os.environ['MKL_NUM_THREADS'] = '4'
os.environ['OMP_NUM_THREADS'] = '4'
os.environ['JOBLIB_START_METHOD'] = 'forkserver'
INPUT_PATH = r'.. /input'
class ItemSelector(BaseEstimator, TransformerMixin):
def __init__(self, field, start_time=time.time()):
self.field = field
self.start_time = start_time
def fit(self, x, y=None):
return self
def transform(self, dataframe):
print(f'[{time.time() -self.start_time}] select {self.field}')
 dt = dataframe[self.field].dtype
if is_categorical_dtype(dt):
return dataframe[self.field].cat.codes[:, None]
elif is_numeric_dtype(dt):
return dataframe[self.field][:, None]
else:
return dataframe[self.field]
class DropColumnsByDf(BaseEstimator, TransformerMixin):
def __init__(self, min_df=1, max_df=1.0):
self.min_df = min_df
self.max_df = max_df
def fit(self, X, y=None):
m = X.tocsc()
self.nnz_cols =(( m != 0 ).sum(axis=0)>= self.min_df ).A1
if self.max_df < 1.0:
max_df = m.shape[0] * self.max_df
self.nnz_cols = self.nnz_cols &(( m != 0 ).sum(axis=0)<= max_df ).A1
return self
def transform(self, X, y=None):
m = X.tocsc()
return m[:, self.nnz_cols]
def get_rmsle(y_true, y_pred):
return np.sqrt(mean_squared_log_error(np.expm1(y_true), np.expm1(y_pred)))
 def split_sub(text):
return re.sub('[^0-9'a-zA-Z]+', ' ', text)
 def preprocess_pandas(train, test, start_time=time.time()):
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
print('Train shape : ', train.shape)
 nrow_train = train.shape[0]
y_train = train[list_classes]
merge: pd.DataFrame = pd.concat([train, test])
 del train
del test
gc.collect()
merge['comment_text'] = merge['comment_text'] \ 
.fillna('')\ 
.str.lower() \ 
.apply(lambda x: split_sub(x))
print(f'[{time.time() - start_time}] Missing filled.')
 return merge, y_train, nrow_train
def intersect_drop_columns(train: csr_matrix, valid: csr_matrix, min_df=0):
t = train.tocsc()
v = valid.tocsc()
nnz_train =(( t != 0 ).sum(axis=0)>= min_df ).A1
nnz_valid =(( v != 0 ).sum(axis=0)>= min_df ).A1
nnz_cols = nnz_train & nnz_valid
res = t[:, nnz_cols], v[:, nnz_cols]
return res
def wordCount(text):
try:
if text == 'No description yet':
return 0
else:
text = text.lower()
words = [w for w in text.split(" ")]
return len(words)
 except:
return 0
mp.set_start_method('forkserver', True)
 start_time = time.time()
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 print(f'[{time.time() - start_time}] Finished to load data')
 print('Train shape: ', train.shape)
 print('Test shape: ', test.shape)
 train['char_len'] = train['comment_text'].apply(lambda x: len(x))
test['char_len'] = test['comment_text'].apply(lambda x: len(x))
train['desc_len'] = train['comment_text'].apply(lambda x: wordCount(x))
test['desc_len'] = test['comment_text'].apply(lambda x: wordCount(x))
repl = {
"&lt;3": " good ",
":d": " good ",
":dd": " good ",
":p": " good ",
"8)": " good ",
":-)": " good ",
":)": " good ",
";)": " good ",
"(-:": " good ",
"(:": " good ",
"yay!": " good ",
"yay": " good ",
"yaay": " good ",
"yaaay": " good ",
"yaaaay": " good ",
"yaaaaay": " good ",
":/": " bad ",
":&gt;": " sad ",
":')": " sad ",
":-(": " bad ",
":(": " bad ",
":s": " bad ",
":-s": " bad ",
"&lt;3": " heart ",
":d": " smile ",
":p": " smile ",
":dd": " smile ",
"8)": " smile ",
":-)": " smile ",
":)": " smile ",
";)": " smile ",
"(-:": " smile ",
"(:": " smile ",
":/": " worry ",
":&gt;": " angry ",
":')": " sad ",
":-(": " sad ",
":(": " sad ",
":s": " sad ",
":-s": " sad ",
r"\br\b": "are",
r"\bu\b": "you",
r"\bhaha\b": "ha",
r"\bhahaha\b": "ha",
r"\bdon't\b": "do not",
r"\bdoesn't\b": "does not",
r"\bdidn't\b": "did not",
r"\bhasn't\b": "has not",
r"\bhaven't\b": "have not",
r"\bhadn't\b": "had not",
r"\bwon't\b": "will not",
r"\bwouldn't\b": "would not",
r"\bcan't\b": "can not",
r"\bcannot\b": "can not",
r"\bi'm\b": "i am",
"m": "am",
"r": "are",
"u": "you",
"haha": "ha",
"hahaha": "ha",
"don't": "do not",
"doesn't": "does not",
"didn't": "did not",
"hasn't": "has not",
"haven't": "have not",
"hadn't": "had not",
"won't": "will not",
"wouldn't": "would not",
"can't": "can not",
"cannot": "can not",
"i'm": "i am",
"m": "am",
"i'll" : "i will",
"its" : "it is",
"it's" : "it is",
"'s" : " is",
"that's" : "that is",
"weren't" : "were not",
}
keys = [i for i in repl.keys() ]
new_train_data = []
new_test_data = []
ltr = train["comment_text"].tolist()
lte = test["comment_text"].tolist()
for i in ltr:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_train_data.append(xx)
 for i in lte:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_test_data.append(xx)
 train["new_comment_text"] = new_train_data
test["new_comment_text"] = new_test_data
print(f'[{time.time() - start_time}] Crap Removed.')
 trate = train["new_comment_text"].tolist()
tete = test["new_comment_text"].tolist()
for i, c in enumerate(trate):
trate[i] = re.sub('[^a-zA-Z ?!]+', '', str(trate[i] ).lower())
 for i, c in enumerate(tete):
tete[i] = re.sub('[^a-zA-Z ?!]+', '', tete[i])
 train["comment_text"] = trate
test["comment_text"] = tete
stem = SnowballStemmer('english')
 def token_stemmer(text):
tokens = nltk.word_tokenize(text)
 tokens = [stem.stem(item)for item in tokens]
return ' '.join(tokens)
 train['comment_text'] = train['comment_text'].apply(lambda x: token_stemmer(x))
test['comment_text'] = test['comment_text'].apply(lambda x: token_stemmer(x))
print(f'[{time.time() - start_time}] Stemming Completed.')
 abuses = pd.read_csv('.. /input/abusive-words3/abusive_words.csv', header=None)
 def has_abuse(text):
abuse_dict={} 
(abuses[0] ).apply(lambda x: abuse_dict.update({str(x): text.count(str(x)) }))
return abuse_dict
train['abuse_dict'] = train['new_comment_text'].apply(lambda x: has_abuse(x))
test['abuse_dict'] = test['new_comment_text'].apply(lambda x: has_abuse(x))
print(f'[{time.time() - start_time}] Abuses Dict Completed.')
 train['num_abuses'] = train['abuse_dict'].apply(lambda x: sum(x.values()))
test['num_abuses'] = test['abuse_dict'].apply(lambda x: sum(x.values()))
print(f'[{time.time() - start_time}] Abuses Completed.')
 merge, y_train, nrow_train = preprocess_pandas(train, test, start_time)
 meta_params = {
'desc_ngram':(1, 3),
'desc_max_f': 150000,
'desc_max_df': 0.5,
'desc_min_df': 10}
stopwords = frozenset(['the', 'a', 'an', 'is', 'it', 'this', ])
 vectorizer = FeatureUnion([ 
('comment_text_2', Pipeline([ 
('select', ItemSelector('comment_text', start_time=start_time)) , 
('transform', TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
stop_words='english',
token_pattern=r'\w{1,}',
ngram_range=(1, 1),
max_features=15000 
)) , 
('drop_cols', DropColumnsByDf(min_df=2))
])) , 
('comment_text_3', Pipeline([ 
('select', ItemSelector('comment_text', start_time=start_time)) , 
('transform', TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
stop_words='english',
ngram_range=(2, 6),
max_features=20000 
)) , 
('drop_cols', DropColumnsByDf(min_df=2))
])) , 
('abuse_dict', Pipeline([ 
('select', ItemSelector('abuse_dict', start_time=start_time)) , 
('ohe', DictVectorizer(sparse=False))
])) , 
('num_abuses', Pipeline([ 
('select', ItemSelector('num_abuses', start_time=start_time)) , 
('ohe', OneHotEncoder())
 ]))
], n_jobs=1)
 sparse_merge = vectorizer.fit_transform(merge)
 print(f'[{time.time() - start_time}] Merge vectorized')
 print(sparse_merge.shape)
 tfidf_transformer = TfidfTransformer()
X = tfidf_transformer.fit_transform(sparse_merge)
 print(f'[{time.time() - start_time}] TF/IDF completed')
 X_train = X[:nrow_train]
print(X_train.shape)
 X_test = X[nrow_train:]
del merge
del sparse_merge
del vectorizer
del tfidf_transformer
gc.collect()
X_train, X_test = intersect_drop_columns(X_train, X_test, min_df=1)
 print(f'[{time.time() - start_time}] Drop only in train or test cols: {X_train.shape[1]}')
 gc.collect()
losses = []
predictions = {'id': test['id']}
for class_name in list_classes:
train_target = train[class_name]
classifier = LogisticRegression(solver='sag')
 cv_loss = np.mean(cross_val_score(classifier, X_train, train_target, cv=3, scoring='roc_auc'))
losses.append(cv_loss)
 print('CV score for class {} is {}'.format(class_name, cv_loss))
classifier.fit(X_train, train_target)
 predictions[class_name] = classifier.predict_proba(X_test)[:, 1]
print(f'[{time.time() - start_time}] Predict Ridge completed.')
 submission = pd.DataFrame.from_dict(predictions)
 submission.to_csv("submission_ridge.csv", index=False)
 print("Evaluating the model on validation data...")