Kaggle competition: Toxic Comment Classification Challenge
%%time
rfr = RandomForestRegressor(n_estimators=200, max_depth=10, min_samples_leaf=4, max_features=0.2, n_jobs=-1, random_state=0)
 rfr_train , rfr_test , model = get_sklearn_data(train_new , test_new , rfr , train_new['y'] , col)
<init_hyperparams>
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
train = pd.read_csv('.. /input/train.csv' ).fillna(' ')
 test = pd.read_csv('.. /input/test.csv' ).fillna(' ' )
%%time
params = {
'objective': 'regression',
'metric': 'rmse',
'boosting': 'gbdt',
'learning_rate': 0.0045 ,
'verbose': 0,
'num_iterations': 500,
'bagging_fraction': 0.95,
'bagging_freq': 1,
'bagging_seed': 42,
'feature_fraction': 0.95,
'feature_fraction_seed': 42,
'max_bin': 100,
'max_depth': 3,
'num_rounds': 800
}
lgb_train, lgb_test = get_lgb_data(train_new, test_new , col , train_new['y'] , params , 800 )<prepare_x_and_y>
repl = {
"yay!": " good ",
"yay": " good ",
"yaay": " good ",
"yaaay": " good ",
"yaaaay": " good ",
"yaaaaay": " good ",
":/": " bad ",
":&gt;": " sad ",
":')": " sad ",
":-(": " frown ",
":(": " frown ",
":s": " frown ",
":-s": " frown ",
"&lt;3": " heart ",
":d": " smile ",
":p": " smile ",
":dd": " smile ",
"8)": " smile ",
":-)": " smile ",
":)": " smile ",
";)": " smile ",
"(-:": " smile ",
"(:": " smile ",
":/": " worry ",
":&gt;": " angry ",
":')": " sad ",
":-(": " sad ",
":(": " sad ",
":s": " sad ",
":-s": " sad ",
r"\br\b": "are",
r"\bu\b": "you",
r"\bhaha\b": "ha",
r"\bhahaha\b": "ha",
r"\bdon't\b": "do not",
r"\bdoesn't\b": "does not",
r"\bdidn't\b": "did not",
r"\bhasn't\b": "has not",
r"\bhaven't\b": "have not",
r"\bhadn't\b": "had not",
r"\bwon't\b": "will not",
r"\bwouldn't\b": "would not",
r"\bcan't\b": "can not",
r"\bcannot\b": "can not",
r"\bi'm\b": "i am",
"m": "am",
"r": "are",
"u": "you",
"haha": "ha",
"hahaha": "ha",
"don't": "do not",
"doesn't": "does not",
"didn't": "did not",
"hasn't": "has not",
"haven't": "have not",
"hadn't": "had not",
"won't": "will not",
"wouldn't": "would not",
"can't": "can not",
"cannot": "can not",
"i'm": "i am",
"m": "am",
"i'll" : "i will",
"its" : "it is",
"it's" : "it is",
"'s" : " is",
"that's" : "that is",
"weren't" : "were not",
}
keys = [i for i in repl.keys() ]
new_train_data = []
new_test_data = []
ltr = train["comment_text"].tolist()
lte = test["comment_text"].tolist()
for i in ltr:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_train_data.append(xx)
 for i in lte:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_test_data.append(xx)
 train["new_comment_text"] = new_train_data
test["new_comment_text"] = new_test_data
trate = train["new_comment_text"].tolist()
tete = test["new_comment_text"].tolist()
for i, c in enumerate(trate):
trate[i] = re.sub('[^a-zA-Z ?!]+', '', str(trate[i] ).lower())
 for i, c in enumerate(tete):
tete[i] = re.sub('[^a-zA-Z ?!]+', '', tete[i])
 train["comment_text"] = trate
test["comment_text"] = tete
del trate, tete
train.drop(["new_comment_text"], axis=1, inplace=True)
 test.drop(["new_comment_text"], axis=1, inplace=True )
train_new , test_new = get_additional_features(train.drop(columns = object_list),test.drop(columns = object_list), ID = True)
 train_new = train_new.sample(frac = 1 , random_state = 98)
 train_new.drop(columns=columns_drop , inplace = True)
 test_new.drop(columns = columns_drop , inplace = True)
 col = list(test_new.columns)
 y_mean = np.mean(train_new['y'])
 xgb_params = {
'n_trees': 520,
'eta': 0.0045,
'max_depth': 4,
'subsample': 0.93,
'eval_metric': 'rmse',
'base_score': y_mean,
'silent': True,
'seed': 42,
}
dtrain = xgb.DMatrix(train_new[col], train_new.y)
 dtrain_test = xgb.DMatrix(train_new[col])
 dtest = xgb.DMatrix(test_new[col])
 num_boost_rounds = 1250
model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round=num_boost_rounds)
 y_pred = model.predict(dtest )<drop_column>
train_text = train['comment_text']
test_text = test['comment_text']
all_text = pd.concat([train_text, test_text] )
stack_train = gb1_train[['label' , 'predicted']]
stack_train.columns = [['label','gbrp']]
stack_train['lgb'] = lgb_train['predicted']
stack_train['lasso'] = lasso_train['predicted']
stack_test = gb1_test[['ID' , 'y']]
stack_test.columns = [['ID' , 'gbrp']]
stack_test['lgb'] = lgb_test['y']
stack_test['lasso'] = lasso_test['y']
stack_test = stack_test.drop(['ID'] ,axis = 1)
 col = list(stack_test.columns)
 params = {
'eta': 0.005,
'max_depth': 2,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': y_mean,
'silent': 1
}
dtrain = xgb.DMatrix(stack_train[col], stack_train['label'])
 dtest = xgb.DMatrix(stack_test[col])
 xgb_cvalid = xgb.cv(params, dtrain, num_boost_round=2000, early_stopping_rounds=20, verbose_eval=50, show_stdv=True,seed=42)
 xgb_cvalid[['train-rmse-mean', 'test-rmse-mean']].plot()
print('Performance does not improve from '+str(len(xgb_cvalid)) +' rounds')
 model = xgb.train(params,dtrain,num_boost_round =900)
 pred_1 = model.predict(dtest)
 xgb.plot_importance(model, max_num_features = 3)
 plt.show()<create_dataframe>
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s): return re_tok.sub(r' \1 ', s ).split()
Average = 0.70*y_pred + 0.30*pred_1
sub = pd.DataFrame({'ID':test['ID'],'y':Average})
 sub1 = pd.DataFrame({'ID':test['ID'],'y':y_pred})
 sub2 = pd.DataFrame({'ID':test['ID'],'y':pred_1} )<set_options>
cont_patterns = [ 
(b'US', b'United States'), 
(b'IT', b'Information Technology'), 
(b'(W|w)on't', b'will not'), 
(b'(C|c)an't', b'can not'), 
(b'(I|i)'m', b'i am'), 
(b'(A|a)in't', b'is not'), 
(b'(\w+)'ll', b'\g<1> will'), 
(b'(\w+)n't', b'\g<1> not'), 
(b'(\w+)'ve', b'\g<1> have'), 
(b'(\w+)'s', b'\g<1> is'), 
(b'(\w+)'re', b'\g<1> are'), 
(b'(\w+)'d', b'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def prepare_for_char_n_gram(text):

 in r""
clean = bytes(text.lower() , encoding="utf-8")
 and \t
clean = clean.replace(b"
", b" ")
 clean = clean.replace(b"\t", b" ")
 clean = clean.replace(b"\b", b" ")
 clean = clean.replace(b"\r", b" ")
 for(pattern, repl)in patterns:
clean = re.sub(pattern, repl, clean)
 exclude = re.compile(b'[%s]' % re.escape(bytes(string.punctuation, encoding='utf-8')))
 clean = b" ".join([exclude.sub(b'', token)for token in clean.split() ])
 clean = re.sub(b"\d+", b" ", clean)
 clean = re.sub(b'\s+', b' ', clean)
 clean = re.sub(b'\s+$', b'', clean)
 clean = re.sub(b" ", b"
clean = b"
return str(clean, 'utf-8')
 def count_regexp_occ(regexp="", text=None):

 return len(re.findall(regexp, text))
%matplotlib inline<load_from_csv>
def get_indicators_and_clean_comments(df):

 ", x))
df["raw_word_len"] = df["comment_text"].apply(lambda x: len(x.split()))
df["raw_char_len"] = df["comment_text"].apply(lambda x: len(x))
df["nb_upper"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[A-Z]", x))
df["nb_fk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ff]\S{2}[Kk]", x))
df["nb_sk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ss]\S{2}[Kk]", x))
df["nb_dk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[dD]ick", x))
df["nb_you"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\W[Yy]ou\W", x))
df["nb_mother"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wmother\W", x))
df["nb_ng"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wnigger\W", x))
df["start_with_columns"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"^\:+", x))
df["has_timestamp"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\d{2}|:\d{2}", x))
df["has_date_long"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{2}:\d{2}, \d{1,2} \w+ \d{4}", x))
df["has_date_short"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{1,2} \w+ \d{4}", x))
df["has_mail"] = df["comment_text"].apply(
 lambda x: count_regexp_occ(r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+', x)
)
 df["has_emphasize_equal"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\={2}.+\={2}", x))
df["has_emphasize_quotes"] = df["comment_text"].apply(lambda x: count_regexp_occ(r""{4}\S+"{4}", x))
df["clean_comment"] = df["comment_text"].apply(lambda x: prepare_for_char_n_gram(x))
df["clean_word_len"] = df["clean_comment"].apply(lambda x: len(x.split()))
df["clean_char_len"] = df["clean_comment"].apply(lambda x: len(x))
df["clean_chars"] = df["clean_comment"].apply(lambda x: len(set(x)))
 df["clean_chars_ratio"] = df["clean_comment"].apply(lambda x: len(set(x)))/ df["clean_comment"].apply(
 lambda x: 1 + min(99, len(x)) )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
num_features = [f_ for f_ in train.columns
if f_ not in ["comment_text", "clean_comment", "id", "remaining_chars", 'has_ip_address'] + class_names]
for f in num_features:
all_cut = pd.cut(pd.concat([train[f], test[f]], axis=0), bins=20, labels=False, retbins=False)
 train[f] = all_cut.values[:train.shape[0]]
test[f] = all_cut.values[train.shape[0]:]
train_num_features = train[num_features].values
test_num_features = test[num_features].values
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<count_unique_values>
train_text = train['clean_comment'].fillna("")
 test_text = test['clean_comment'].fillna("")
 all_text = pd.concat([train_text, test_text] )
uniq_vals = {}
for col in cat_cols:
uniq_vals[col] = len(train[col].unique())
 cat_cols<concatenate>
word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
tokenizer=lambda x: regex.findall(r'[^\p{P}\W]+', x),
analyzer='word',
token_pattern=None,
min_df=5,
ngram_range=(1, 2),
max_features=60000)
train['eval_set'] = 0; test['eval_set'] = 1
df = pd.concat([train, test], axis=0, copy=True,sort = True)
 df.reset_index(drop=True, inplace=True)
 df<groupby>
word_vectorizer.fit(all_text )
def add_new_col(x):
if x not in new_col.keys() :
return int(len(new_col.keys())/2)
 return new_col[x]
for c in cat_cols:
new_col = train_cleaned.groupby(c ).y.mean().sort_values().reset_index()
new_col = new_col.reset_index().set_index(c ).drop('y', axis=1)['index'].to_dict()
df[c + '_new'] = df[c].apply(add_new_col)
 df_new = df.drop(cat_cols, axis=1)
 df_new.head()<prepare_output>
train_word_features = word_vectorizer.transform(train_text)
 test_word_features = word_vectorizer.transform(test_text )
import xgboost as xgb
from xgboost import XGBRegressor
from sklearn.metrics import r2_score
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold 
<compute_train_metric>
train_features = hstack([train_word_features, train_links_n, train_num_features] ).tocsr()
test_features = hstack([test_word_features, test_links_n, test_num_features] ).tocsr()
xgb_params = {
'n_trees': 500,
'eta': 0.005,
'max_depth': 3,
'subsample': 0.9,
'colsample_bytree': 0.6,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': np.log(y_mean),
'silent': 1
}
dtrain = xgb.DMatrix(X_train, np.array(np.log(y_train)))
 dtest = xgb.DMatrix(X_test)
 def the_metric(y_pred, y):
y_true = y.get_label()
return 'r2', r2_score(y_true, y_pred)
 cv_result = xgb.cv(xgb_params,
dtrain,
num_boost_round=2000,
nfold = 3,
early_stopping_rounds=50,
feval=the_metric,
verbose_eval=100,
show_stdv=False 
)
 num_boost_rounds = len(cv_result)
 print('num_boost_rounds=' + str(num_boost_rounds))
model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round=num_boost_rounds)
 y_train_pred = np.exp(model.predict(dtrain))
y_pred = np.exp(model.predict(dtest))
print('First 5 predicted test values:', y_pred[:5])
<save_to_csv>
X_train, X_test, y_train, y_test = train_test_split(train_features, train['toxic'], test_size=0.3, random_state=42 )
output = pd.DataFrame({'id': test['ID'].astype(np.int32), 'y': y_pred})
 output.to_csv('sub_16_encoded.csv', index=False )<set_options>
lgb_train = lgb.Dataset(X_train , y_train)
 lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
 params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective':'binary',
'metric': {'auc'},
'nthread': -1,
'feature_fraction': 0.4,
'num_leaves': 50,
'verbose': 0
}
print('Start training...')
 gbm = lgb.train(params,
lgb_train,
num_boost_round=200,
valid_sets=lgb_eval,
early_stopping_rounds=20 )
rcParams['figure.figsize'] = 50,20
img=cv2.imread(".. /input/private-score/score.JPG")
 plt.imshow(img )<load_from_csv>
all_parameters = {
'C' : [1.048113, 0.1930, 0.596362, 0.25595, 0.449843, 0.25595],
'tol' : [0.1, 0.1, 0.046416, 0.0215443, 0.1, 0.01],
'solver' : ['lbfgs', 'newton-cg', 'lbfgs', 'newton-cg', 'newton-cg', 'lbfgs'],
'fit_intercept' : [True, True, True, True, True, True],
'penalty' : ['l2', 'l2', 'l2', 'l2', 'l2', 'l2'],
'class_weight' : [None, 'balanced', 'balanced', 'balanced', 'balanced', 'balanced'],
}
train = pd.read_csv('.. /input/mercedes-benz-greener-manufacturing/train.csv')
 test = pd.read_csv('.. /input/mercedes-benz-greener-manufacturing/test.csv')
 df = train<count_unique_values>
submission = pd.DataFrame.from_dict({'id': test['id']} )
for i in df_cat.columns:
print('The unique values in '+i+' are: ',df[i].nunique() ,'
',df_cat[i].unique() ,'
',"--------------------------------------------------------------------------------" )<count_missing_values>
for j, class_name in enumerate(class_names):
classifier = LogisticRegression(
 C=all_parameters['C'][j],
max_iter=200,
tol=all_parameters['tol'][j],
solver=all_parameters['solver'][j],
fit_intercept=all_parameters['fit_intercept'][j],
penalty=all_parameters['penalty'][j],
dual=False,
class_weight=all_parameters['class_weight'][j],
verbose=0)
 train_target = train[class_name]
classifier.fit(train_features, train_target)
 submission[class_name] = classifier.predict_proba(test_features)[:, 1]
print(class_name )
print(df.isnull().sum().sum(axis=0))<feature_engineering>
submission.to_csv('submission.csv', index=False )
<filter><EOS> 
submission.to_csv('submission.csv', index=False )